commit 92a66db3df0fc8d544b6b5301e46208c7cd99df8 Author: Cole Speelman Date: Tue Aug 18 21:47:08 2026 -0400 docs: publish Preview 19 dogfood evidence Export the reviewed allowlisted snapshot from private source commit 05928cebd01b586cf9e9d4b8c8537a7605a6068c. This records the exact candidate, bounded capacity result, stateful migration scratch requirement, authenticated batch identity proof, and immediate live acceptance evidence. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..119b43f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: AGPL-3.0-only +* text=auto +*.go text eol=lf +*.sando text eol=lf +*.sh text eol=lf +*.md text eol=lf +*.json text eol=lf +*.svg text eol=lf +*.css text eol=lf +*.js text eol=lf +LICENSE text eol=lf + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..97f7a6c --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: AGPL-3.0-only +/bin/ +/dist/ +/.cache/ +*.local.json +*.db +*.db-shm +*.db-wal +/test/browser/node_modules/ +/test/browser/playwright-report/ +/test/browser/test-results/ diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..a67600f --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,4 @@ +Copyright 2026 Cole Speelman + +Except where a file or subtree says otherwise, this repository is licensed +under AGPL-3.0-only. Reusable examples under examples/ are licensed 0BSD. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2beb9e1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,662 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + diff --git a/LICENSES.md b/LICENSES.md new file mode 100644 index 0000000..8123404 --- /dev/null +++ b/LICENSES.md @@ -0,0 +1,20 @@ + + +# Licence map + +| Path | Licence | +| --- | --- | +| Go application and internal packages | AGPL-3.0-only | +| `scripts/`, `.gitea/`, documentation, and release machinery | AGPL-3.0-only | +| `examples/` | 0BSD | +| Browser campaign source and tooling | AGPL-3.0-only | + +The browser campaign pins Apache-2.0-licensed Playwright as a development-only +dependency. Its optional macOS watcher is MIT-licensed. Neither is linked into +or packaged with the Observatory binary. + +Every text source file carries an SPDX identifier or, where strict JSON cannot +accept comments, a matching `.license` sidecar or machine-readable licence +field. Generated Sandwich Hime outputs inherit the licence of their adjacent +`.sando` source and retain exact generator provenance. The complete AGPL and +0BSD licence texts are included in `LICENSE` and `examples/LICENSE`. diff --git a/PUBLIC-SNAPSHOT.json b/PUBLIC-SNAPSHOT.json new file mode 100644 index 0000000..562875a --- /dev/null +++ b/PUBLIC-SNAPSHOT.json @@ -0,0 +1 @@ +{"schema_version":1,"source_commit":"05928cebd01b586cf9e9d4b8c8537a7605a6068c","source_tree":"4f1a87967c842c19141c03018e9350261f4d2a50","source_date_epoch":1787103921,"file_count":199} diff --git a/PUBLIC-SNAPSHOT.sha256 b/PUBLIC-SNAPSHOT.sha256 new file mode 100644 index 0000000..96b23fd --- /dev/null +++ b/PUBLIC-SNAPSHOT.sha256 @@ -0,0 +1 @@ +4757b0ca0e9ae9930006510bc1e277d8adb4398abf7b27b271704028dee7d5fb PUBLIC-SNAPSHOT.json diff --git a/README.md b/README.md new file mode 100644 index 0000000..5b2aea4 --- /dev/null +++ b/README.md @@ -0,0 +1,159 @@ + + +# Gamertan Observatory + +Gamertan Observatory is a self-hosted, organization-aware observability platform in active private development. It is being built for logs, metrics, traces, incidents, and deployment evidence on modest Linux servers. + +This repository currently contains the first security and durability vertical slice. It is not a public preview yet. + +## Current development surface + +- strict, versioned server configuration; +- organization/project/environment/service-scoped ingestion sources; +- hashed, rotatable source credentials; +- bounded log, metric, trace, and deployment batches; +- immutable checksummed zstd raw segments committed before acknowledgement, + with an atomic cold forensic tier instead of destructive hot-window expiry; +- SQLite control state and independently migratable per-organization projections; +- sequence deduplication, replay rejection, and crash recovery; +- enforced organization-aware hot/cold retention, crash-recoverable archival, + indefinite forensic preservation by default, explicitly enabled final + retirement, projection compaction, budgeted cold queries, and + queryable five-minute metric rollups; +- an exact five-minute HTTP-status/normalized-route log projection for the + common error-count view, with one bounded raw fragment at an unaligned time + boundary rather than a whole-window rescan; +- an offline, explicitly approved per-organization projection rebuild that + validates checksummed raw truth, rebuilds activated descriptors beside the + live database, and atomically replaces only the disposable projection; +- pinned Web Foundations `v0.1.0-preview.3` users, secure sessions, + personal organizations, teams, invitations, and scoped access grants; +- local single-use operator bootstrap with an application-generated one-time + credential, mandatory password replacement, and transactional session + revocation; platform operation remains deliberately separated from + telemetry access; +- a typed, bounded query AST and parser; +- one validated AST shared by text and visual-builder inputs, including safe + regular expressions, aggregates, grouping, windows, sorting, and limits; +- explain planning that injects resource scope independently, resolves field + descriptors, reports indexes and estimated scan, enforces budgets, and + requires a separate permission for sensitive or unknown fields; +- bounded typed execution over per-organization projections, including + filters, safe regular expressions, lookback windows, aggregation buckets, + grouping, exact raw-sample percentiles, explicitly marked approximate + rollup percentiles, sorting, and stable tabular results; +- same-origin session, health, readiness, native ingestion, query-parse, + query-explain, and authenticated query-execution HTTP endpoints; +- strict agent configuration, private outage spooling, and HTTPS delivery; +- continuous bounded file tailing with durable cursors, truncation/rename + handling, independent stream ordering, and crash recovery from spool + checkpoints; +- short-lived, single-use, scope-bound agent enrollment and self-revocation if + a newly issued credential cannot be persisted; +- whitelist-only adapters for Caddy access JSON, Web Foundations requestlog + JSONL, and Tend deployment events; minimized defaults omit queries, + addresses, referrers, user agents, cookies, credentials, and arbitrary + fields, while explicitly configured privacy-policy-backed fields remain + bounded and individually selectable. +- authenticated OTLP/HTTP protobuf endpoints for logs, metrics, and traces, + with bounded identity/gzip decoding, source-owned scope, secret-key + deny rules, typed signal conversion, and automatic per-stream sequencing. +- unprivileged Linux host metrics from bounded `/proc`, cgroup v2, named + filesystems, and explicitly selected PID files; no shell, root agent, + Docker socket, environment read, or command-line capture is involved. +- idempotent per-organization descriptor proposals aggregated once per raw + segment; unknown fields remain sensitive, high-cardinality, and unindexed + until an organization owner reviews them; +- explicit descriptor activation from a strict private review file, with each + custom index built beside the active per-organization version and switched + atomically; prior index versions remain intact and later ingestion follows + the active descriptor registry; +- versioned saved queries and dashboard definitions with organization-scoped + panel references, optimistic revisions, server-generated identifiers, and a + source-control export that omits tenant and operator runtime metadata; +- authorized CLI export and explicitly approved atomic import of those strict + dashboard bundles, with all destination identities and resource scopes + independently revalidated; +- a server-rendered Sandwich Hime interface for public orientation, local + sign-in, organization selection, recent log/metric/trace/deployment tables, + typed-text and assisted saved-query creation, one-panel dashboard creation, + bounded dashboard execution, accessible visual summaries with full table + alternatives, and strict source-control-safe JSON export; +- a bounded organization-authorized SSE invalidation channel that carries only + generic refresh events; the interface, tables, navigation, and sign-out flow + remain functional without JavaScript; +- durable organization-scoped alert rules that execute bounded saved-query + ASTs, open incidents after configurable consecutive matches, and preserve + pending, firing, acknowledged, silenced, and resolved lifecycle events; +- an accessible server-rendered incident inbox with ordinary CSRF-protected + response forms and no telemetry or incident detail in SSE payloads; +- an installable PWA shell, application badge state, and an explicit opt-in + read-only offline incident snapshot that excludes response capabilities and + high-risk private fields; +- optional privacy-preserving Web Push with an explicit user gesture, + organization-scoped subscriptions, delivery-time authorization, a bounded + non-blocking queue, and one fixed generic encrypted message; +- content-addressed static assets and a strict CSP with no inline-style or + inline-script exception; the production Sandwich Hime dependency boundary + contains only the pinned Apache-2.0 Sando runtime, not the Hime-san compiler. +- a Tend schema-2 singleton deployment contract with a stateless, loopback-only + candidate that cannot open or mutate Observatory data, followed by routed + public-origin validation and automatic last-good restoration. + +The server injects resource scope from an authenticated source. Payloads cannot select an organization or resource. Unknown fields remain stored in raw truth, but are not indexed by default. + +## Deliberate current limits + +Self-service invitation delivery is deliberately excluded from the first +preview, and the visual query builder remains deliberately smaller than the +complete typed query language. Tend activation, rollback, and identical- +artifact redeployment have been exercised against the live Observatory +service. The separate medium-fleet agent soak and browser-vendor Web Push +delivery evidence are not complete. No release tag should be created until the +published preview gates in `docs/ROADMAP.md` are satisfied. + +## Development + +```sh +./scripts/preview-gate.sh +``` + +Linux is the supported deployment platform. Development requires Go 1.26.6 with `GOTOOLCHAIN=local`. + +The local first-operator procedure is documented in +[`docs/BOOTSTRAP.md`](docs/BOOTSTRAP.md). It is a development surface, not yet +a public-preview installation promise. + +The typed text/builder query contract and authorized local workflow are +documented in [`docs/QUERY.md`](docs/QUERY.md). +Descriptor discovery and its current review boundary are documented in +[`docs/SCHEMA.md`](docs/SCHEMA.md). +Retention, approved per-organization overrides, quota enforcement, and metric +rollup behavior are documented in [`docs/RETENTION.md`](docs/RETENTION.md). +The bounded preview gate includes the complete verifier, short fuzz smoke, and +a development capacity proof. Extended security, release-scale capacity, fleet, +and soak campaigns remain explicit milestone evidence. See +[`docs/CAPACITY.md`](docs/CAPACITY.md). +The retained optimization ledger and ordered lower-level storage case studies +are documented in [`docs/PERFORMANCE.md`](docs/PERFORMANCE.md). +The proposed raw-first, lazy/eager time-partitioned read model is documented as +an unimplemented design in +[`docs/ADAPTIVE_PROJECTIONS.md`](docs/ADAPTIVE_PROJECTIONS.md). +The persisted dashboard model and export boundary are documented in +[`docs/DASHBOARDS.md`](docs/DASHBOARDS.md). +The current Sandwich Hime interface and progressive-enhancement boundary are +documented in [`docs/INTERFACE.md`](docs/INTERFACE.md). +The bounded rule evaluator and durable response lifecycle are documented in +[`docs/INCIDENTS.md`](docs/INCIDENTS.md). +The installable shell and opt-in offline privacy boundary are documented in +[`docs/PWA.md`](docs/PWA.md). +The strict, non-authoritative Tend deployment-evidence boundary is documented +in [`docs/TEND.md`](docs/TEND.md). +The application deployment and isolated candidate boundary are documented in +[`docs/DEPLOYMENT.md`](docs/DEPLOYMENT.md). +Offline corruption recovery and projection rebuild are documented in +[`docs/RECOVERY.md`](docs/RECOVERY.md). + +## Licensing + +Application, agent, query, storage, migration, and release code is AGPL-3.0-only. Copyable files under `examples/` are 0BSD. See `LICENSES.md`. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..d17177f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,110 @@ + + +# Security policy + +Observatory is pre-release software and has no supported public version yet. Please report suspected vulnerabilities privately to `security@sandwichhime.com`. Do not include credentials, production telemetry, raw request bodies, personal data, or exploit traffic in an initial report. + +The maintainer aims to acknowledge a report within three business days, provide an initial triage within seven business days, and send progress updates at least every fourteen days. These are best-effort targets, not a service-level agreement or bounty promise. + +## Current trust boundary + +Agents, server operators, enrolled source configuration, the host kernel, SQLite, and the pinned Go dependency graph are trusted. Telemetry payloads, compressed input, query text, files being tailed, archive paths, and network peers are untrusted. + +Current demonstrated controls include: + +- source scope is loaded from a hashed credential rather than payload fields; +- production configuration and credentials remain absolute, root-owned, + mode-`0600` regular files; the unprivileged agent consumes systemd-provided + runtime copies confined to `CREDENTIALS_DIRECTORY`; +- decompressed input, batches, records, attributes, query stages, rows, memory, time, and scanned bytes have explicit bounds; +- OTLP/HTTP authenticates a source before parsing, accepts protobuf over + identity or gzip only, independently caps compressed and decompressed bytes, + discards known credential-bearing attribute keys, and derives tenant scope + exclusively from the enrolled source; +- raw segments are checksummed, compressed, synchronized, and atomically renamed before acknowledgement; +- sequence gaps, old replay, and altered duplicate sequences fail closed; +- organization projections are separate SQLite files; +- control and projection databases, SQLite sidecars, process locks, agent + state, spool entries, tailed files, and reviewed descriptors reject + symlinks; database and lock files also reject additional hard links; +- live commands hold a shared data-directory lock while offline migration and + projection replacement require exclusive ownership; +- error responses do not echo source values; +- browser sessions use opaque digest-backed credentials in Secure, HttpOnly, + `SameSite=Strict`, `__Host-` cookies. Ordinary HTML forms require a + purpose-bound CSRF token; JSON and JavaScript endpoints independently + require canonical same-origin evidence; +- platform operator permission grants no telemetry access; query planning + separately checks an organization/resource-scoped grant and sensitive-field + permission; +- invitations are single-use and expiring, team grants remain organization + scoped, revoked bindings stop authorizing, and short-lived break-glass + access creates an organization-visible append-only audit event; +- first-operator bootstrap is local and serialized by an OS lock. The primary + flow generates a one-time credential into a new mode-`0600`, non-symlink + file; the account cannot use application or API features until replacing + that credential, which revokes every session and requires a fresh login; +- administrative credential recovery is local-only and creates a new private + file rather than accepting a password argument. Credential replacement, + mandatory rotation, all-session revocation, and a secret-free audit event + commit atomically; Observatory exposes no public reset endpoint; +- every HTML form uses an ordinary server POST and a session- or cookie-bound + purpose token as its primary CSRF proof. Origin and Fetch Metadata provide a + second contradiction check: explicit cross-site evidence and tokenless + requests fail closed, while absent or opaque browser metadata cannot strand + a valid server-rendered form; +- initial file adapters use positive field whitelists and remove query strings, + credentials, cookies, client addresses, bodies, and arbitrary headers before + durable agent spooling; +- Linux metrics read a fixed bounded file set without shell execution, + enumerate no processes, require explicit PID-file/cgroup/filesystem + selectors, reject symlink sources, and retain stable configured names rather + than local paths, command lines, environments, or raw PIDs; +- spool deletion requires the local content digest and a path beneath the + private pending root. +- collected file records advance only after an envelope containing the next + cursor is durably spooled; discarded complete records advance only after the + cursor and bounded counters are saved, and startup repairs cursors from + pending envelopes before reading more source bytes; +- collectors refuse symlink sources, cap one cycle's read work, preserve + partial lines, and detect truncation and same-directory inode rotation; +- enrollment tokens are hashed, short-lived, scope-bound, single-use, and + exchanged over a redirect-refusing HTTPS client; a source credential can + revoke itself when local persistence fails; +- visual and textual queries share one validated AST; organization scope is a + separate planner input, unknown fields require sensitive-data permission, + projection files are opened in query-only mode, and execution enforces + independent time, decoded-byte, memory, and result-row budgets. +- unknown fields generate only idempotent per-segment proposal counts, byte + estimates, inferred types, and generic example queries; proposals retain no + observed values, default to sensitive/unindexed, and require an independent + organization-scoped schema-management permission to review; batches with + more than 1,024 distinct attribute keys are rejected before raw commit. +- reviewed descriptor files are bounded, strict JSON held in private regular + non-symlink files; activation caps the active descriptor registry, validates + typed values without coercing failures, builds a new index beside the live + version, and atomically switches only after the complete build succeeds; + ingestion and activation coordinate per organization, and prior versions + remain intact. +- saved query text is bounded and parsed before storage, its versioned typed + AST is revalidated against the text on every read, and dashboard panel + foreign keys include the organization boundary; optimistic revisions reject + lost updates, and dashboard exports omit organization/operator runtime + metadata. +- an approved offline rebuild reads every organization segment through the + checksum-verifying raw store, reconstructs base and activated descriptor + projections beside live, and atomically replaces only that organization's + disposable projection; corrupt raw truth fails closed before activation. +- PWA offline incident snapshots require an explicit browser action and omit + response capabilities and high-risk fields; optional Web Push reauthorizes + at delivery and encrypts one fixed generic sentence through a bounded, + non-blocking queue. + +This is a maintainer self-assessment, not an independent audit. Invitation UI, +the real-browser PWA campaign, production deployment, and the capacity/fleet +soak remain unfinished and must not be described as supported yet. Bootstrap +is intentionally single-use; a storage failure after the user row is +committed but before all personal-organization grants complete currently +requires restoring the empty control database before retrying. The current +executable evidence map is in +[`docs/SECURITY_CAMPAIGN.md`](docs/SECURITY_CAMPAIGN.md). diff --git a/cmd/observatory/main.go b/cmd/observatory/main.go new file mode 100644 index 0000000..39a73f5 --- /dev/null +++ b/cmd/observatory/main.go @@ -0,0 +1,1607 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "bytes" + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + "net" + "net/http" + "net/netip" + "os" + "os/signal" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + obsagent "gamertan.com/observatory/internal/agent" + "gamertan.com/observatory/internal/agentclient" + "gamertan.com/observatory/internal/config" + "gamertan.com/observatory/internal/httpserver" + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" + "gamertan.com/observatory/internal/storage" + "gamertan.com/observatory/internal/version" + "gamertan.com/observatory/internal/webpush" + "gamertan.com/web/access" + "gamertan.com/web/auth" + "gamertan.com/web/organizations" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "observatory:", err) + os.Exit(1) + } +} + +func run(args []string) error { + if len(args) == 0 { + if address := os.Getenv("OBSERVATORY_TEND_CANDIDATE_LISTEN"); address != "" { + return serveTendCandidate(address) + } + return errors.New("command required: check, server, agent, admin, migrate, query, export, import, version") + } + switch args[0] { + case "version": + fs := flag.NewFlagSet("version", flag.ContinueOnError) + asJSON := fs.Bool("json", false, "print machine-readable version information") + if err := fs.Parse(args[1:]); err != nil { + return err + } + info := version.Current() + if *asJSON { + return json.NewEncoder(os.Stdout).Encode(info) + } + fmt.Printf("observatory %s (%s, %s)\n", info.Version, info.Commit, info.Go) + return nil + case "check": + cfg, err := commandConfig(args[0], args[1:]) + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + if err := store.Recover(context.Background()); err != nil { + return err + } + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + fmt.Println("ok") + return nil + case "migrate": + return migrateCommand(args[1:]) + case "server": + cfg, err := commandConfig("server", args[1:]) + if err != nil { + return err + } + return serve(cfg) + case "admin": + return adminCommand(args[1:]) + case "agent": + return agentCommand(args[1:]) + case "query": + return queryCommand(args[1:]) + case "export": + return exportCommand(args[1:]) + case "import": + return importCommand(args[1:]) + default: + return fmt.Errorf("unknown command %q", args[0]) + } +} + +func serveTendCandidate(value string) error { + address, err := netip.ParseAddrPort(value) + if err != nil || !address.Addr().IsLoopback() || address.Port() == 0 { + return errors.New("OBSERVATORY_TEND_CANDIDATE_LISTEN must be a loopback IP address with a nonzero port") + } + server := &http.Server{ + Addr: address.String(), + Handler: tendCandidateHandler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 30 * time.Second, + MaxHeaderBytes: 16 << 10, + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + result := make(chan error, 1) + go func() { result <- server.ListenAndServe() }() + select { + case listenErr := <-result: + if errors.Is(listenErr, http.ErrServerClosed) { + return nil + } + return listenErr + case <-ctx.Done(): + shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return server.Shutdown(shutdown) + } +} + +func tendCandidateHandler() http.Handler { + const marker = "Gamertan Observatory candidate\n" + plain := func(body string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body))) + if r.Method != http.MethodHead { + _, _ = io.WriteString(w, body) + } + } + } + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", plain("ok\n")) + mux.HandleFunc("HEAD /healthz", plain("ok\n")) + mux.HandleFunc("GET /readyz", plain("ready\n")) + mux.HandleFunc("HEAD /readyz", plain("ready\n")) + mux.HandleFunc("GET /{$}", plain(marker)) + mux.HandleFunc("HEAD /{$}", plain(marker)) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Content-Security-Policy", "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'") + w.Header().Set("Cross-Origin-Opener-Policy", "same-origin") + w.Header().Set("Permissions-Policy", "camera=(), geolocation=(), microphone=()") + w.Header().Set("Referrer-Policy", "no-referrer") + w.Header().Set("X-Content-Type-Options", "nosniff") + mux.ServeHTTP(w, r) + }) +} + +func exportCommand(args []string) error { + fs := flag.NewFlagSet("export", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow a non-root config owner for local development") + actor := fs.String("actor-user-id", "", "authorized user exporting the dashboard") + organizationID := fs.String("organization-id", "", "organization identifier") + dashboard := fs.String("dashboard", "", "dashboard slug") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("export accepts no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + scope := access.Scope{OrganizationID: *organizationID} + if err = identities.ValidateResourceScope(context.Background(), scope); err != nil { + return err + } + decision, err := identities.Access.Authorize(context.Background(), *actor, scope, identity.PermissionDashboardsRead) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to export dashboards for the requested organization") + } + bundle, err := store.ExportDashboard(context.Background(), *organizationID, *dashboard) + if err != nil { + return err + } + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + return encoder.Encode(bundle) +} + +func importCommand(args []string) error { + fs := flag.NewFlagSet("import", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow a non-root config owner for local development") + actor := fs.String("actor-user-id", "", "authorized user importing the dashboard") + organizationID := fs.String("organization-id", "", "destination organization identifier") + approveOrganization := fs.String("approve-organization", "", "exact destination organization identifier approving the import") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("import accepts no positional arguments; provide one dashboard bundle on stdin") + } + if *organizationID == "" || *approveOrganization != *organizationID { + return errors.New("dashboard import requires the exact organization identifier in approve-organization") + } + bundle, err := readDashboardBundle(os.Stdin) + if err != nil { + return err + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + scope := access.Scope{OrganizationID: *organizationID} + if err = identities.ValidateResourceScope(context.Background(), scope); err != nil { + return err + } + decision, err := identities.Access.Authorize(context.Background(), *actor, scope, identity.PermissionDashboardsManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to import dashboards for the requested organization") + } + for _, definition := range bundle.SavedQueries { + requested := access.Scope{OrganizationID: *organizationID, ProjectID: definition.Scope.ProjectID, EnvironmentID: definition.Scope.EnvironmentID, ServiceID: definition.Scope.ServiceID} + if err = identities.ValidateResourceScope(context.Background(), requested); err != nil { + return errors.New("dashboard import references an unavailable resource scope") + } + } + imported, err := store.ImportDashboard(context.Background(), storage.DashboardImportInput{OrganizationID: *organizationID, ActorUserID: *actor, MaxRows: cfg.Query.MaxRows, Bundle: bundle}, time.Now().UTC()) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(struct { + Version int `json:"version"` + OrganizationID string `json:"organization_id"` + DashboardID string `json:"dashboard_id"` + Slug string `json:"slug"` + }{storage.DashboardVersion, imported.OrganizationID, imported.ID, imported.Slug}) +} + +func readDashboardBundle(reader io.Reader) (storage.DashboardExport, error) { + const maximum = 1 << 20 + body, err := io.ReadAll(io.LimitReader(reader, maximum+1)) + if err != nil || len(body) > maximum { + return storage.DashboardExport{}, errors.New("dashboard import must contain one JSON value no larger than 1 MiB") + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + var bundle storage.DashboardExport + if err = decoder.Decode(&bundle); err != nil { + return storage.DashboardExport{}, errors.New("dashboard import JSON is invalid") + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return storage.DashboardExport{}, errors.New("dashboard import must contain one JSON value no larger than 1 MiB") + } + return bundle, nil +} + +func migrateCommand(args []string) error { + fs := flag.NewFlagSet("migrate", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow a non-root config owner for local development") + rebuildOrganization := fs.String("rebuild-organization", "", "organization projection to rebuild from raw truth") + approveRebuildOrganization := fs.String("approve-rebuild-organization", "", "exact organization identifier approving destructive projection replacement") + applyRetention := fs.Bool("apply-retention", false, "apply configured retention and compaction after recovery") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("migrate accepts no positional arguments") + } + if *rebuildOrganization == "" && *approveRebuildOrganization != "" { + return errors.New("approve-rebuild-organization requires rebuild-organization") + } + if *rebuildOrganization != "" && *approveRebuildOrganization != *rebuildOrganization { + return errors.New("projection rebuild requires the exact organization identifier in approve-rebuild-organization") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, true) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + var report *storage.RebuildReport + var retentionReport *storage.RetentionReport + if *rebuildOrganization != "" { + rebuilt, rebuildErr := store.RebuildOrganization(context.Background(), *rebuildOrganization, time.Now().UTC()) + if rebuildErr != nil { + return rebuildErr + } + report = &rebuilt + } + if err = store.Recover(context.Background()); err != nil { + return err + } + if *applyRetention { + retained, retentionErr := store.ApplyRetention(context.Background(), retentionPolicy(cfg), time.Now().UTC()) + if retentionErr != nil { + return retentionErr + } + retentionReport = &retained + } + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + if *applyRetention { + if _, err = identities.PruneEvidence(context.Background(), cfg.Retention.EvidenceDays, time.Now().UTC()); err != nil { + return err + } + } + if report != nil && retentionReport != nil { + return json.NewEncoder(os.Stdout).Encode(struct { + Rebuild *storage.RebuildReport `json:"rebuild"` + Retention *storage.RetentionReport `json:"retention"` + }{report, retentionReport}) + } + if report != nil { + return json.NewEncoder(os.Stdout).Encode(report) + } + if retentionReport != nil { + return json.NewEncoder(os.Stdout).Encode(retentionReport) + } + fmt.Println("ok") + return nil +} + +func queryCommand(args []string) error { + fs := flag.NewFlagSet("query", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root config ownership for local development") + actor := fs.String("actor-user-id", "", "authorized user executing the local query") + organizationID := fs.String("organization-id", "", "organization identifier") + projectID := fs.String("project-id", "", "optional project identifier") + environmentID := fs.String("environment-id", "", "optional environment identifier") + serviceID := fs.String("service-id", "", "optional service identifier") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("query accepts no positional arguments; provide query text on stdin") + } + text, err := io.ReadAll(io.LimitReader(os.Stdin, 16_385)) + if err != nil || len(text) > 16_384 { + return errors.New("query input exceeds 16384 bytes") + } + ast, err := query.Parse(strings.TrimSpace(string(text)), 10_000) + if err != nil { + return err + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + requested := access.Scope{OrganizationID: *organizationID, ProjectID: *projectID, EnvironmentID: *environmentID, ServiceID: *serviceID} + decision, err := identities.Access.Authorize(context.Background(), *actor, requested, identity.PermissionTelemetryQuery) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to query the requested scope") + } + if err = identities.ValidateResourceScope(context.Background(), requested); err != nil { + return err + } + sensitive, err := identities.Access.Authorize(context.Background(), *actor, requested, identity.PermissionTelemetryReadSensitive) + if err != nil { + return err + } + result, err := store.Query(context.Background(), ast, query.Scope{ + OrganizationID: *organizationID, ProjectID: *projectID, + EnvironmentID: *environmentID, ServiceID: *serviceID, + Sensitive: sensitive.Allowed, + }, query.Budget{MaxDuration: cfg.Query.MaxDuration, MaxRows: cfg.Query.MaxRows, MaxScannedBytes: cfg.Query.MaxScannedBytes, MaxMemoryBytes: cfg.Query.MaxMemoryBytes}, time.Now().UTC()) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(result) +} + +func agentCommand(args []string) error { + if len(args) > 0 && args[0] == "enroll" { + return agentEnrollCommand(args[1:]) + } + fs := flag.NewFlagSet("agent", flag.ContinueOnError) + path := fs.String("config", "/etc/gamertan-observatory/agent.json", "absolute agent configuration path") + credentialPath := fs.String("credential-file", "", "absolute credential path overriding credential_file") + systemdCredentials := fs.Bool("systemd-credentials", false, "read config and credential from CREDENTIALS_DIRECTORY") + once := fs.Bool("once", false, "run one collect-and-deliver cycle") + unsafeOwner := fs.Bool("development-allow-nonroot-config", false, "allow non-root config and credential ownership for local development") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("agent accepts no positional arguments") + } + policy := config.FilePolicy{RequireRoot: !*unsafeOwner} + if *systemdCredentials { + if *unsafeOwner { + return errors.New("systemd-credentials and development-allow-nonroot-config are mutually exclusive") + } + var err error + policy, err = config.SystemdCredentialPolicy() + if err != nil { + return err + } + } + cfg, err := config.LoadAgent(*path, policy) + if err != nil { + return err + } + selectedCredentialPath := cfg.CredentialFile + if *credentialPath != "" { + selectedCredentialPath = *credentialPath + } + credential, err := config.LoadCredential(selectedCredentialPath, policy) + if err != nil { + return err + } + runner, err := obsagent.Open(cfg, credential, nil) + if err != nil { + return err + } + if *once { + return runner.RunOnce(context.Background(), time.Now().UTC()) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + ticker := time.NewTicker(cfg.FlushEvery) + defer ticker.Stop() + for { + if err = runner.RunOnce(ctx, time.Now().UTC()); err != nil && !errors.Is(err, context.Canceled) { + log.Printf("observatory agent cycle: %v", err) + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +func agentEnrollCommand(args []string) error { + fs := flag.NewFlagSet("agent enroll", flag.ContinueOnError) + path := fs.String("config", "/etc/gamertan-observatory/agent.json", "absolute agent configuration path") + enrollmentFile := fs.String("enrollment-file", "", "absolute private enrollment-token file") + unsafeOwner := fs.Bool("development-allow-nonroot-config", false, "allow non-root config and enrollment ownership for local development") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("agent enroll accepts no positional arguments") + } + cfg, err := config.LoadAgent(*path, config.FilePolicy{RequireRoot: !*unsafeOwner}) + if err != nil { + return err + } + token, err := config.LoadEnrollmentToken(*enrollmentFile, config.FilePolicy{RequireRoot: !*unsafeOwner}) + if err != nil { + return err + } + result, err := agentclient.Enroll(context.Background(), cfg.ServerURL, token, nil) + if err != nil { + return err + } + if err = config.WriteCredential(cfg.CredentialFile, result.Credential); err != nil { + if revokeErr := agentclient.RevokeSource(context.Background(), cfg.ServerURL, result.Credential, nil); revokeErr != nil { + return fmt.Errorf("persist enrolled credential: %w; automatic source revocation also failed: %v", err, revokeErr) + } + return fmt.Errorf("persist enrolled credential: %w; enrolled source was revoked", err) + } + return json.NewEncoder(os.Stdout).Encode(struct { + SourceID string `json:"source_id"` + }{result.SourceID}) +} + +type serverConfigFlags struct { + path *string + allowNonRoot *bool + systemdCredential *bool +} + +func addServerConfigFlags(fs *flag.FlagSet, developmentHelp string) serverConfigFlags { + return serverConfigFlags{ + path: fs.String("config", "/etc/gamertan-observatory/server.json", "absolute server configuration path"), + allowNonRoot: fs.Bool("development-allow-nonroot-config", false, developmentHelp), + systemdCredential: fs.Bool("systemd-credential-config", false, "load the server configuration from CREDENTIALS_DIRECTORY"), + } +} + +func (flags serverConfigFlags) load() (config.Server, error) { + if *flags.allowNonRoot && *flags.systemdCredential { + return config.Server{}, errors.New("development non-root configuration and systemd credentials are mutually exclusive") + } + policy := config.FilePolicy{RequireRoot: !*flags.allowNonRoot} + if *flags.systemdCredential { + var err error + policy, err = config.SystemdCredentialPolicy() + if err != nil { + return config.Server{}, err + } + } + return config.LoadServer(*flags.path, policy) +} + +func (flags serverConfigFlags) requireRootAuxiliaryFile() bool { + return !*flags.allowNonRoot +} + +func commandConfig(name string, args []string) (config.Server, error) { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow a non-root config owner for local development") + if err := fs.Parse(args); err != nil { + return config.Server{}, err + } + if fs.NArg() != 0 { + return config.Server{}, fmt.Errorf("%s accepts no positional arguments", name) + } + return configFlags.load() +} + +func optionalPushDispatcher(notifier *webpush.Notifier) httpserver.PushDispatcher { + if notifier == nil { + return nil + } + return notifier +} + +func serve(cfg config.Server) error { + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + if err := store.RecoverRaw(context.Background()); err != nil { + return err + } + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + var pushNotifier *webpush.Notifier + var pushPublicKey string + if cfg.WebPush != nil { + pushSender, senderErr := webpush.New(webpush.Options{PrivateKey: cfg.WebPush.PrivateKey, Subject: cfg.WebPush.Subject, Timeout: cfg.WebPush.Timeout}) + if senderErr != nil { + return senderErr + } + pushNotifier, err = webpush.NewNotifier(store, identities.Access, pushSender, cfg.WebPush.QueueCapacity) + if err != nil { + return err + } + pushPublicKey = pushSender.PublicKey() + } + application, err := httpserver.New(store, identities, httpserver.Options{ + PublicOrigin: cfg.PublicURL, MaxBodyBytes: cfg.MaxBodyBytes, MaxConcurrentIngest: cfg.MaxConcurrentIngest, + MaxQueryRows: cfg.Query.MaxRows, SessionLifetime: cfg.SessionLifetime, + QueryBudget: query.Budget{MaxDuration: cfg.Query.MaxDuration, MaxRows: cfg.Query.MaxRows, MaxScannedBytes: cfg.Query.MaxScannedBytes, MaxMemoryBytes: cfg.Query.MaxMemoryBytes}, + PushPublicKey: pushPublicKey, PushDispatcher: optionalPushDispatcher(pushNotifier), + }) + if err != nil { + return err + } + handler := application.Handler() + server := &http.Server{Addr: cfg.Listen, Handler: handler, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 30 * time.Second, IdleTimeout: 90 * time.Second, MaxHeaderBytes: 32 << 10} + listener, err := net.Listen("tcp", cfg.Listen) + if err != nil { + return fmt.Errorf("listen: %w", err) + } + defer listener.Close() + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + backgroundJobs := make([]func(context.Context), 0, 4) + backgroundJobs = append(backgroundJobs, func(background context.Context) { + store.RunProjector(background, time.Second, func(error) { + log.Print("observatory projection temporarily unavailable") + }) + }) + if pushNotifier != nil { + backgroundJobs = append(backgroundJobs, pushNotifier.Run) + } + backgroundJobs = append(backgroundJobs, func(background context.Context) { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + apply := func(now time.Time) { + if _, retentionErr := store.ApplyRetention(background, retentionPolicy(cfg), now.UTC()); retentionErr != nil && !errors.Is(retentionErr, context.Canceled) { + log.Print("observatory retention unavailable") + } + if _, retentionErr := identities.PruneEvidence(background, cfg.Retention.EvidenceDays, now.UTC()); retentionErr != nil && !errors.Is(retentionErr, context.Canceled) { + log.Print("observatory identity retention unavailable") + } + } + apply(time.Now()) + for { + select { + case <-background.Done(): + return + case now := <-ticker.C: + apply(now) + } + } + }) + backgroundJobs = append(backgroundJobs, func(background context.Context) { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + if _, evaluationErr := application.EvaluateAlerts(background); evaluationErr != nil && !errors.Is(evaluationErr, context.Canceled) { + log.Print("observatory alert evaluation unavailable") + } + select { + case <-background.Done(): + return + case <-ticker.C: + } + } + }) + return serveHTTPWithBackground(ctx, server, listener, backgroundJobs...) +} + +func serveHTTPWithBackground(ctx context.Context, server *http.Server, listener net.Listener, jobs ...func(context.Context)) error { + background, stopBackground := context.WithCancel(ctx) + var backgroundJobs sync.WaitGroup + defer func() { + stopBackground() + backgroundJobs.Wait() + }() + result := make(chan error, 1) + go func() { result <- server.Serve(listener) }() + for _, job := range jobs { + backgroundJobs.Add(1) + go func() { + defer backgroundJobs.Done() + job(background) + }() + } + select { + case err := <-result: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + case <-ctx.Done(): + shutdown, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := server.Shutdown(shutdown); err != nil { + return fmt.Errorf("shutdown: %w", err) + } + log.Print("observatory server stopped") + return nil + } +} + +func adminCommand(args []string) error { + if len(args) == 0 { + return errors.New("admin command required: bootstrap, user create or reset-password, project create, environment create, service create, invitation create or accept, enrollment create, web-push generate-key, descriptors list, activate, reject, or retention set") + } + switch args[0] { + case "bootstrap": + return adminBootstrapCommand(args[1:]) + case "user": + if len(args) < 2 { + return errors.New("admin user command required: create or reset-password") + } + switch args[1] { + case "create": + return adminUserCreateCommand(args[2:]) + case "reset-password": + return adminUserResetPasswordCommand(args[2:]) + default: + return errors.New("admin user command required: create or reset-password") + } + case "project": + if len(args) < 2 || args[1] != "create" { + return errors.New("admin project command required: create") + } + return adminProjectCreateCommand(args[2:]) + case "environment": + if len(args) < 2 || args[1] != "create" { + return errors.New("admin environment command required: create") + } + return adminEnvironmentCreateCommand(args[2:]) + case "service": + if len(args) < 2 || args[1] != "create" { + return errors.New("admin service command required: create") + } + return adminServiceCreateCommand(args[2:]) + case "invitation": + if len(args) < 2 { + return errors.New("admin invitation command required: create or accept") + } + switch args[1] { + case "create": + return adminInvitationCreateCommand(args[2:]) + case "accept": + return adminInvitationAcceptCommand(args[2:]) + default: + return errors.New("admin invitation command required: create or accept") + } + case "enrollment": + if len(args) < 2 || args[1] != "create" { + return errors.New("admin enrollment command required: create") + } + return adminEnrollmentCreateCommand(args[2:]) + case "web-push": + if len(args) < 2 || args[1] != "generate-key" { + return errors.New("admin web-push command required: generate-key") + } + return adminWebPushGenerateKeyCommand(args[2:]) + case "descriptors": + if len(args) < 2 { + return errors.New("admin descriptors command required: list, activate, or reject") + } + switch args[1] { + case "list": + return adminDescriptorListCommand(args[2:]) + case "activate": + return adminDescriptorActivateCommand(args[2:]) + case "reject": + return adminDescriptorRejectCommand(args[2:]) + default: + return errors.New("admin descriptors command required: list, activate, or reject") + } + case "retention": + if len(args) < 2 || args[1] != "set" { + return errors.New("admin retention command required: set") + } + return adminRetentionSetCommand(args[2:]) + default: + return errors.New("admin command required: bootstrap, user create or reset-password, project create, environment create, service create, invitation create or accept, enrollment create, web-push generate-key, descriptors list, activate, reject, or retention set") + } +} + +func retentionPolicy(cfg config.Server) storage.RetentionPolicy { + return storage.RetentionPolicy{ + RawLogsDays: cfg.Retention.RawLogsDays, RawTracesDays: cfg.Retention.RawTracesDays, + RawMetricsDays: cfg.Retention.RawMetricsDays, ColdRawDays: cfg.Retention.ColdRawDays, MetricRollupsDays: cfg.Retention.MetricRollupsDays, + DeleteColdRaw: cfg.Retention.DeleteColdRaw, EvidenceDays: cfg.Retention.EvidenceDays, + } +} + +func adminRetentionSetCommand(args []string) error { + fs := flag.NewFlagSet("admin retention set", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration ownership for local development") + actor := fs.String("actor-user-id", "", "organization owner changing retention") + organizationID := fs.String("organization-id", "", "organization identifier") + rawLogs := fs.Int("raw-logs-days", 0, "raw log retention in days") + rawTraces := fs.Int("raw-traces-days", 0, "raw trace retention in days") + rawMetrics := fs.Int("raw-metrics-days", 0, "raw metric retention in days") + coldRaw := fs.Int("cold-raw-days", 0, "cold forensic raw-segment retention in days") + deleteColdRaw := fs.Bool("delete-cold-raw", false, "explicitly delete cold raw segments after their retention window") + metricRollups := fs.Int("metric-rollups-days", 0, "five-minute metric rollup retention in days") + evidence := fs.Int("evidence-days", 0, "deployment, incident, and audit evidence retention in days") + approveExtension := fs.String("approve-extension", "", "exact organization identifier approving retention beyond server defaults") + quotaBytes := fs.Int64("quota-bytes", 0, "positive storage quota required for an approved extension") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin retention set accepts no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + requested := access.Scope{OrganizationID: *organizationID} + if err = identities.ValidateResourceScope(context.Background(), requested); err != nil { + return err + } + decision, err := identities.Access.Authorize(context.Background(), *actor, requested, identity.PermissionOrganizationManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to change retention for the requested organization") + } + policy, err := store.SetOrganizationRetention(context.Background(), storage.SetRetentionInput{ + OrganizationID: *organizationID, ActorUserID: *actor, Defaults: retentionPolicy(cfg), + Policy: storage.RetentionPolicy{RawLogsDays: *rawLogs, RawTracesDays: *rawTraces, RawMetricsDays: *rawMetrics, ColdRawDays: *coldRaw, DeleteColdRaw: *deleteColdRaw, MetricRollupsDays: *metricRollups, EvidenceDays: *evidence}, + ApproveExtensionFor: *approveExtension, QuotaBytes: *quotaBytes, + }, time.Now().UTC()) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(policy) +} + +func adminWebPushGenerateKeyCommand(args []string) error { + fs := flag.NewFlagSet("admin web-push generate-key", flag.ContinueOnError) + output := fs.String("output-file", "", "new mode-0600 Web Push private-key file") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 || *output == "" { + return errors.New("admin web-push generate-key requires --output-file and no positional arguments") + } + key, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + return errors.New("generate Web Push private key") + } + if err = config.WriteWebPushPrivateKey(*output, key.Bytes()); err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(struct { + PrivateKeyFile string `json:"private_key_file"` + }{*output}) +} + +func adminDescriptorActivateCommand(args []string) error { + fs := flag.NewFlagSet("admin descriptors activate", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration and descriptor ownership for local development") + actor := fs.String("actor-user-id", "", "organization member activating the reviewed descriptor") + organizationID := fs.String("organization-id", "", "organization identifier") + descriptorFile := fs.String("descriptor-file", "", "absolute private reviewed-descriptor JSON file") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin descriptors activate accepts no positional arguments") + } + reviewed, err := readReviewedDescriptor(*descriptorFile, configFlags.requireRootAuxiliaryFile()) + if err != nil { + return err + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + requested := access.Scope{OrganizationID: *organizationID} + decision, err := identities.Access.Authorize(context.Background(), *actor, requested, identity.PermissionSchemaManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to activate descriptors for the requested organization") + } + if err = identities.ValidateResourceScope(context.Background(), requested); err != nil { + return err + } + activation, err := store.ActivateDescriptor(context.Background(), *organizationID, reviewed, time.Now().UTC()) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(activation) +} + +func adminDescriptorRejectCommand(args []string) error { + fs := flag.NewFlagSet("admin descriptors reject", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration ownership for local development") + actor := fs.String("actor-user-id", "", "organization member rejecting the descriptor proposal") + organizationID := fs.String("organization-id", "", "organization identifier") + signalName := fs.String("signal", "", "proposal signal: logs, metrics, traces, or deployments") + field := fs.String("field", "", "proposal field") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin descriptors reject accepts no positional arguments") + } + signal := model.Signal(*signalName) + switch signal { + case model.SignalLogs, model.SignalMetrics, model.SignalTraces, model.SignalDeployments: + default: + return errors.New("descriptor signal is invalid") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + requested := access.Scope{OrganizationID: *organizationID} + decision, err := identities.Access.Authorize(context.Background(), *actor, requested, identity.PermissionSchemaManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to reject descriptors for the requested organization") + } + if err = identities.ValidateResourceScope(context.Background(), requested); err != nil { + return err + } + if err = store.RejectDescriptorProposal(context.Background(), *organizationID, signal, *field); err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(struct { + OrganizationID string `json:"organization_id"` + Signal model.Signal `json:"signal"` + Field string `json:"field"` + Status string `json:"status"` + }{*organizationID, signal, query.CanonicalField(*field), "rejected"}) +} + +func readReviewedDescriptor(path string, requireRoot bool) (schema.Descriptor, error) { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return schema.Descriptor{}, errors.New("reviewed descriptor path must be absolute and clean") + } + info, err := os.Lstat(path) + if err != nil { + return schema.Descriptor{}, fmt.Errorf("inspect reviewed descriptor: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm() != 0o600 || info.Size() < 2 || info.Size() > 64<<10 { + return schema.Descriptor{}, errors.New("reviewed descriptor must be a private regular non-symlink file no larger than 64 KiB") + } + if requireRoot { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Uid != 0 { + return schema.Descriptor{}, errors.New("reviewed descriptor must be owned by root") + } + } + file, err := os.Open(path) + if err != nil { + return schema.Descriptor{}, fmt.Errorf("open reviewed descriptor: %w", err) + } + defer file.Close() + decoder := json.NewDecoder(io.LimitReader(file, 64<<10)) + decoder.DisallowUnknownFields() + var descriptor schema.Descriptor + if err = decoder.Decode(&descriptor); err != nil { + return schema.Descriptor{}, errors.New("reviewed descriptor JSON is invalid") + } + var extra any + if err = decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return schema.Descriptor{}, errors.New("reviewed descriptor JSON must contain one value") + } + if err = descriptor.Validate(); err != nil { + return schema.Descriptor{}, fmt.Errorf("reviewed descriptor: %w", err) + } + return descriptor, nil +} + +func adminDescriptorListCommand(args []string) error { + fs := flag.NewFlagSet("admin descriptors list", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration ownership for local development") + actor := fs.String("actor-user-id", "", "organization member reviewing descriptor proposals") + organizationID := fs.String("organization-id", "", "organization identifier") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin descriptors list accepts no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + requested := access.Scope{OrganizationID: *organizationID} + decision, err := identities.Access.Authorize(context.Background(), *actor, requested, identity.PermissionSchemaManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to review descriptors for the requested organization") + } + if err = identities.ValidateResourceScope(context.Background(), requested); err != nil { + return err + } + proposals, err := store.DescriptorProposals(context.Background(), *organizationID) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(proposals) +} + +func adminBootstrapCommand(args []string) error { + fs := flag.NewFlagSet("admin bootstrap", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root config and secret ownership for local development") + username := fs.String("username", "", "first operator username") + email := fs.String("email", "", "first operator email address") + displayName := fs.String("display-name", "", "first operator display name") + passwordFile := fs.String("password-file", "", "absolute path to a private one-line password file") + generatedPasswordFile := fs.String("generate-password-file", "", "create an exclusive private one-time password file at this absolute path") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin bootstrap accepts no positional arguments") + } + if (*passwordFile == "") == (*generatedPasswordFile == "") { + return errors.New("provide exactly one of --password-file or --generate-password-file") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + password := "" + requirePasswordChange := *generatedPasswordFile != "" + if requirePasswordChange { + password, err = auth.GenerateTemporaryPassword(nil) + if err == nil { + err = identity.WriteSecret(*generatedPasswordFile, password) + } + } else { + password, err = identity.ReadSecret(*passwordFile, configFlags.requireRootAuxiliaryFile()) + } + if err != nil { + return err + } + result, err := identities.Bootstrap(context.Background(), identity.BootstrapInput{Username: *username, Email: *email, DisplayName: *displayName, Password: password, RequirePasswordChange: requirePasswordChange}) + if err != nil { + if requirePasswordChange { + return errors.Join(err, identity.RemoveSecret(*generatedPasswordFile, configFlags.requireRootAuxiliaryFile())) + } + return err + } + return json.NewEncoder(os.Stdout).Encode(struct { + UserID string `json:"user_id"` + OrganizationID string `json:"personal_organization_id"` + PasswordChangeRequired bool `json:"password_change_required"` + }{result.User.ID, result.Organization.ID, result.User.PasswordChangeRequired}) +} + +func adminUserCreateCommand(args []string) error { + fs := flag.NewFlagSet("admin user create", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root config and secret ownership for local development") + username := fs.String("username", "", "new user's username") + email := fs.String("email", "", "new user's email address") + displayName := fs.String("display-name", "", "new user's display name") + passwordFile := fs.String("password-file", "", "absolute path to a private one-line password file") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin user create accepts no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + password, err := identity.ReadSecret(*passwordFile, configFlags.requireRootAuxiliaryFile()) + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + provisioned, err := identities.ProvisionUser(context.Background(), auth.CreateUser{Username: *username, Email: *email, DisplayName: *displayName, Password: password}) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(struct { + UserID string `json:"user_id"` + Email string `json:"email"` + PersonalOrganizationID string `json:"personal_organization_id"` + }{provisioned.User.ID, provisioned.User.Email, provisioned.Organization.ID}) +} + +func adminUserResetPasswordCommand(args []string) error { + fs := flag.NewFlagSet("admin user reset-password", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root config and secret ownership for local development") + identifier := fs.String("identifier", "", "existing username or email address") + generatedPasswordFile := fs.String("generate-password-file", "", "create an exclusive private one-time password file at this absolute path") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin user reset-password accepts no positional arguments") + } + if strings.TrimSpace(*identifier) == "" || *generatedPasswordFile == "" { + return errors.New("admin user reset-password requires --identifier and --generate-password-file") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + password, err := auth.GenerateTemporaryPassword(nil) + if err == nil { + err = identity.WriteSecret(*generatedPasswordFile, password) + } + if err != nil { + return err + } + removeCredential := func(requireRoot bool) error { + return identity.RemoveSecret(*generatedPasswordFile, requireRoot) + } + persisted, err := identity.ReadSecret(*generatedPasswordFile, configFlags.requireRootAuxiliaryFile()) + if err != nil || persisted != password { + return errors.Join(errors.New("admin user reset-password could not verify private credential delivery"), removeCredential(false)) + } + user, err := identities.Auth.ResetPassword(context.Background(), auth.AdministrativePasswordReset{Identifier: *identifier, TemporaryPassword: password}) + if err != nil { + return errors.Join(err, removeCredential(configFlags.requireRootAuxiliaryFile())) + } + return json.NewEncoder(os.Stdout).Encode(struct { + UserID string `json:"user_id"` + Username string `json:"username"` + PasswordChangeRequired bool `json:"password_change_required"` + SessionsRevoked bool `json:"sessions_revoked"` + }{user.ID, user.Username, user.PasswordChangeRequired, true}) +} + +func adminProjectCreateCommand(args []string) error { + fs := flag.NewFlagSet("admin project create", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration ownership for local development") + actor := fs.String("actor-user-id", "", "organization owner creating the project") + organizationID := fs.String("organization-id", "", "organization identifier") + slug := fs.String("slug", "", "project slug") + name := fs.String("name", "", "project name") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin project create accepts no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + scope := access.Scope{OrganizationID: *organizationID} + decision, err := identities.Access.Authorize(context.Background(), *actor, scope, identity.PermissionOrganizationManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to create projects for the requested organization") + } + if err = identities.ValidateResourceScope(context.Background(), scope); err != nil { + return err + } + project, err := identities.Organizations.CreateProject(context.Background(), organizations.CreateProject{OrganizationID: *organizationID, Slug: *slug, Name: *name}) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(struct { + OrganizationID string `json:"organization_id"` + ProjectID string `json:"project_id"` + Slug string `json:"slug"` + }{project.OrganizationID, project.ID, project.Slug}) +} + +func adminEnvironmentCreateCommand(args []string) error { + fs := flag.NewFlagSet("admin environment create", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration ownership for local development") + actor := fs.String("actor-user-id", "", "organization owner creating the environment") + organizationID := fs.String("organization-id", "", "organization identifier") + projectID := fs.String("project-id", "", "parent project identifier") + slug := fs.String("slug", "", "environment slug") + name := fs.String("name", "", "environment name") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin environment create accepts no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + organizationScope := access.Scope{OrganizationID: *organizationID} + decision, err := identities.Access.Authorize(context.Background(), *actor, organizationScope, identity.PermissionOrganizationManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to create environments for the requested organization") + } + if err = identities.ValidateResourceScope(context.Background(), access.Scope{OrganizationID: *organizationID, ProjectID: *projectID}); err != nil { + return err + } + environment, err := identities.Organizations.CreateEnvironment(context.Background(), organizations.CreateEnvironment{OrganizationID: *organizationID, ProjectID: *projectID, Slug: *slug, Name: *name}) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(struct { + OrganizationID string `json:"organization_id"` + ProjectID string `json:"project_id"` + EnvironmentID string `json:"environment_id"` + Slug string `json:"slug"` + }{environment.OrganizationID, environment.ProjectID, environment.ID, environment.Slug}) +} + +func adminServiceCreateCommand(args []string) error { + fs := flag.NewFlagSet("admin service create", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration ownership for local development") + actor := fs.String("actor-user-id", "", "organization owner creating the application service") + organizationID := fs.String("organization-id", "", "organization identifier") + projectID := fs.String("project-id", "", "parent project identifier") + environmentID := fs.String("environment-id", "", "parent environment identifier") + slug := fs.String("slug", "", "application service slug") + name := fs.String("name", "", "application service name") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin service create accepts no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + organizationScope := access.Scope{OrganizationID: *organizationID} + parentScope := access.Scope{OrganizationID: *organizationID, ProjectID: *projectID, EnvironmentID: *environmentID} + decision, err := identities.Access.Authorize(context.Background(), *actor, organizationScope, identity.PermissionOrganizationManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to create application services for the requested organization") + } + if err = identities.ValidateResourceScope(context.Background(), parentScope); err != nil { + return err + } + application, err := identities.Organizations.CreateApplicationService(context.Background(), organizations.CreateApplicationService{OrganizationID: *organizationID, ProjectID: *projectID, EnvironmentID: *environmentID, Slug: *slug, Name: *name}) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(struct { + OrganizationID string `json:"organization_id"` + ProjectID string `json:"project_id"` + EnvironmentID string `json:"environment_id"` + ServiceID string `json:"service_id"` + Slug string `json:"slug"` + }{application.OrganizationID, application.ProjectID, application.EnvironmentID, application.ID, application.Slug}) +} + +func adminInvitationCreateCommand(args []string) error { + fs := flag.NewFlagSet("admin invitation create", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration and token ownership for local development") + actor := fs.String("actor-user-id", "", "organization owner creating the invitation") + organizationID := fs.String("organization-id", "", "organization identifier") + email := fs.String("email", "", "invited user's exact email address") + lifetime := fs.Duration("lifetime", 15*time.Minute, "single-use invitation lifetime") + output := fs.String("output-file", "", "new mode-0600 invitation-token file") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 || *output == "" { + return errors.New("admin invitation create requires --output-file and no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + requested := access.Scope{OrganizationID: *organizationID} + if err = identities.ValidateResourceScope(context.Background(), requested); err != nil { + return err + } + decision, err := identities.Access.Authorize(context.Background(), *actor, requested, identity.PermissionOrganizationManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to create invitations for the requested organization") + } + raw, invitation, err := identities.Organizations.Invite(context.Background(), *organizationID, *email, *actor, *lifetime) + if err != nil { + return err + } + if err = identity.WriteSecret(*output, raw); err != nil { + if cancelErr := identities.CancelUnusedInvitation(context.Background(), invitation.Digest); cancelErr != nil { + return fmt.Errorf("persist invitation token: %w; automatic invitation cancellation also failed: %v", err, cancelErr) + } + return fmt.Errorf("persist invitation token: %w; invitation was cancelled", err) + } + return json.NewEncoder(os.Stdout).Encode(struct { + OrganizationID string `json:"organization_id"` + Email string `json:"email"` + ExpiresAt time.Time `json:"expires_at"` + InvitationFile string `json:"invitation_file"` + }{invitation.OrganizationID, invitation.Email, invitation.ExpiresAt, *output}) +} + +func adminInvitationAcceptCommand(args []string) error { + fs := flag.NewFlagSet("admin invitation accept", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration and token ownership for local development") + userID := fs.String("user-id", "", "invited user's identifier") + invitationFile := fs.String("invitation-file", "", "absolute private invitation-token file") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin invitation accept accepts no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + raw, err := identity.ReadSecret(*invitationFile, configFlags.requireRootAuxiliaryFile()) + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + if err = identities.Organizations.AcceptInvitation(context.Background(), raw, *userID); err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(struct { + UserID string `json:"user_id"` + Status string `json:"status"` + }{*userID, "accepted"}) +} + +func adminEnrollmentCreateCommand(args []string) error { + fs := flag.NewFlagSet("admin enrollment create", flag.ContinueOnError) + configFlags := addServerConfigFlags(fs, "allow non-root configuration ownership for local development") + actor := fs.String("actor-user-id", "", "organization member creating the enrollment") + sourceID := fs.String("source-id", "", "new source identifier") + organizationID := fs.String("organization-id", "", "organization identifier") + projectID := fs.String("project-id", "", "project identifier") + environmentID := fs.String("environment-id", "", "environment identifier") + serviceID := fs.String("service-id", "", "service identifier") + lifetime := fs.Duration("lifetime", 15*time.Minute, "single-use enrollment lifetime") + output := fs.String("output-file", "", "new mode-0600 enrollment-token file") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("admin enrollment create accepts no positional arguments") + } + cfg, err := configFlags.load() + if err != nil { + return err + } + processLock, err := storage.AcquireProcessLock(cfg.DataDir, false) + if err != nil { + return err + } + defer processLock.Close() + store, err := storage.Open(cfg.DataDir) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(cfg.DataDir) + if err != nil { + return err + } + defer identities.Close() + requested := access.Scope{OrganizationID: *organizationID, ProjectID: *projectID, EnvironmentID: *environmentID, ServiceID: *serviceID} + decision, err := identities.Access.Authorize(context.Background(), *actor, requested, identity.PermissionSourcesManage) + if err != nil { + return err + } + if !decision.Allowed { + return errors.New("actor is not authorized to manage sources in the requested scope") + } + if err = identities.ValidateResourceScope(context.Background(), requested); err != nil { + return err + } + scope := model.Scope{OrganizationID: *organizationID, ProjectID: *projectID, EnvironmentID: *environmentID, ServiceID: *serviceID} + now := time.Now().UTC() + token, enrollment, err := store.CreateEnrollment(context.Background(), *sourceID, scope, *actor, *lifetime, now) + if err != nil { + return err + } + if err = config.WriteEnrollmentToken(*output, token); err != nil { + if cancelErr := store.CancelEnrollment(context.Background(), token); cancelErr != nil { + return fmt.Errorf("persist enrollment token: %w; automatic cancellation also failed: %v", err, cancelErr) + } + return fmt.Errorf("persist enrollment token: %w; enrollment was cancelled", err) + } + return json.NewEncoder(os.Stdout).Encode(struct { + SourceID string `json:"source_id"` + ExpiresAt time.Time `json:"expires_at"` + }{enrollment.SourceID, enrollment.ExpiresAt}) +} diff --git a/cmd/observatory/main_test.go b/cmd/observatory/main_test.go new file mode 100644 index 0000000..7ca651e --- /dev/null +++ b/cmd/observatory/main_test.go @@ -0,0 +1,824 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "gamertan.com/observatory/internal/config" + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/schema" + "gamertan.com/observatory/internal/storage" + "gamertan.com/web/access" + "gamertan.com/web/auth" +) + +func TestNoCommandRequiresAnExplicitTendCandidateAddress(t *testing.T) { + t.Setenv("OBSERVATORY_TEND_CANDIDATE_LISTEN", "") + if err := run(nil); err == nil || !strings.Contains(err.Error(), "command required") { + t.Fatalf("run without command err=%v", err) + } +} + +func TestTendCandidateHandlerIsStatelessAndBounded(t *testing.T) { + handler := tendCandidateHandler() + for _, test := range []struct { + method, path, body string + }{ + {http.MethodGet, "/", "Gamertan Observatory candidate\n"}, + {http.MethodHead, "/", ""}, + {http.MethodGet, "/healthz", "ok\n"}, + {http.MethodHead, "/healthz", ""}, + {http.MethodGet, "/readyz", "ready\n"}, + {http.MethodHead, "/readyz", ""}, + } { + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(test.method, "http://observatory.example"+test.path, nil)) + response := recorder.Result() + body, err := io.ReadAll(response.Body) + response.Body.Close() + if err != nil || response.StatusCode != http.StatusOK || string(body) != test.body { + t.Fatalf("%s %s status=%d body=%q err=%v", test.method, test.path, response.StatusCode, body, err) + } + if response.Header.Get("Cache-Control") != "no-store" || response.Header.Get("X-Content-Type-Options") != "nosniff" || response.Header.Get("Content-Security-Policy") == "" { + t.Fatalf("%s %s security headers=%v", test.method, test.path, response.Header) + } + } + for _, test := range []struct { + method, path string + }{ + {http.MethodPost, "/"}, + {http.MethodGet, "/api/v1/ingest/native"}, + {http.MethodGet, "/app/"}, + } { + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(test.method, "http://observatory.example"+test.path, nil)) + if recorder.Code == http.StatusOK { + t.Fatalf("candidate exposed %s %s", test.method, test.path) + } + } +} + +func TestTendCandidateAddressRejectsNonLoopbackAndMalformedValues(t *testing.T) { + for _, value := range []string{"", "localhost:18094", "0.0.0.0:18094", "192.0.2.1:18094", "127.0.0.1:0", "127.0.0.1", "127.0.0.1:70000"} { + if err := serveTendCandidate(value); err == nil || !strings.Contains(err.Error(), "loopback IP address") { + t.Fatalf("unsafe candidate address %q err=%v", value, err) + } + } +} + +func TestTendCandidateProcessIsReadOnly(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if errors.Is(err, syscall.EPERM) { + t.Skip("loopback sockets are prohibited by this test sandbox") + } + t.Fatal(err) + } + address := listener.Addr().String() + if err = listener.Close(); err != nil { + t.Fatal(err) + } + workingDirectory := t.TempDir() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestTendCandidateProcessHelper$") + command.Dir = workingDirectory + command.Env = append(os.Environ(), "OBSERVATORY_TEND_CANDIDATE_HELPER=1", "OBSERVATORY_TEND_CANDIDATE_LISTEN="+address) + var output bytes.Buffer + command.Stdout = &output + command.Stderr = &output + if err = command.Start(); err != nil { + t.Fatal(err) + } + client := &http.Client{Timeout: 250 * time.Millisecond} + ready := false + for ctx.Err() == nil { + response, requestErr := client.Get("http://" + address + "/readyz") + if requestErr == nil { + body, readErr := io.ReadAll(response.Body) + response.Body.Close() + if readErr == nil && response.StatusCode == http.StatusOK && string(body) == "ready\n" { + ready = true + break + } + } + time.Sleep(20 * time.Millisecond) + } + if !ready { + _ = command.Process.Kill() + _ = command.Wait() + t.Fatalf("candidate did not become ready: %s", output.String()) + } + if err = command.Process.Signal(os.Interrupt); err != nil { + _ = command.Process.Kill() + t.Fatal(err) + } + if err = command.Wait(); err != nil { + t.Fatalf("candidate exit err=%v output=%s", err, output.String()) + } + entries, err := os.ReadDir(workingDirectory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("candidate wrote files: %v", entries) + } +} + +func TestTendCandidateProcessHelper(t *testing.T) { + if os.Getenv("OBSERVATORY_TEND_CANDIDATE_HELPER") != "1" { + return + } + if err := run(nil); err != nil { + t.Fatal(err) + } +} + +func TestHTTPServerAnswersReadinessWhileBackgroundMaintenanceIsBlocked(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if errors.Is(err, syscall.EPERM) { + t.Skip("loopback sockets are prohibited by this test sandbox") + } + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + maintenanceStarted := make(chan struct{}) + maintenanceStopped := make(chan struct{}) + server := &http.Server{Handler: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/readyz" { + http.NotFound(writer, request) + return + } + _, _ = io.WriteString(writer, "ready\n") + })} + result := make(chan error, 1) + go func() { + result <- serveHTTPWithBackground(ctx, server, listener, func(background context.Context) { + close(maintenanceStarted) + <-background.Done() + close(maintenanceStopped) + }) + }() + select { + case <-maintenanceStarted: + case <-time.After(time.Second): + cancel() + t.Fatal("background maintenance did not start") + } + client := &http.Client{Timeout: 250 * time.Millisecond} + response, err := client.Get("http://" + listener.Addr().String() + "/readyz") + if err != nil { + cancel() + t.Fatal(err) + } + body, readErr := io.ReadAll(response.Body) + response.Body.Close() + if readErr != nil || response.StatusCode != http.StatusOK || string(body) != "ready\n" { + cancel() + t.Fatalf("readiness status=%d body=%q err=%v", response.StatusCode, body, readErr) + } + cancel() + select { + case err = <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("server did not stop after cancellation") + } + select { + case <-maintenanceStopped: + default: + t.Fatal("server returned before background maintenance stopped") + } +} + +func TestCommandConfigSupportsConfinedSystemdCredentials(t *testing.T) { + root := t.TempDir() + credentialDirectory := filepath.Join(root, "credentials") + if err := os.Mkdir(credentialDirectory, 0o700); err != nil { + t.Fatal(err) + } + data := filepath.Join(root, "data") + path := writeTestServerConfig(t, credentialDirectory, data) + if err := os.Chmod(path, 0o400); err != nil { + t.Fatal(err) + } + t.Setenv("CREDENTIALS_DIRECTORY", credentialDirectory) + cfg, err := commandConfig("server", []string{"--config", path, "--systemd-credential-config"}) + if err != nil || cfg.DataDir != data { + t.Fatalf("config=%+v err=%v", cfg, err) + } + if err = os.Chmod(path, 0o440); err != nil { + t.Fatal(err) + } + if cfg, err = commandConfig("server", []string{"--config", path, "--systemd-credential-config"}); err != nil || cfg.DataDir != data { + t.Fatalf("systemd mode-0440 config=%+v err=%v", cfg, err) + } + if _, err = commandConfig("server", []string{"--config", path, "--systemd-credential-config", "--development-allow-nonroot-config"}); err == nil || !strings.Contains(err.Error(), "mutually exclusive") { + t.Fatalf("conflicting config policy err=%v", err) + } + out := filepath.Join(root, "outside.json") + body, readErr := os.ReadFile(path) + if readErr != nil { + t.Fatal(readErr) + } + if err = os.WriteFile(out, body, 0o400); err != nil { + t.Fatal(err) + } + if _, err = commandConfig("server", []string{"--config", out, "--systemd-credential-config"}); err == nil || !strings.Contains(err.Error(), "direct child") { + t.Fatalf("outside credential err=%v", err) + } +} + +func TestOptionalPushDispatcherDoesNotBoxANilNotifier(t *testing.T) { + if dispatcher := optionalPushDispatcher(nil); dispatcher != nil { + t.Fatalf("nil notifier became non-nil dispatcher: %#v", dispatcher) + } +} + +func TestAdminWebPushKeyGenerationIsPrivateAndExclusive(t *testing.T) { + path := filepath.Join(t.TempDir(), "web-push.json") + if err := adminWebPushGenerateKeyCommand([]string{"--output-file", path}); err != nil { + t.Fatal(err) + } + key, err := config.LoadWebPushPrivateKey(path, config.FilePolicy{}) + if err != nil || len(key) != 32 { + t.Fatalf("key length=%d err=%v", len(key), err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("mode=%v err=%v", info, err) + } + if err = adminWebPushGenerateKeyCommand([]string{"--output-file", path}); err == nil { + t.Fatal("key generation overwrote an existing secret") + } +} + +func TestAdminBootstrapUsesSecretFileAndIsSingleUse(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + configPath := filepath.Join(root, "server.json") + passwordPath := filepath.Join(root, "password") + configuration := fmt.Sprintf(`{"schema":1,"listen":"127.0.0.1:9010","public_url":"https://observatory.example","data_dir":%q,"max_body_bytes":1048576,"session_lifetime":"12h","query":{"max_duration":"2s","max_rows":1000,"max_scanned_bytes":10485760,"max_memory_bytes":8388608},"retention":{"raw_logs_days":30,"raw_traces_days":30,"raw_metrics_days":14,"cold_raw_days":400,"metric_rollups_days":400,"evidence_days":400}}`, data) + if err := os.WriteFile(configPath, []byte(configuration), 0o600); err != nil { + t.Fatal(err) + } + const password = "correct horse battery staple" + if err := os.WriteFile(passwordPath, []byte(password+"\n"), 0o600); err != nil { + t.Fatal(err) + } + args := []string{"admin", "bootstrap", "--config", configPath, "--username", "operator", "--email", "operator@example.test", "--display-name", "First Operator", "--password-file", passwordPath, "--development-allow-nonroot-config"} + if err := run(args); err != nil { + t.Fatal(err) + } + if err := run(args); !errors.Is(err, identity.ErrAlreadyBootstrapped) { + t.Fatalf("second bootstrap err=%v", err) + } + entries, err := os.ReadDir(data) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() || entry.Name() == ".bootstrap.lock" { + continue + } + body, readErr := os.ReadFile(filepath.Join(data, entry.Name())) + if readErr == nil && strings.Contains(string(body), password) { + t.Fatalf("password leaked into %s", entry.Name()) + } + } +} + +func TestAdminBootstrapGeneratesExclusiveOneTimeCredential(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + configPath := writeTestServerConfig(t, root, data) + passwordPath := filepath.Join(root, "generated-bootstrap-password") + args := []string{"--config", configPath, "--username", "speelman", "--email", "crspeelman@gmail.com", "--display-name", "Cole Speelman", "--generate-password-file", passwordPath, "--development-allow-nonroot-config"} + var result struct { + UserID string `json:"user_id"` + OrganizationID string `json:"personal_organization_id"` + PasswordChangeRequired bool `json:"password_change_required"` + } + captureCommandJSON(t, adminBootstrapCommand, args, &result) + if result.UserID == "" || result.OrganizationID == "" || !result.PasswordChangeRequired { + t.Fatalf("result=%+v", result) + } + info, err := os.Lstat(passwordPath) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + t.Fatalf("password file info=%v err=%v", info, err) + } + password, err := identity.ReadSecret(passwordPath, false) + if err != nil || len(password) != 43 || strings.ContainsAny(password, "\r\n") { + t.Fatalf("generated credential length=%d err=%v", len(password), err) + } + identities, err := identity.Open(data) + if err != nil { + t.Fatal(err) + } + _, principal, err := identities.Auth.Authenticate(t.Context(), "speelman", password, time.Hour) + if err != nil || principal.User.Email != "crspeelman@gmail.com" || principal.User.DisplayName != "Cole Speelman" || !principal.User.PasswordChangeRequired { + identities.Close() + t.Fatalf("principal=%+v err=%v", principal, err) + } + if err = identities.Close(); err != nil { + t.Fatal(err) + } + secondPath := filepath.Join(root, "second-bootstrap-password") + second := append([]string{}, args...) + second[9] = secondPath + if err = adminBootstrapCommand(second); !errors.Is(err, identity.ErrAlreadyBootstrapped) { + t.Fatalf("second bootstrap err=%v", err) + } + if _, err = os.Lstat(secondPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed bootstrap retained generated credential: %v", err) + } + for _, invalid := range [][]string{ + {"--config", configPath, "--username", "x", "--email", "x@example.test", "--display-name", "X", "--development-allow-nonroot-config"}, + {"--config", configPath, "--username", "x", "--email", "x@example.test", "--display-name", "X", "--password-file", passwordPath, "--generate-password-file", secondPath, "--development-allow-nonroot-config"}, + } { + if err = adminBootstrapCommand(invalid); err == nil || !strings.Contains(err.Error(), "exactly one") { + t.Fatalf("invalid bootstrap flags err=%v", err) + } + } +} + +func TestAdminBootstrapSupportsConfinedSystemdCredentials(t *testing.T) { + root := t.TempDir() + credentials := filepath.Join(root, "credentials") + if err := os.Mkdir(credentials, 0o700); err != nil { + t.Fatal(err) + } + data := filepath.Join(root, "data") + configPath := writeTestServerConfig(t, credentials, data) + if err := os.Chmod(configPath, 0o440); err != nil { + t.Fatal(err) + } + passwordPath := filepath.Join(root, "generated-password") + t.Setenv("CREDENTIALS_DIRECTORY", credentials) + args := []string{"--config", configPath, "--username", "operator", "--email", "operator@example.test", "--display-name", "First Operator", "--generate-password-file", passwordPath, "--systemd-credential-config"} + var result struct { + PasswordChangeRequired bool `json:"password_change_required"` + } + captureCommandJSON(t, adminBootstrapCommand, args, &result) + if !result.PasswordChangeRequired { + t.Fatalf("result=%+v", result) + } + if _, err := identity.ReadSecret(passwordPath, false); err != nil { + t.Fatal(err) + } + conflicting := append(append([]string{}, args...), "--development-allow-nonroot-config") + if err := adminBootstrapCommand(conflicting); err == nil || !strings.Contains(err.Error(), "mutually exclusive") { + t.Fatalf("conflicting config policies err=%v", err) + } +} + +func TestAdminUserResetPasswordGeneratesPrivateCredentialAndRevokesSessions(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + configPath := writeTestServerConfig(t, root, data) + bootstrapPath := filepath.Join(root, "bootstrap-password") + common := []string{"--config", configPath, "--development-allow-nonroot-config"} + var bootstrap struct { + UserID string `json:"user_id"` + } + captureCommandJSON(t, adminBootstrapCommand, append(append([]string{}, common...), "--username", "speelman", "--email", "operator@example.test", "--display-name", "Operator", "--generate-password-file", bootstrapPath), &bootstrap) + bootstrapPassword, err := identity.ReadSecret(bootstrapPath, false) + if err != nil { + t.Fatal(err) + } + identities, err := identity.Open(data) + if err != nil { + t.Fatal(err) + } + token, _, err := identities.Auth.Authenticate(t.Context(), "speelman", bootstrapPassword, time.Hour) + if err != nil { + identities.Close() + t.Fatal(err) + } + if err = identities.Close(); err != nil { + t.Fatal(err) + } + recoveryPath := filepath.Join(root, "recovery-password") + var result struct { + UserID string `json:"user_id"` + Username string `json:"username"` + PasswordChangeRequired bool `json:"password_change_required"` + SessionsRevoked bool `json:"sessions_revoked"` + } + body := captureCommandJSON(t, adminUserResetPasswordCommand, append(append([]string{}, common...), "--identifier", "operator@example.test", "--generate-password-file", recoveryPath), &result) + if result.UserID != bootstrap.UserID || result.Username != "speelman" || !result.PasswordChangeRequired || !result.SessionsRevoked { + t.Fatalf("result=%+v", result) + } + recoveryPassword, err := identity.ReadSecret(recoveryPath, false) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(body), recoveryPassword) || strings.Contains(string(body), recoveryPath) { + t.Fatal("command output disclosed the recovery credential or its path") + } + identities, err = identity.Open(data) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + if _, err = identities.Auth.Session(t.Context(), token); !errors.Is(err, auth.ErrSessionNotFound) { + t.Fatalf("old session survived recovery: %v", err) + } + if _, _, err = identities.Auth.Authenticate(t.Context(), "speelman", bootstrapPassword, time.Hour); !errors.Is(err, auth.ErrInvalidCredentials) { + t.Fatalf("bootstrap credential survived recovery: %v", err) + } + _, principal, err := identities.Auth.Authenticate(t.Context(), "speelman", recoveryPassword, time.Hour) + if err != nil || !principal.User.PasswordChangeRequired { + t.Fatalf("recovery principal=%+v err=%v", principal, err) + } + if err = adminUserResetPasswordCommand(append(append([]string{}, common...), "--identifier", "speelman", "--generate-password-file", recoveryPath)); err == nil { + t.Fatal("recovery credential file was overwritten") + } + missingPath := filepath.Join(root, "missing-user-password") + if err = adminUserResetPasswordCommand(append(append([]string{}, common...), "--identifier", "missing-user", "--generate-password-file", missingPath)); !errors.Is(err, auth.ErrUserNotFound) { + t.Fatalf("missing user err=%v", err) + } + if _, err = os.Lstat(missingPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed recovery retained credential: %v", err) + } +} + +func TestAdminHierarchyAndEnrollmentSupportConfinedSystemdCredentials(t *testing.T) { + root := t.TempDir() + credentials := filepath.Join(root, "credentials") + if err := os.Mkdir(credentials, 0o700); err != nil { + t.Fatal(err) + } + data := filepath.Join(root, "data") + configPath := writeTestServerConfig(t, credentials, data) + if err := os.Chmod(configPath, 0o440); err != nil { + t.Fatal(err) + } + t.Setenv("CREDENTIALS_DIRECTORY", credentials) + common := []string{"--config", configPath, "--systemd-credential-config"} + passwordPath := filepath.Join(root, "generated-password") + var owner struct { + UserID string `json:"user_id"` + OrganizationID string `json:"personal_organization_id"` + } + bootstrap := append(append([]string{}, common...), "--username", "operator", "--email", "operator@example.test", "--display-name", "Operator", "--generate-password-file", passwordPath) + captureCommandJSON(t, adminBootstrapCommand, bootstrap, &owner) + + var project struct { + ProjectID string `json:"project_id"` + } + projectArgs := append(append([]string{}, common...), "--actor-user-id", owner.UserID, "--organization-id", owner.OrganizationID, "--slug", "observatory", "--name", "Gamertan Observatory") + captureCommandJSON(t, adminProjectCreateCommand, projectArgs, &project) + var environment struct { + EnvironmentID string `json:"environment_id"` + } + environmentArgs := append(append([]string{}, common...), "--actor-user-id", owner.UserID, "--organization-id", owner.OrganizationID, "--project-id", project.ProjectID, "--slug", "production", "--name", "Production") + captureCommandJSON(t, adminEnvironmentCreateCommand, environmentArgs, &environment) + var service struct { + ServiceID string `json:"service_id"` + } + serviceArgs := append(append([]string{}, common...), "--actor-user-id", owner.UserID, "--organization-id", owner.OrganizationID, "--project-id", project.ProjectID, "--environment-id", environment.EnvironmentID, "--slug", "observatory", "--name", "Observatory") + captureCommandJSON(t, adminServiceCreateCommand, serviceArgs, &service) + + enrollmentFile := filepath.Join(root, "enrollment-token") + var enrollment struct { + SourceID string `json:"source_id"` + } + enrollmentArgs := append(append([]string{}, common...), "--actor-user-id", owner.UserID, "--source-id", "observatory-production", "--organization-id", owner.OrganizationID, "--project-id", project.ProjectID, "--environment-id", environment.EnvironmentID, "--service-id", service.ServiceID, "--output-file", enrollmentFile) + captureCommandJSON(t, adminEnrollmentCreateCommand, enrollmentArgs, &enrollment) + if project.ProjectID == "" || environment.EnvironmentID == "" || service.ServiceID == "" || enrollment.SourceID != "observatory-production" { + t.Fatalf("project=%+v environment=%+v service=%+v enrollment=%+v", project, environment, service, enrollment) + } + if _, err := config.LoadEnrollmentToken(enrollmentFile, config.FilePolicy{}); err != nil { + t.Fatal(err) + } + var proposals []schema.Proposal + descriptors := append(append([]string{}, common...), "--actor-user-id", owner.UserID, "--organization-id", owner.OrganizationID) + captureCommandJSON(t, adminDescriptorListCommand, descriptors, &proposals) +} + +func TestAdminResourceCommandsCreateAnAuthorizedEnrollableHierarchy(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + configPath := writeTestServerConfig(t, root, data) + ownerPassword := filepath.Join(root, "owner-password") + memberPassword := filepath.Join(root, "member-password") + if err := os.WriteFile(ownerPassword, []byte("correct horse battery staple\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(memberPassword, []byte("another correct horse battery staple\n"), 0o600); err != nil { + t.Fatal(err) + } + common := []string{"--config", configPath, "--development-allow-nonroot-config"} + var owner struct { + UserID string `json:"user_id"` + OrganizationID string `json:"personal_organization_id"` + } + captureCommandJSON(t, adminBootstrapCommand, append(append([]string{}, common...), "--username", "operator", "--email", "operator@example.test", "--display-name", "First Operator", "--password-file", ownerPassword), &owner) + + var project struct { + OrganizationID string `json:"organization_id"` + ProjectID string `json:"project_id"` + Slug string `json:"slug"` + } + projectArgs := append([]string{"admin", "project", "create"}, common...) + projectArgs = append(projectArgs, "--actor-user-id", owner.UserID, "--organization-id", owner.OrganizationID, "--slug", "gamertan", "--name", "Gamertan") + captureCommandJSON(t, run, projectArgs, &project) + if project.OrganizationID != owner.OrganizationID || project.ProjectID == "" || project.Slug != "gamertan" { + t.Fatalf("project=%+v", project) + } + + var environment struct { + OrganizationID string `json:"organization_id"` + ProjectID string `json:"project_id"` + EnvironmentID string `json:"environment_id"` + Slug string `json:"slug"` + } + environmentArgs := append([]string{"admin", "environment", "create"}, common...) + environmentArgs = append(environmentArgs, "--actor-user-id", owner.UserID, "--organization-id", owner.OrganizationID, "--project-id", project.ProjectID, "--slug", "production", "--name", "Production") + captureCommandJSON(t, run, environmentArgs, &environment) + if environment.ProjectID != project.ProjectID || environment.EnvironmentID == "" || environment.Slug != "production" { + t.Fatalf("environment=%+v", environment) + } + + var application struct { + OrganizationID string `json:"organization_id"` + ProjectID string `json:"project_id"` + EnvironmentID string `json:"environment_id"` + ServiceID string `json:"service_id"` + Slug string `json:"slug"` + } + serviceArgs := append([]string{"admin", "service", "create"}, common...) + serviceArgs = append(serviceArgs, "--actor-user-id", owner.UserID, "--organization-id", owner.OrganizationID, "--project-id", project.ProjectID, "--environment-id", environment.EnvironmentID, "--slug", "observatory", "--name", "Gamertan Observatory") + captureCommandJSON(t, run, serviceArgs, &application) + if application.EnvironmentID != environment.EnvironmentID || application.ServiceID == "" || application.Slug != "observatory" { + t.Fatalf("application=%+v", application) + } + + identities, err := identity.Open(data) + if err != nil { + t.Fatal(err) + } + if err = identities.ValidateResourceScope(context.Background(), access.Scope{OrganizationID: owner.OrganizationID, ProjectID: project.ProjectID, EnvironmentID: environment.EnvironmentID, ServiceID: application.ServiceID}); err != nil { + identities.Close() + t.Fatal(err) + } + if err = identities.Close(); err != nil { + t.Fatal(err) + } + + var member struct { + UserID string `json:"user_id"` + } + captureCommandJSON(t, adminUserCreateCommand, append(append([]string{}, common...), "--username", "member", "--email", "member@example.test", "--display-name", "Member", "--password-file", memberPassword), &member) + unauthorized := append(append([]string{}, common...), "--actor-user-id", member.UserID, "--organization-id", owner.OrganizationID, "--slug", "forbidden", "--name", "Forbidden") + if err = adminProjectCreateCommand(unauthorized); err == nil || !strings.Contains(err.Error(), "not authorized") { + t.Fatalf("unauthorized project err=%v", err) + } + wrongParent := append(append([]string{}, common...), "--actor-user-id", owner.UserID, "--organization-id", owner.OrganizationID, "--project-id", member.UserID, "--slug", "invalid", "--name", "Invalid") + if err = adminEnvironmentCreateCommand(wrongParent); !errors.Is(err, identity.ErrResourceNotFound) { + t.Fatalf("wrong parent err=%v", err) + } +} + +func captureCommandJSON(t *testing.T, command func([]string) error, args []string, target any) []byte { + t.Helper() + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + original := os.Stdout + os.Stdout = writer + commandErr := command(args) + closeErr := writer.Close() + os.Stdout = original + body, readErr := io.ReadAll(reader) + reader.Close() + if commandErr != nil { + t.Fatal(commandErr) + } + if closeErr != nil || readErr != nil { + t.Fatalf("capture close=%v read=%v", closeErr, readErr) + } + if err = json.Unmarshal(body, target); err != nil { + t.Fatalf("decode command output: %v", err) + } + return body +} + +func TestAdminUserAndInvitationCommandsProvisionPrivateAndSharedAccess(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + configPath := writeTestServerConfig(t, root, data) + ownerPassword := filepath.Join(root, "owner-password") + memberPassword := filepath.Join(root, "member-password") + otherPassword := filepath.Join(root, "other-password") + invitationPath := filepath.Join(root, "invitation") + if err := os.WriteFile(ownerPassword, []byte("correct horse battery staple\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(memberPassword, []byte("another correct horse battery staple\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(otherPassword, []byte("a third correct horse battery staple\n"), 0o600); err != nil { + t.Fatal(err) + } + common := []string{"--config", configPath, "--development-allow-nonroot-config"} + if err := adminBootstrapCommand(append(append([]string{}, common...), "--username", "operator", "--email", "operator@example.test", "--display-name", "First Operator", "--password-file", ownerPassword)); err != nil { + t.Fatal(err) + } + if err := adminUserCreateCommand(append(append([]string{}, common...), "--username", "responder", "--email", "responder@example.test", "--display-name", "Incident Responder", "--password-file", memberPassword)); err != nil { + t.Fatal(err) + } + if err := adminUserCreateCommand(append(append([]string{}, common...), "--username", "other-user", "--email", "other@example.test", "--display-name", "Other User", "--password-file", otherPassword)); err != nil { + t.Fatal(err) + } + identities, err := identity.Open(data) + if err != nil { + t.Fatal(err) + } + _, owner, err := identities.Auth.Authenticate(t.Context(), "operator", "correct horse battery staple", time.Hour) + if err != nil { + identities.Close() + t.Fatal(err) + } + _, member, err := identities.Auth.Authenticate(t.Context(), "responder", "another correct horse battery staple", time.Hour) + if err != nil { + identities.Close() + t.Fatal(err) + } + _, other, err := identities.Auth.Authenticate(t.Context(), "other-user", "a third correct horse battery staple", time.Hour) + if err != nil { + identities.Close() + t.Fatal(err) + } + ownerOrganizations, err := identities.OrganizationsForUser(t.Context(), owner.User.ID) + if err != nil || len(ownerOrganizations) != 1 || !ownerOrganizations[0].Personal { + identities.Close() + t.Fatalf("owner organizations=%+v err=%v", ownerOrganizations, err) + } + memberOrganizations, err := identities.OrganizationsForUser(t.Context(), member.User.ID) + if err != nil || len(memberOrganizations) != 1 || !memberOrganizations[0].Personal { + identities.Close() + t.Fatalf("member organizations=%+v err=%v", memberOrganizations, err) + } + if err = identities.Close(); err != nil { + t.Fatal(err) + } + createArgs := append(append([]string{}, common...), "--actor-user-id", owner.User.ID, "--organization-id", ownerOrganizations[0].ID, "--email", member.User.Email, "--output-file", invitationPath) + if err = adminInvitationCreateCommand(createArgs); err != nil { + t.Fatal(err) + } + info, err := os.Lstat(invitationPath) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + t.Fatalf("invitation file=%v err=%v", info, err) + } + acceptArgs := append(append([]string{}, common...), "--user-id", member.User.ID, "--invitation-file", invitationPath) + wrongUserArgs := append(append([]string{}, common...), "--user-id", other.User.ID, "--invitation-file", invitationPath) + if err = adminInvitationAcceptCommand(wrongUserArgs); err == nil { + t.Fatal("invitation was accepted by a user with a different email") + } + if err = adminInvitationAcceptCommand(acceptArgs); err != nil { + t.Fatal(err) + } + if err = adminInvitationAcceptCommand(acceptArgs); err == nil { + t.Fatal("single-use invitation was accepted twice") + } + identities, err = identity.Open(data) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + memberOrganizations, err = identities.OrganizationsForUser(t.Context(), member.User.ID) + if err != nil || len(memberOrganizations) != 2 { + t.Fatalf("member organizations=%+v err=%v", memberOrganizations, err) + } + unauthorized := append(append([]string{}, common...), "--actor-user-id", member.User.ID, "--organization-id", ownerOrganizations[0].ID, "--email", "other@example.test", "--output-file", filepath.Join(root, "unauthorized-invitation")) + if err = adminInvitationCreateCommand(unauthorized); err == nil || !strings.Contains(err.Error(), "not authorized") { + t.Fatalf("unauthorized invitation err=%v", err) + } +} + +func TestMigrateProjectionRebuildRequiresExactApproval(t *testing.T) { + if err := migrateCommand([]string{"--rebuild-organization", "organization-a"}); err == nil || !strings.Contains(err.Error(), "exact organization identifier") { + t.Fatalf("missing approval err=%v", err) + } + if err := migrateCommand([]string{"--approve-rebuild-organization", "organization-a"}); err == nil || !strings.Contains(err.Error(), "requires rebuild-organization") { + t.Fatalf("orphan approval err=%v", err) + } + if err := migrateCommand([]string{"--rebuild-organization", "organization-a", "--approve-rebuild-organization", "organization-b"}); err == nil || !strings.Contains(err.Error(), "exact organization identifier") { + t.Fatalf("mismatched approval err=%v", err) + } +} + +func TestDashboardImportJSONIsStrictBoundedAndExplicitlyApproved(t *testing.T) { + valid := `{"version":1,"dashboard":{"slug":"operations","name":"Operations","description":"Portable view.","panels":[{"id":"panel-old","position":0,"title":"Requests","visualization":"table","saved_query_id":"query-old"}]},"saved_queries":[{"id":"query-old","name":"Requests","description":"Recent requests.","query":"logs | limit 10","scope":{}}]}` + bundle, err := readDashboardBundle(strings.NewReader(valid)) + if err != nil || bundle.Version != storage.DashboardVersion || bundle.Dashboard.Slug != "operations" { + t.Fatalf("bundle=%+v err=%v", bundle, err) + } + for name, body := range map[string]string{ + "unknown": strings.Replace(valid, `"version":1`, `"version":1,"secret":"no"`, 1), + "trailing": valid + `{}`, + "oversize": valid + strings.Repeat(" ", 1<<20), + } { + t.Run(name, func(t *testing.T) { + if _, err := readDashboardBundle(strings.NewReader(body)); err == nil { + t.Fatal("unsafe dashboard bundle was accepted") + } + }) + } + if err = importCommand([]string{"--organization-id", "organization-a"}); err == nil || !strings.Contains(err.Error(), "exact organization identifier") { + t.Fatalf("missing approval err=%v", err) + } +} + +func TestOfflineMigrationRefusesLiveDataDirectory(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + configPath := writeTestServerConfig(t, root, data) + lock, err := storage.AcquireProcessLock(data, false) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + err = migrateCommand([]string{"--config", configPath, "--development-allow-nonroot-config"}) + if err == nil || !strings.Contains(err.Error(), "active in another process") { + t.Fatalf("migration overlapped live lock: %v", err) + } +} + +func writeTestServerConfig(t *testing.T, root, data string) string { + t.Helper() + path := filepath.Join(root, "server.json") + body := fmt.Sprintf(`{"schema":1,"listen":"127.0.0.1:9010","public_url":"https://observatory.example","data_dir":%q,"max_body_bytes":1048576,"session_lifetime":"12h","query":{"max_duration":"2s","max_rows":1000,"max_scanned_bytes":10485760,"max_memory_bytes":8388608},"retention":{"raw_logs_days":30,"raw_traces_days":30,"raw_metrics_days":14,"cold_raw_days":400,"metric_rollups_days":400,"evidence_days":400}}`, data) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestReviewedDescriptorFileIsStrictPrivateAndNonSymlink(t *testing.T) { + descriptor := schema.Descriptor{ + Version: schema.DescriptorVersion, Signal: model.SignalMetrics, Field: "workshop.queue_depth", + Type: schema.TypeInteger, Meaning: "Reviewed queue depth for one service.", + Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, + Index: schema.IndexRange, Retention: schema.RetentionRaw, ProjectionVersion: 1, + } + body, err := json.Marshal(descriptor) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "descriptor.json") + if err = os.WriteFile(path, append(body, '\n'), 0o600); err != nil { + t.Fatal(err) + } + got, err := readReviewedDescriptor(path, false) + if err != nil || got != descriptor { + t.Fatalf("descriptor=%+v err=%v", got, err) + } + if err = os.WriteFile(path, []byte(`{"version":1,"signal":"metrics","field":"workshop.queue_depth","type":"integer","meaning":"Reviewed queue depth for one service.","sensitivity":"internal","cardinality":"low","index":"range","retention":"raw","projection_version":1,"unknown":true}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err = readReviewedDescriptor(path, false); err == nil { + t.Fatal("unknown descriptor property was accepted") + } + if err = os.WriteFile(path, body, 0o644); err != nil { + t.Fatal(err) + } + if err = os.Chmod(path, 0o644); err != nil { + t.Fatal(err) + } + if _, err = readReviewedDescriptor(path, false); err == nil { + t.Fatal("weak descriptor mode was accepted") + } + if err = os.Chmod(path, 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(filepath.Dir(path), "descriptor-link.json") + if err = os.Symlink(path, link); err == nil { + if _, err = readReviewedDescriptor(link, false); err == nil { + t.Fatal("descriptor symlink was accepted") + } + } +} diff --git a/cmd/observatory/release_test.go b/cmd/observatory/release_test.go new file mode 100644 index 0000000..856a1ff --- /dev/null +++ b/cmd/observatory/release_test.go @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "os" + "strings" + "testing" + + "gamertan.com/observatory/internal/config" +) + +func TestReleaseWebPushUsesAServiceScopedSystemdCredential(t *testing.T) { + serverBody, err := os.ReadFile("../../release/server.json") + if err != nil { + t.Fatal(err) + } + decoder := json.NewDecoder(bytes.NewReader(serverBody)) + decoder.DisallowUnknownFields() + var server config.Server + if err = decoder.Decode(&server); err != nil { + t.Fatal(err) + } + if err = decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + t.Fatal("release server configuration contains trailing JSON") + } + const runtimeKey = "/run/credentials/gamertan-observatory.service/web-push.json" + if server.MaxBodyBytes != 32<<20 || server.MaxConcurrentIngest != 8 || server.WebPush == nil || server.WebPush.PrivateKeyFile != runtimeKey || server.WebPush.Subject != "mailto:security@sandwichhime.com" || server.WebPush.QueueCapacity != 64 || server.WebPush.RequestTimeout != "10s" { + t.Fatalf("web push release configuration=%+v", server.WebPush) + } + + unitBody, err := os.ReadFile("../../release/observatory.service") + if err != nil { + t.Fatal(err) + } + unit := string(unitBody) + for _, required := range []string{ + "LoadCredential=server.json:/etc/gamertan-observatory/server.json", + "LoadCredential=web-push.json:/etc/gamertan-observatory/web-push.json", + "ExecStart=/opt/gamertan-observatory/current/observatory server --config %d/server.json --systemd-credential-config", + } { + if strings.Count(unit, required) != 1 { + t.Fatalf("release unit requires one %q", required) + } + } + if bytes.Contains(serverBody, []byte("private_key\"")) || bytes.Contains(serverBody, []byte("BEGIN PRIVATE KEY")) { + t.Fatal("release server configuration contains private-key material") + } +} + +func TestReleaseCaddyAccessLogIsFilteredAndBounded(t *testing.T) { + body, err := os.ReadFile("../../release/Caddyfile.observatory") + if err != nil { + t.Fatal(err) + } + configuration := string(body) + for _, required := range []string{ + "log observatory_access {", + "output file /var/log/caddy/observatory-access.jsonl {", + "mode 0640", + "roll_size 100mb", + "roll_keep 10", + "roll_keep_for 720h", + `request>uri regexp \?.*$ ""`, + "request>remote_ip delete", + "request>remote_port delete", + "request>client_ip delete", + "request>headers delete", + "resp_headers delete", + "user_id delete", + "log_append request_id {http.response.header.X-Request-ID}", + } { + if strings.Count(configuration, required) != 1 { + t.Fatalf("release Caddy configuration requires one %q", required) + } + } + for _, forbidden := range []string{"log_credentials", "Cookie", "Authorization", "query {"} { + if strings.Contains(configuration, forbidden) { + t.Fatalf("release Caddy configuration contains forbidden %q", forbidden) + } + } +} + +func TestReleaseAgentIsUnprivilegedScopedAndPrivacyMinimized(t *testing.T) { + body, err := os.ReadFile("../../release/agent.json") + if err != nil { + t.Fatal(err) + } + testPath := t.TempDir() + "/agent.json" + if err = os.WriteFile(testPath, body, 0o600); err != nil { + t.Fatal(err) + } + agent, err := config.LoadAgent(testPath, config.FilePolicy{}) + if err != nil { + t.Fatalf("validate release agent configuration: %v", err) + } + if agent.Schema != 1 || agent.ServerURL != "https://observatory.gamertan.com" || agent.CredentialFile != "/etc/gamertan-observatory/agent-credential.json" || agent.SpoolDir != "/var/lib/gamertan-observatory-agent" || agent.StateFile != "/var/lib/gamertan-observatory-agent/state.json" || agent.MaxSpoolBytes != 5<<30 || agent.MaxSpoolAgeText != "72h" || agent.BatchRecords != 5000 || agent.FlushInterval != "1s" { + t.Fatalf("release agent configuration=%+v", agent) + } + if len(agent.Sources) != 6 { + t.Fatalf("release agent source count=%d", len(agent.Sources)) + } + wantSources := []struct{ kind, path, stream string }{ + {"caddy_json", "/var/log/caddy/eqlwiki-edge.jsonl", "eql-edge"}, + {"requestlog_jsonl", "/var/log/eqlwiki/access.jsonl", "eql-application"}, + {"tend_events_jsonl", "/opt/gamertancom/deployment-events.jsonl", "tend-gamertancom"}, + {"tend_events_jsonl", "/opt/sandwich-hime-site/deployment-events.jsonl", "tend-sandwich-hime-site"}, + {"tend_events_jsonl", "/opt/gamertan-observatory/deployment-events.jsonl", "tend-observatory"}, + {"linux_metrics", "", "public-node-host-metrics"}, + } + for index, want := range wantSources { + source := agent.Sources[index] + if source.Kind != want.kind || source.Path != want.path || source.StreamID != want.stream || len(source.SensitiveFields) != 0 { + t.Fatalf("release agent source[%d]=%+v", index, source) + } + } + metrics := agent.Sources[5].LinuxMetrics + if metrics == nil || metrics.ProcRoot != "/proc" || metrics.CgroupRoot != "/sys/fs/cgroup" || len(metrics.Filesystems) != 1 || metrics.Filesystems[0].Name != "root" || metrics.Filesystems[0].Path != "/" || len(metrics.ControlGroups) != 6 { + t.Fatalf("release Linux metrics=%+v", metrics) + } + wantCgroups := map[string]string{ + "caddy": "system.slice/caddy.service", + "eql": "system.slice/system-eqlwiki.slice", + "gamertan": "system.slice/system-gamertancom.slice", + "sandwich-hime": "system.slice/sandwich-hime-site.service", + "observatory": "system.slice/gamertan-observatory.service", + "observatory-agent": "system.slice/gamertan-observatory-agent.service", + } + for _, selected := range metrics.ControlGroups { + if wantCgroups[selected.Name] != selected.Path { + t.Fatalf("unexpected release cgroup selector=%+v", selected) + } + delete(wantCgroups, selected.Name) + } + if len(wantCgroups) != 0 { + t.Fatalf("missing release cgroup selectors=%v", wantCgroups) + } + if bytes.Contains(body, []byte("observatory-access")) || bytes.Contains(body, []byte("sensitive_fields")) { + t.Fatal("release agent enables a feedback loop or sensitive evidence") + } + + unitBody, err := os.ReadFile("../../release/observatory-agent.service") + if err != nil { + t.Fatal(err) + } + unit := string(unitBody) + for _, required := range []string{ + "User=observatory-agent", + "Group=observatory-agent", + "SupplementaryGroups=caddy eqlwiki gamertancom", + "PartOf=gamertan-observatory.service", + "LoadCredential=agent.json:/etc/gamertan-observatory/agent.json", + "LoadCredential=agent-credential.json:/etc/gamertan-observatory/agent-credential.json", + "ExecStart=/opt/gamertan-observatory/current/observatory agent --systemd-credentials --config %d/agent.json --credential-file %d/agent-credential.json", + "StateDirectory=gamertan-observatory-agent", + "StateDirectoryMode=0700", + "NoNewPrivileges=yes", + "ProtectSystem=strict", + "ProtectProc=invisible", + "CapabilityBoundingSet=", + "AmbientCapabilities=", + "ReadOnlyPaths=/proc /sys/fs/cgroup /var/log/caddy /var/log/eqlwiki /opt/gamertancom/deployment-events.jsonl /opt/sandwich-hime-site/deployment-events.jsonl /opt/gamertan-observatory/deployment-events.jsonl", + "ReadWritePaths=/var/lib/gamertan-observatory-agent", + "MemoryMax=256M", + } { + if strings.Count(unit, required) != 1 { + t.Fatalf("release agent unit requires one %q", required) + } + } + for _, forbidden := range []string{"User=root", "docker.sock", "sh -c", "curl ", "wget ", "PrivateNetwork=yes"} { + if strings.Contains(unit, forbidden) { + t.Fatalf("release agent unit contains forbidden %q", forbidden) + } + } +} + +func TestSensitiveCaddyExampleKeepsAnExplicitProducerBoundary(t *testing.T) { + body, err := os.ReadFile("../../examples/Caddyfile.sensitive-access-log") + if err != nil { + t.Fatal(err) + } + configuration := string(body) + for _, required := range []string{ + "mode 0640", + "roll_size 100mb", + "roll_keep 10", + "roll_keep_for 720h", + "request>uri query {", + "delete access_token", + "delete api_key", + "delete authorization", + "delete code", + "delete credential", + "delete key", + "delete password", + "delete secret", + "\n\t\t\t\tdelete session\n", + "delete session_id", + "delete token", + "client_ip ip_mask 24 56", + "request>remote_ip delete", + "request>remote_port delete", + "request>client_ip delete", + "request>headers delete", + "resp_headers delete", + "user_id delete", + "log_append request_id {http.response.header.X-Request-ID}", + "log_append client_ip {http.request.client_ip}", + "log_append referrer {http.request.header.Referer}", + "log_append user_agent {http.request.header.User-Agent}", + } { + if strings.Count(configuration, required) != 1 { + t.Fatalf("sensitive Caddy example requires one %q", required) + } + } + for _, forbidden := range []string{"log_credentials", "Cookie", "Authorization"} { + if strings.Contains(configuration, forbidden) { + t.Fatalf("sensitive Caddy example contains forbidden %q", forbidden) + } + } +} diff --git a/docs/ADAPTIVE_PROJECTIONS.md b/docs/ADAPTIVE_PROJECTIONS.md new file mode 100644 index 0000000..1530c0f --- /dev/null +++ b/docs/ADAPTIVE_PROJECTIONS.md @@ -0,0 +1,293 @@ + + +# Adaptive time-partitioned projections + +Status: design candidate; not an implemented capability or release claim. + +## Purpose + +Observatory should preserve forensic evidence by default without requiring +every accepted observation to become a fully indexed row immediately. The raw +segment and control catalogue already contain the durable identity, tenant, +signal, source, sequence, byte size, and observed-time range needed to defer +most read-model work. + +The proposed model is an event lake with adaptive materializations: + +1. acknowledge immutable raw evidence and its small catalogue entry; +2. keep a bounded recent window available for live authorized views; +3. incrementally maintain only projections required by currently observed + views, explicitly warmed workloads, or reviewed central correlations; +4. build ad-hoc historical materializations on demand from the relevant raw + time partitions; and +5. evict any derived projection that no longer justifies its storage cost. + +Raw evidence remains authoritative. Every memory view, SQLite partition, +index, rollup, and query cache remains disposable and reconstructable. + +## Batch identity, not per-record deduplication + +An enrolled agent is trusted to classify and serialize the records it was +configured to collect. Observatory does not need to maintain a global +per-record deduplication index in the ingestion path. The durable unit is one +bounded authenticated batch identified by its enrolled source, stream, and +monotonic sequence. Its retained segment catalogue records the first and last +observed timestamps, record and byte counts, and the exact segment and +canonical logical-batch digests. Agent epochs separately namespace locally +generated alert-transition sequences across process-state replacement. + +The server derives organization, project, environment, and service scope from +the enrollment. It commits the immutable batch once, advances the stream +watermark atomically, and acknowledges its exact identity. Retrying the same +sequence and digest is an idempotent no-op. Reusing a sequence with different +bytes is quarantined as corruption. A sequence gap is retained as an explicit +continuity event rather than hidden. Timestamp ranges select storage and query +partitions but are not deduplication keys because legitimate batches may +overlap and host clocks may drift. + +### Implemented native v2 early-replay envelope + +The agent sends new native batches to `/api/v2/ingest/native`. A small +authenticated header envelope is available before the JSON record body, so an +exact replay takes a cheaper path. The envelope contains only: + +- protocol version, stream ID, signal, and monotonic sequence; +- the SHA-256 of the exact encoded body and the canonical logical-batch digest; +- record count and encoded byte count; and +- first and last observed timestamps. + +Source and tenant scope remain server-derived from the enrolled credential. +For a sequence already acknowledged with both digests, the server hashes and +discards the bounded body, returns the retained raw-segment identity, and avoids +JSON decoding, compression, catalogue writes, or projection wakeups. It must +still read the complete bounded body and verify its encoded digest before +acknowledging; trusting an enrolled agent does not make a buggy reused sequence +safe to accept without checking the bytes. + +For a new sequence, the existing validation and raw-first commit remain +mandatory. The server checks the declared count, byte size, signal, and time +bounds against the decoded batch before it commits or acknowledges it. An +envelope mismatch is a rejected batch, not a reason to rewrite telemetry. +Overlapping time ranges, late data, and repeated values remain valid. This is +an ingestion-cost optimization and audit index, not content-based record +deduplication. + +The legacy `/api/v1/ingest/native` endpoint remains available during the +preview compatibility window. A stream last accepted through v1 has blank +envelope columns. Its first matching v2 retry follows the full validation path +once, verifies the retained segment identity, and backfills the v2 metadata; +later exact retries use the early path. + +## Partition shape + +Partition first by organization, signal, and UTC time—not by arbitrary field +names or user-supplied identifiers. A starting comparison should use one +SQLite database per organization, signal, and UTC day. This keeps a normal +30-day hot window near 90 files per organization, permits logs, metrics, and +traces to project concurrently, narrows index working sets, and makes expiry a +bounded file operation. + +Hourly partitions may be evaluated for unusually high-volume days only after +the daily prototype is measured. Five-minute, hourly, and daily *rollups* are +aggregation resolutions; they should not be confused with the physical file +boundary. The partition catalogue must tell the planner which exact files and +rollup resolutions cover a requested window. + +Each materialized partition records: + +- organization, signal, UTC start and end; +- projection schema and generation; +- the exact raw segment digests or a durable high-watermark plus delta ledger; +- row, byte, and rollup counts; +- creation, last-query, and expiry times; +- build state and failure state; and +- the reviewed query evidence that justified eager or cached materialization. + +Late observations update only their observed-time partition. The new +generation is built beside the active one and atomically activated. Queries +must either use one complete generation or merge its explicit delta; they may +never silently omit late evidence. + +## Materialization classes + +### Live tail + +A bounded per-organization memory ring receives records only after their raw +segment has been durably acknowledged. It is keyed by segment digest and record +index, capped by both age and bytes, and safe to lose. It serves the recent log +tail, current metric samples, and trace arrivals while the disk projector is +behind. Authorization is checked before the ring is opened, and disk results +are deduplicated when a query overlaps the ring. + +SSE is the default UI transport for these server-to-client updates. It works +through ordinary HTTPS, has browser reconnection semantics, and is directly +testable. WebSockets remain a future option for a feature that needs genuine +bidirectional streaming; form mutations continue to use server-side POST, +redirect, and flash-message flows. + +### Agent-owned alert evaluation + +Single-source alert rules should normally execute at the enrolled agent that +already sees the stream. Rules are bounded, declarative, locally configured, +and versioned. The server cannot remotely add a source path, collector, shell +command, or executable rule. An agent evaluates its hot in-memory batch state +without requiring the server to project that stream and emits only bounded +rule-evaluation results: matched, clear, or an explicit evaluation error. The +server remains the authority that applies consecutive-match policy and turns +those evaluations into pending, firing, acknowledged, silenced, and resolved +incident states. + +Each transition is authenticated by its enrolled source and binds: + +- rule identifier and version; +- source, stream, and agent epoch; +- evaluation window and state; +- exact raw batch sequence range and segment digest evidence; and +- a monotonic transition sequence and observed timestamp. + +It contains no telemetry values by default. The server deduplicates the +transition and binds it to the exact retained raw segment before accepting it. +One agent cannot assert another source's scope. Reusing a sequence with +different bytes is corruption, while replaying the same sequence and digest +is an idempotent no-op. During the compatibility phase the server records this +source evidence without mutating incident state; the existing central +evaluator remains the oracle until differential comparisons prove equivalent +results and failure behavior. + +This model deliberately trusts an authenticated agent to classify and +evaluate its own data. It still protects ordinary retries, crashes, replay, +configuration drift, and a buggy agent. It cannot prove that a compromised or +offline agent reported honestly. The server therefore retains source +heartbeat, sequence-gap, and enrollment-revocation checks: a dead agent cannot +report its own absence. Cross-source or cross-service correlation remains an +explicit, bounded central workload rather than the default alert path. + +The first implementation slice evaluates filter-only log rules over one exact +durable batch. It deliberately excludes historical windows and cross-batch +state while the source and central results are compared. A raw batch remains +in the agent spool until both the raw admission and its source evaluation have +matching acknowledgements. This gives the transition durable retry behavior +without introducing a second telemetry store at the edge. + +Incident evidence labels distinguish `source_reported`, +`server_observed_absence`, and `centrally_correlated` conclusions. The current +server-side saved-query evaluator remains the compatibility oracle until edge +evaluation passes differential tests, disconnect/replay testing, and a +production soak. Merely pinning or saving a dashboard never enables eager +evaluation. + +### Warm + +Frequently used saved queries retain their materializations for a configured +period. Query evidence may propose warming or eviction, but an administrator +approves durable storage-policy changes. + +### Observed leases + +An authorized live Explore or dashboard session may create a bounded +materialization lease for its exact organization, query, and time window. +Every observer of the same materialization key shares one build/cache entry; +50 or 1,000 viewers increase notification fan-out, not projection work. The +SSE connection carries invalidation notices and renews demand; it does not +carry telemetry and is not itself storage authority. Equivalent future +WebSocket transport would obey the same lease contract. + +The key includes the canonical typed query AST, authorized scope, bounded time +window, descriptor generation, and materialization schema. A leased +materialization may contain a query-specific temporary index, but Observatory +does not create a permanent organization-wide index for every dashboard. +Repeated measured benefit may produce a reviewed warm-index proposal with its +estimated build, storage, and write-amplification cost. Approval and quota—not +viewer count alone—promote it to durable policy. + +Leases are deduplicated, expire after a disconnect grace period, and are +limited per user and organization. Connection churn cannot create a new +unbounded job, extend retention, or bypass query cost and sensitive-field +permissions. A completed materialization may cool into the ordinary warm cache +or be evicted. Agent-owned rules and server heartbeat checks remain +independent of browser observers; a central cross-source rule acquires its own +explicit, bounded lease. + +### Lazy + +An ad-hoc historical query first uses the segment catalogue to identify exact +time and scope candidates. A small query may scan them directly under the +ordinary time, byte, row, and memory budgets. A larger query creates a bounded +materialization job. The UI reports its state and progress, then refreshes the +result through SSE; the no-JavaScript path remains a normal status page with a +manual refresh. + +The first query is allowed to be slower. It is not allowed to become an +unbounded request, hold an HTTP connection indefinitely, or bypass query +budgets. A materialization that exceeds its approved budget stops with a +specific, resumable status rather than the generic “temporarily unavailable” +response. + +## Read and write separation + +The ingestion plane is append-oriented: validate, compress, checksum, sync, +catalogue, acknowledge. It does not wait for query indexing. + +The read plane is projection-oriented: select only authorized organization, +signal, and time partitions; use exact rollups where semantically valid; merge +partition-local partial results; then apply the final sort and limit. Query +planning limits partition fan-out and reports the raw bytes, materializations, +indexes, permissions, and expected cold work in `explain`. + +This preserves SQLite's operational simplicity while allowing concurrent +writers across independent partition files. It also creates a fair test of +SQLite itself: if partition-local insertion remains the measured bottleneck +after index review, the same raw segments and partition contract can be used +to compare another embedded representation. + +## What this does not solve automatically + +- A memory tail improves visibility, not durable projector throughput. +- Lazy materialization trades continuous background cost for first-query cost. +- Too many tiny partitions increase file, migration, and planning overhead. +- Cross-partition summaries need deterministic mergeable aggregate state. +- Agent rules still consume bounded local CPU and memory, and central + heartbeat checks still run when no user is online. +- Cross-source rules require reviewed central work and cannot be reduced to + one agent's local stream. +- Agent compromise or suppression remains a trust boundary; authentication + proves which enrolled source reported an event, not that the host itself was + honest. +- Historical regular-expression or high-cardinality queries still require + strict scan and memory budgets. +- Application-supplied sensitive data remains governed by descriptor, + authorization, retention, and export policy regardless of storage tier. + +## Staged proof + +1. Add aggregate cost-centre instrumentation without changing behavior. +2. Generalize the existing budgeted cold-segment reader to prove authorized + bounded queries over selected hot raw segments; compare every result with + the current SQLite projection. +3. Add a strict partition catalogue and build one signal/day materialization + beside the current projection. Differentially test queries, late data, + duplicates, corruption, cancellation, and restart. +4. Add the bounded recent-memory view and prove overlap deduplication. +5. Move one single-source rule to agent evaluation and one observed dashboard + to a shared lease. Differentially compare the rule with the current server + evaluator and prove that many dashboard viewers do not duplicate work. +6. Prove agent restart, offline spool, sequence gaps, duplicate transitions, + changed rule versions, unavailable-node detection, and lazy evidence + verification. Keep one reviewed cross-source rule central. +7. Run same-host 100,000- and one-million-observation comparisons, then the + four-CPU/eight-GiB release campaign. +8. Stop universal row projection only after the adaptive path reproduces the + current query, incident, retention, rebuild, and authorization behavior. + +The first slice of step 2 is implemented as a logs-only, non-public candidate +that reads both hot and cold retained batches under the ordinary typed-query +budgets. Differential tests cover filters, sorting, summaries, regular +expressions, resource scope, an absent projection, and incomplete retention +transitions. The production projection remains the oracle. This candidate +holds the organization lock during its scan and therefore is not yet the +catalogue-snapshot or shared-lease implementation described above. + +The published capacity gate is not weakened to make this design pass. If the +product intentionally changes first-query behavior, that contract and its +separate warm-query boundary must be reviewed explicitly before the fixture is +changed. diff --git a/docs/AGENT.md b/docs/AGENT.md new file mode 100644 index 0000000..fdbbb6c --- /dev/null +++ b/docs/AGENT.md @@ -0,0 +1,263 @@ + + +# Agent boundary + +The Observatory agent is an unprivileged outbound collector. Its local JSON +configuration is authoritative: the server cannot add a path, collector, +command, socket, or environment value remotely. + +Configuration and the enrolled credential remain separate root-owned +mode-`0600` regular files under `/etc`. The example systemd unit uses +`LoadCredential=` to expose private, read-only runtime copies to the dedicated +service account. `observatory agent --systemd-credentials` confines both paths +to direct children of systemd's `CREDENTIALS_DIRECTORY`; the agent itself does +not run as root. Its service account receives read access only to the explicitly +selected Caddy, application, and Tend streams. It does not need a Docker socket, +an inbound port, a shell plugin, or write access to the source logs. + +The producer and its operator remain authoritative for what a source log is +allowed to contain. Observatory is not a universal DLP system and cannot infer +an organization's private vocabulary or prove that a renamed field is safe. +The initial adapters therefore use a documented fixed field set before durable +spooling, plus a small source-specific `sensitive_fields` list. This is a +defence-in-depth boundary and a useful safe default—not permission to log a +secret upstream. An absent list means the privacy-minimized fields below, and +Observatory never widens it remotely. + +- Caddy: method, query-free path, status, response bytes, duration, and a + syntactically bounded request ID. The shipped Caddy example appends that ID + from the completed response as a dedicated top-level field, then removes + the complete request-header and response-header maps, client addresses, + user identity, and the URI query before the edge record reaches disk; +- Web Foundations requestlog: method, normalized route, status, response bytes, + duration, authorization outcome, and request ID; +- Tend: the strict version-1 bounded activation and rollback event fields; + unknown, malformed, unsafe, oversized, or trailing data is rejected. + +The optional `linux_metrics` source reads aggregate CPU ticks, load, uptime, +memory, network counters, named filesystem capacity, selected cgroup v2 +counters, and processes named by explicit PID files. It never enumerates all +processes, reads command lines or environments, invokes a shell, or opens a +collector port. Configuration assigns stable public labels so local paths and +PIDs are not stored as telemetry attributes. A disappearing selected process +or cgroup emits an `up=0` metric while other host evidence remains durable. + +Cookies, authorization headers, request and response bodies, arbitrary +headers, and unknown JSON fields never enter the native batch. By default, +raw query strings, client addresses, referrers, user agents, and anonymous +session identifiers do not enter it either. An application whose documented +privacy policy and operating need permit richer evidence may opt in per source: + +```json +{ + "kind": "caddy_json", + "path": "/var/log/caddy/example-access.jsonl", + "stream_id": "caddy-access", + "sensitive_fields": ["client_ip", "query", "referrer", "user_agent"] +} +``` + +`caddy_json` accepts `client_ip`, `query`, `referrer`, and `user_agent`. +`requestlog_jsonl` accepts those fields plus `session_id`. Unsupported and +duplicate names fail configuration validation. Values remain bounded, and IP +addresses must parse as IPv4 or IPv6. Selected values enter Observatory as +unknown sensitive, high-cardinality, unindexed raw fields until an authorized +schema review explicitly classifies them. Query and referrer values can carry +credentials or personal data even when collection is disclosed; filter known +secret-bearing parameters at the producer, restrict access, and choose a +retention window appropriate to the application's policy. The copyable +`Caddyfile.sensitive-access-log` demonstrates the intended producer boundary. + +Source scope is not read from any log; it comes from the server-side enrollment +record after credential verification. + +## Batching and delivery cadence + +`flush_interval` is the maximum delay between collection cycles while the +agent is healthy. `batch_records` caps the number of observations read into +one durable batch for one stream during that cycle; it is not a reason to wait +for a quiet stream to fill. Each batch is committed to the local spool before +its source cursor advances, and delivery removes it only after an exact server +acknowledgement. + +Larger batches amortize filesystem, compression, HTTPS, and server admission +costs. A shorter interval reduces live-view latency. The checked-in profile +uses up to 5,000 records once per second. Operators can select any validated +combination from 1 to 5,000 records and 100 milliseconds to one minute based +on source volume and desired freshness. The tailer also caps bytes read during +one cycle, so an existing multi-gigabyte file is caught up over bounded cycles +rather than loaded into memory at once. + +The file tailer reads at most 4 MiB of source data per cycle while the checked- +in server accepts at most 32 MiB per authenticated request. That headroom +accounts for native-batch JSON framing without making request memory +unbounded. The server also admits at most eight concurrent authenticated +ingestion bodies. A future byte-aware batcher should make the source/request +relationship explicit for custom profiles that choose different ceilings. + +The spool fails closed at its configured byte or age budget. A server +acknowledgement must match the exact source, stream, sequence, and canonical +logical-batch digest before the agent removes its local batch. The local spool +envelope retains a separate digest because it also contains the cursor +checkpoint; private storage encodings are not treated as protocol identities. +Each pending envelope carries +a bounded cursor checkpoint. Startup applies those checkpoints before it reads +new bytes, so a crash between the spool commit and state-file replacement +cannot duplicate a different payload under the same sequence. Collected records +advance only after that envelope is durable; deliberately discarded complete +records advance only after their bounded counters and cursor are durably saved. + +Delivery uses the versioned native v2 endpoint. The agent supplies bounded +stream, sequence, signal, exact encoded-body digest, canonical logical-batch +digest, record/byte counts, and first/last observation times as request +metadata. Source and organization scope still come only from the enrolled +credential. On an exact retry the server reads and hashes the complete bounded +body but does not decode, recompress, recatalogue, or reproject it. Time ranges +help select storage and query partitions; overlapping ranges and repeated +values remain valid and are never treated as record-level duplicates. + +## Local alert rules + +An agent may evaluate an optional, locally configured rule against one exact +log batch after that batch is durable. The first supported rule shape is +deliberately narrow: one `caddy_json` or `requestlog_jsonl` stream, one or more +typed `where` filters, no sort, summary, or historical window, and a bounded +minimum match count. It uses the same typed filter implementation as the +central query evaluator. + +```json +{ + "alert_rules": [ + { + "version": 1, + "id": "http-failures", + "revision": 1, + "stream_id": "application-request", + "query": "logs | where status >= 500 | limit 10", + "minimum_matches": 1 + } + ] +} +``` + +The identifier and revision must name an enabled, source-scoped server rule +whose saved query has the same intended semantics. Configuration remains +local; the server cannot install or edit this rule. After the raw batch is +acknowledged, the agent reports only `matched`, `clear`, or `error`, its local +rule identity, the exact raw segment identity, the batch time range, and a +monotonic sequence. It sends no telemetry values in that transition. + +The raw spool entry is removed only after both acknowledgements match. If the +transition request fails, retrying first replays the same raw batch as an +idempotent no-op and then replays the same transition. During this +compatibility phase the server stores the authenticated source result but does +not mutate incident state from it. The existing central evaluator remains the +oracle until differential tests and a production soak demonstrate equivalent +results and failures. Multi-batch windows, cross-source rules, and server-owned +incident confirmation remain central work in this preview. + +The tailer preserves incomplete lines, discards oversized or malformed +complete records without retaining their contents, caps read work per cycle, +detects copy-truncation, and follows an unread inode through a rename in the +configured file's directory. If the old inode has already disappeared, the +agent records a discontinuity and begins the new file; it does not claim that +unrecoverable bytes were observed. + +An organization-authorized local administrator writes a short-lived +enrollment token directly to a new mode-`0600` file. `observatory agent enroll` +exchanges it once over HTTPS and creates the configured credential file without +overwriting any existing secret. If the credential cannot be written, the +client asks the server to revoke the newly enrolled source. + +Create the source's resource hierarchy through the supported offline +administration commands before issuing an enrollment. Each command acquires +the same shared process lock as the live server, validates its exact parent +scope, requires the actor's `organization.manage` grant, and returns only +generated public identifiers. Normal hierarchy and enrollment administration +can run while the server is live. Exclusive migrations still require the +server to stop. Never edit the control database directly. + +```sh +sudo observatory admin project create \ + --config /etc/gamertan-observatory/server.json \ + --actor-user-id USER_ID \ + --organization-id ORGANIZATION_ID \ + --slug gamertan \ + --name 'Gamertan' + +sudo observatory admin environment create \ + --config /etc/gamertan-observatory/server.json \ + --actor-user-id USER_ID \ + --organization-id ORGANIZATION_ID \ + --project-id PROJECT_ID \ + --slug production \ + --name 'Production' + +sudo observatory admin service create \ + --config /etc/gamertan-observatory/server.json \ + --actor-user-id USER_ID \ + --organization-id ORGANIZATION_ID \ + --project-id PROJECT_ID \ + --environment-id ENVIRONMENT_ID \ + --slug web \ + --name 'Web application' +``` + +Creating the hierarchy is intentionally incremental: a successfully created +project remains a valid resource if a later environment or service command is +rejected. Slugs are unique within their parent, and failed authorization or +parent validation creates nothing. + +```sh +sudo observatory admin enrollment create \ + --config /etc/gamertan-observatory/server.json \ + --actor-user-id USER_ID \ + --source-id HOST_SOURCE_ID \ + --organization-id ORGANIZATION_ID \ + --project-id PROJECT_ID \ + --environment-id ENVIRONMENT_ID \ + --service-id SERVICE_ID \ + --lifetime 15m \ + --output-file /etc/gamertan-observatory/new-agent-enrollment.json + +sudo observatory agent enroll \ + --config /etc/gamertan-observatory/agent.json \ + --enrollment-file /etc/gamertan-observatory/new-agent-enrollment.json + +sudo systemctl enable --now observatory-agent.service +``` + +The token and credential never appear in command arguments: only their file +paths do. Remove the enrollment-token file after a successful exchange. The +copyable unit in `examples/observatory-agent.service` documents the supported +unprivileged runtime boundary. + +## Production dogfood profile + +The checked-in `release/agent.json` and `release/observatory-agent.service` +describe the first production profile. One host-scoped source credential is +bound to the public application node. Locally authoritative collectors keep +EQL's edge and application evidence in distinct streams and collect aggregate +Linux and selected service-cgroup metrics. This scope does not imply that all +telemetry belongs to the Observatory application merely because Observatory +receives it. + +The profile intentionally omits `sensitive_fields`. Client addresses, query +strings, referrers, user agents, and session identifiers therefore remain out +of the agent spool and server. They may be enabled later, per stream, only +after the producer filter, privacy policy, access grants, and retention window +have been reviewed. The capability remains supported; privacy minimization is +the production default rather than an architectural prohibition. + +The Observatory origin's own Caddy access log is not collected by this first +profile. Ingesting the agent's requests from that log would create a perpetual +self-observation loop. The profile does collect the bounded Tend event files +for Gamertan, Sandwich Hime, and Observatory as three independent streams. +Their application release state remains authoritative; ingestion is evidence, +not a deployment dependency. + +The service account has supplementary read-only membership in the `caddy` and +`eqlwiki` groups, write access only to its private state directory, no inbound +listener, no Docker socket, and no shell. `PartOf=gamertan-observatory.service` +restarts the agent when a Tend activation moves the shared `current` binary, +so server and agent do not drift across releases. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..c1c7720 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,201 @@ + + +# Architecture + +Observatory preserves three independent evidence layers: + +1. an agent's local spool retains unacknowledged batches; +2. immutable raw segments are committed and checksummed before acknowledgement, + then move unchanged from hot to cold forensic storage; +3. per-organization SQLite projections are disposable query accelerators. + +Durable acceptance and query visibility are deliberately separate. A source +may receive an exact acknowledgement as soon as its immutable segment and +control-plane identity are durable. A bounded background projector then makes +that evidence visible to queries. The authenticated interface reports this +lag for the selected organization instead of pretending that accepted and +indexed mean the same thing. + +The control database stores Web Foundations identities, organizations, scoped +access grants, source credential digests, sequence watermarks, segment +projection state, dashboards, alert rules, incident metadata, and optional +user-owned Web Push endpoints and their organization-scoped mappings. Every +organization's query projection is stored in a distinct SQLite database. +Authorization is applied before opening that projection. + +Saved queries and dashboards live in the control database, but every key and +foreign-key relationship includes the organization identifier. Saved query +text is parsed into the same typed AST at write time; both representations are +stored and revalidated for exact semantic agreement when read. Updates use +optimistic revisions. Dashboard panels reference only saved queries from the +same organization. Source-control exports deliberately omit tenant IDs, +operator IDs, timestamps, and internal revisions. + +The web interface is an ordinary `net/http` application rendered through +compiler-generated Sandwich Hime components. The application buffers complete +HTML before committing status and headers. Hime-san remains a development and +CI tool; the service links only the pinned Sando runtime. Content-hashed CSS +and JavaScript are embedded in the binary and served under immutable URLs. +JavaScript opens an authenticated SSE connection only to learn that newer data +exists. The stream carries no telemetry or resource labels, and dropping it +does not remove any navigation, query result, table, or authentication +function. + +The text language and visual builder both produce the same versioned typed AST. +Planning receives an already-authorized organization/resource scope separately +from that AST; query text cannot select a tenant. Explain output names the +scoped projected source, reviewed descriptors and indexes, estimated scan, +cache eligibility, budgets, and required permissions. Unknown fields remain +raw-query candidates but default to sensitive, high-cardinality, and unindexed. +Execution opens only the already-authorized organization's projection in +query-only mode. Server-owned project, environment, and service constraints +are added independently of the AST; the executor applies typed comparisons, +bounded RE2 regular expressions, time windows, grouping buckets, aggregates, +sorting, row limits, and separate time, decoded-byte, and memory ceilings. +Only default safe columns and fields explicitly referenced by the query enter +the result table. `window 24h` is a lookback; `window(5m)` is a distinct +aggregation bucket in the versioned AST. + +Alert rules reference a saved query in the same organization. A single +evaluator claims each due rule by advancing its next-evaluation time before it +opens the organization's query projection, preventing concurrent duplicate +evaluation. The ordinary query budget remains authoritative. A configured row +threshold determines whether an evaluation matches; consecutive matches move +an incident from pending to firing. Query failure records only the bounded +code `query_unavailable` and leaves an existing incident unchanged. Incident +state and its append-only event sequence live in the control database, contain +no query result or telemetry value, and remain organization-keyed on every +relationship. + +Web Push remains downstream of durable incident state. Only a committed +transition into `firing` enters a bounded, deduplicated in-memory queue. The +dispatcher reauthorizes `incidents.read`, encrypts one fixed generic sentence, +and uses a redirect-free HTTPS client that rejects non-public DNS results. +Vendor delivery cannot block rule evaluation, and delivery failure cannot +change incident state. The service worker ignores payload content and opens +the authenticated application rather than carrying a tenant or incident +identifier through the push relay. + +Unknown attribute keys produce descriptor proposals only after their raw +segment is committed. Proposal evidence is aggregated once per segment using +the segment digest, organization, and field as the idempotency identity. The +control database stores counts, estimated bytes, inferred type, first/last +observation times, and a generic example query—not observed values. Built-in +reviewed fields are excluded. Proposals remain sensitive, high-cardinality, +and unindexed; proposal persistence does not activate a descriptor. + +Descriptor activation is an explicit organization-authorized operation. A +reviewed descriptor is loaded from a bounded private JSON file; unknown +properties and weak or symlinked files are rejected. Observatory copies the +currently active descriptors into a new per-organization projection version, +builds a typed custom-field index by scanning the disposable observation +projection, and changes the active version in the same SQLite transaction. +Invalid historical values are omitted rather than coerced. Existing and newly +ingested observations use the active registry after commit, while prior index +tables remain available for recovery. The control-database proposal status is +held under a write claim during the build and acknowledged only after the +projection commit; retry repairs an interrupted acknowledgement idempotently. + +## Ingestion transaction + +1. Authenticate the source credential and load its server-owned scope. +2. For native v2, validate the bounded transport envelope. If it exactly + matches the current acknowledged stream watermark, hash and discard the + complete body, recheck the watermark under the source lock, and return the + retained acknowledgement without decoding or writing it again. +3. For a new batch, validate decompressed size, record count, field bounds, + timestamps, signal type, sequence, and every envelope/body field. +4. Commit a content-addressed zstd raw segment with `fsync` and atomic rename. +5. Serialize organization quota admission and reject the exact segment before + cataloguing it when the approved quota is exhausted. +6. In one control-database transaction, catalogue the committed segment and + advance the `(source, stream)` watermark with its bounded envelope metadata. +7. Acknowledge the exact sequence, + raw-segment digest, and canonical logical-batch digest. +8. Outside the acknowledgement path, group a bounded number of one + organization's pending segments into one projection transaction. Project + at most four distinct organizations concurrently, while keeping each + tenant's base projection, active custom index, metric rollups, and replay + ledgers in its own transaction. +9. Aggregate unknown-field proposal evidence without retaining observed + values, then mark each projected segment complete in the control database. + +The dogfood server bounds an authenticated request at 32 MiB and admits at +most eight ingestion bodies concurrently. File agents read no more than 4 MiB +of source text into a cycle. These independent bounds preserve batching +headroom while preventing source credentials from creating unbounded request +memory pressure. + +A retry of the same sequence and digest is idempotently acknowledged even +while projection is pending. Reusing a sequence with different bytes or +submitting an older sequence is rejected. If projection fails, the durable +segment remains pending and the projector retries it without holding the +source or listener hostage. Work is bounded and selected fairly across +organizations so one tenant's corrupt segment cannot prevent other tenants +from advancing. + +Server startup first reconciles raw objects and control metadata without +decoding already catalogued objects, then opens the listener and drains +pending projections in the background. The one-shot `check` and `migrate` +commands retain a blocking recovery path for offline verification. + +Every projected metric sample also updates a five-minute aggregate in the same +organization projection transaction. Rollups retain exact count, sum, minimum, +and maximum plus a deterministic bounded histogram for explicitly approximate +percentiles. Only reviewed, non-sensitive, bounded-cardinality dimensions with +the `metric` retention class enter that longer-lived projection. Unknown and +sensitive attributes remain raw-only. Summary queries that can be represented +faithfully select the rollup explicitly in their explain plan; unsupported +query shapes continue to use raw samples. + +The server applies retention after recovery and once per hour. Projected rows +expire at their signal's hot cutoff. Once every record in a segment is outside +that hot window, the control database first records an exact cold destination, +then the unchanged zstd object is atomically renamed into the private cold +tree, and finally its catalog path and tier advance. Recovery completes either +side of that rename idempotently. Cold segments remain checksum-verifiable and +queryable through a slower scan path with the ordinary authorization and +resource budgets. Cold raw evidence is preserved indefinitely by default. +Only a policy with `delete_cold_raw` explicitly enabled marks segments beyond +its final cold cutoff as retiring and removes them through a separate +crash-recoverable lifecycle. Resolved incidents and security audit evidence +use the configured evidence window. Organization overrides may shorten +defaults. Extending a finite server policy to indefinite preservation requires +an exact organization approval and an enforceable storage quota. + +The agent has a separate durability boundary. It commits a private, +checksummed zstd envelope before attempting HTTPS delivery and removes it only +after an acknowledgement binds the exact logical batch. Its envelope digest +continues to protect the checkpoint and compressed local bytes. Collection +adapters use fixed whitelists so known credentials and high-risk fields are +absent before the spool write. Server configuration cannot alter the agent's +selected local sources. + +OTLP/HTTP accepts the standard protobuf wire shape for logs, metrics, and +traces without enabling gRPC, a collector listener, or remote configuration. +The server authenticates the enrolled source before decoding, applies limits +to both compressed and decompressed bytes, removes known credential-bearing +attribute keys, and injects the source's stored resource scope. OTLP payloads +cannot select an organization, project, environment, or service. Automatic +stream sequences are assigned under a per-source lock; native batches retain +their explicit replay-safe sequence and digest acknowledgement contract. + +Linux host metrics are collected in the same unprivileged agent cycle and +enter the same durable spool. The collector reads a fixed set of bounded +`/proc` and cgroup v2 files, named filesystem statistics, and only process IDs +resolved through explicitly configured PID files. Files are required to be +regular and non-symlinked; cgroup traversal is confined component-by-component +beneath the configured root. Local paths, raw PIDs, command lines, and +environment values are not projected. + +## Tenant boundary + +Payloads contain no organization, project, environment, or service selector. +Those values come from the enrolled source record after credential +verification. Published Web Foundations Preview 3 supplies user identities, +personal and shared organizations, resources, teams, invitations, and scoped +query grants, plus the forced initial-password rotation contract. Platform +operator status is a separate policy and does not authorize telemetry access. +A query request names a desired resource, but the server independently +authorizes that scope and computes sensitive-field access before it constructs +the planner input. diff --git a/docs/BOOTSTRAP.md b/docs/BOOTSTRAP.md new file mode 100644 index 0000000..a624201 --- /dev/null +++ b/docs/BOOTSTRAP.md @@ -0,0 +1,152 @@ + + +# Local platform bootstrap + +Bootstrap is deliberately local and single-use. It creates the first user, +marks that user as a platform operator, creates their personal organization, +and grants owner access only inside that organization. Platform operation does +not imply access to another organization's telemetry. + +Production configuration and the generated password file must be root-owned, +mode `0600`, regular files. Observatory creates the file exclusively and +refuses an existing path. The password is never a command argument, output +field, manifest value, log value, or persisted cleartext value. + +```sh +sudo observatory admin bootstrap \ + --config /etc/gamertan-observatory/server.json \ + --username operator \ + --email operator@example.com \ + --display-name 'First Operator' \ + --generate-password-file /root/observatory-bootstrap-password +``` + +The command returns only the opaque user and personal-organization IDs plus +`password_change_required: true`. Read the file locally, sign in, and replace +the temporary credential. That session is restricted to password rotation, +logout, health, and immutable application-shell assets. A successful change +revokes every session, clears the browser cookie, and requires a fresh login. +Delete the generated file only after the replacement login succeeds. + +## Recover local access + +If an operator loses a credential, recover it only from the host. Observatory +does not expose a password-reset route, email flow, recovery token API, or +registration window: + +```sh +sudo env CREDENTIALS_DIRECTORY=/run/credentials/gamertan-observatory.service \ + observatory admin user reset-password \ + --config /run/credentials/gamertan-observatory.service/server.json \ + --systemd-credential-config \ + --identifier speelman \ + --generate-password-file /root/observatory-password-recovery +``` + +The command creates the output file exclusively as a root-owned regular file +with mode `0600`. It then atomically installs that one-time credential, restores +the password-change requirement, revokes every session, and appends a generic +audit event without the credential or its path. If the database operation +fails, the generated file is removed. Read the file locally, complete the +server-rendered password-change form, and delete the file after the new login +succeeds. Never place the credential in shell arguments, chat, logs, manifests, +or deployment state. + +There is no public first-user registration window. The bootstrap command is +single-use, local, and refuses a second platform user. If an operator already +has an appropriately generated password, `--password-file` remains available; +that explicitly supplied credential is treated as permanent and does not set +the forced-change flag. The two password flags are mutually exclusive. + +If bootstrap fails after creating a generated credential, Observatory removes +the file after revalidating its exact path, type, owner, and mode. A process +crash between file creation and database bootstrap can leave a credential with +no account; inspect the control database and remove that exact file before a +reviewed retry. Never replace the path with a symlink or broaden its mode. + +When the installed service loads its configuration and Web Push key through +systemd credentials, use that same confined runtime view rather than copying +or changing credential modes: + +```sh +sudo env CREDENTIALS_DIRECTORY=/run/credentials/gamertan-observatory.service \ + observatory admin bootstrap \ + --config /run/credentials/gamertan-observatory.service/server.json \ + --systemd-credential-config \ + --username operator \ + --email operator@example.com \ + --display-name 'First Operator' \ + --generate-password-file /root/observatory-bootstrap-password +``` + +The systemd mode accepts only a direct child of the declared credential +directory, owned by root or the current service user, with mode `0400`, `0440`, +or `0600`. It is mutually exclusive with the local non-root development flag. +Every command that loads the server configuration—including user, invitation, +resource, enrollment, descriptor, retention, query, import, export, and +migration operations—accepts the same `--systemd-credential-config` boundary. +Operators do not need to copy a runtime credential or weaken its mode merely +to run an administrative command. + +The browser login form carries an independent, short-lived, unpredictable +token in a Secure, HttpOnly, SameSite=Strict cookie and the rendered form. +Authenticated forms use separate purpose-bound tokens derived from the opaque +session. These tokens are the primary CSRF proof for ordinary server POSTs. +`Origin` and Fetch Metadata are independent contradiction checks: explicitly +cross-site requests and tokenless requests remain rejected, while missing or +opaque metadata from privacy tools cannot prevent a valid form submission. +Failures return a fresh server-rendered form with bounded guidance and never +echo the submitted identifier or password. + +## Add a user through an invitation + +Later users are provisioned locally. Each receives an automatically created +personal organization, but no access to anyone else's telemetry. Create a +private password file without placing the password in shell history, then run: + +```sh +sudo observatory admin user create \ + --config /etc/gamertan-observatory/server.json \ + --username responder \ + --email responder@example.com \ + --display-name 'Incident Responder' \ + --password-file /etc/gamertan-observatory/responder-password +``` + +An owner of the destination organization creates an expiring, single-use +invitation. The owner must supply their opaque user and organization IDs: + +```sh +sudo observatory admin invitation create \ + --config /etc/gamertan-observatory/server.json \ + --actor-user-id USER_ID \ + --organization-id ORGANIZATION_ID \ + --email responder@example.com \ + --lifetime 15m \ + --output-file /etc/gamertan-observatory/responder-invitation +``` + +The token is written only to the new mode-`0600` file. Existing paths, +symlinks, multiline values, and weak files fail closed; a database invitation +is cancelled automatically if its token file cannot be persisted. Command +output contains metadata, never the token. Transfer the file through a +separately protected channel and accept it locally: + +```sh +sudo observatory admin invitation accept \ + --config /etc/gamertan-observatory/server.json \ + --user-id INVITED_USER_ID \ + --invitation-file /etc/gamertan-observatory/responder-invitation +``` + +Acceptance requires the provisioned user's exact normalized email address. +The token cannot be reused. Remove both password and invitation files after +the user confirms access. The preview intentionally has no public registration, +email-delivery protocol, or self-service invitation UI. + +Bootstrap and later user provisioning each create a user, personal +organization, and access grant through separately checked storage operations. +Until that sequence becomes one control-database transaction, retain a +verified control-database backup before provisioning identities. If a command +reports a partial provisioning failure, stop and restore or inspect the +database rather than blindly retrying it. diff --git a/docs/CAPACITY.md b/docs/CAPACITY.md new file mode 100644 index 0000000..3ab15a1 --- /dev/null +++ b/docs/CAPACITY.md @@ -0,0 +1,186 @@ + + +# Capacity campaign + +Observatory's release capacity gate is executable and intentionally narrow. It +measures this implementation on one 4-vCPU/8-GiB Linux boundary; it is not a +claim of generic superiority or unlimited scale. + +The campaign: + +- sustains 2,000 mixed observations per second for one hour across four + equally weighted organizations so concurrent isolation and all four + constrained CPUs are exercised; +- absorbs 10,000 mixed observations per second for 60 seconds; +- requires p95 ingestion-to-query visibility below two seconds; +- expands the primary organization to at least ten million observations and + runs indexed log and metric-rollup views over its last 24 hours, requiring + every p95 below three seconds; +- places 72 ordered batches at the edge of the 72-hour outage budget, replays + them, recognizes a duplicate, and removes local evidence only after an exact + logical-batch acknowledgement; +- archives expired raw objects unchanged, queries cold evidence, and removes + only material beyond the final cold cutoff; and +- records bounded RSS and disk growth without emitting a path, host, user, + credential, source value, or telemetry value. + +The aggregate release gate is not a claim that one organization can ingest +10,000 observations per second through its deliberately serialized SQLite +projection. The same fixture accepts `-primary-phase-weight 7` for a separate +70-percent hot-tenant diagnostic and records that weight in its JSON report. +The ten-million-observation query corpus is still built in the primary +organization after the timed phases. + +The workload producer and visibility observer run independently. The observer +queries each primary log batch while traffic continues and records when that +timestamp reaches the read model; it never makes the producer wait for its own +query. The achieved ingest rate therefore measures scheduled durable +acceptance, while the separate visibility distribution still captures +background projection lag under the same load. + +Run a short development proof with: + +```sh +./scripts/capacity-campaign.sh +``` + +The ordinary public-preview gate runs that development proof after the +complete deterministic verifier and five-second smoke passes for each fuzz +target: + +```sh +./scripts/preview-gate.sh +``` + +This is the default dogfood gate. The hour, ten-million-observation, extended +fuzz, fleet, and 24-hour campaigns are milestone tools. They are not required +for every interface iteration and must not be implied by a preview that did +not run them. + +The release campaign must run inside an externally enforced four-CPU, +8-GiB cgroup and requires an explicit mode: + +```sh +OBSERVATORY_CAPACITY_MODE=release ./scripts/capacity-campaign.sh +``` + +The resulting JSON is aggregate evidence. A passing development run does not +replace a run against the exact packaged release commit, its immutable digest, +or the later medium-fleet soak. A campaign that completes every stage but +misses a terminal rate, visibility, query, or memory gate still emits the same +aggregate report with `pass: false` before returning a failure. Earlier stage +failures remain fail-closed and may produce only the bounded stage marker and +error. Individual visibility samples may be observed for up to 30 seconds so +one outlier does not erase the phase evidence; the release boundary remains +the aggregate p95 below two seconds. + +Report schema version 2 adds the backlog present immediately after synthetic +fill, projection-drain duration, storage bytes by raw/control/projection class, +and primary-projection SQLite page bytes by checked-in schema object. These are +aggregate performance counters, not telemetry export. + +## August 17 constrained-run observations + +The first full-duration campaign found a concurrent first-write directory +creation race after completing its one-hour sustain phase. The directory +validator was corrected to tolerate `EEXIST` only long enough to re-inspect the +exact private, non-symlink directory chain, and a 64-writer regression now +protects that boundary. + +The corrected implementation was then exercised in a disposable Debian 12 +container with four CPUs, 8 GiB of memory, a 1,024-process ceiling, a read-only +root filesystem, no network, all Linux capabilities dropped, +`no-new-privileges`, and a non-root user. The container used the pinned image +digest: + +```text +debian@sha256:936abff852736f951dab72d91a1b6337cf04217b2a77a5eaadc7c0f2f1ec1758 +``` + +That run completed 7,200,000 mixed observations in 3,600.38 seconds: about +1,999.79 observations per second, within the one-percent sustain tolerance. +It did not pass the burst gate. The 600,000-observation burst required +227.73 seconds, about 2,634.66 observations per second rather than the required +10,000. The container never reached its memory boundary or an OOM condition; +its observed memory remained below 2 GiB. Aggregate block-I/O counters reached +approximately 57.1 GB while retained test data was approximately 8.1 GB, +identifying synchronous durable-write amplification as the next measured +engineering boundary rather than CPU or memory exhaustion. + +Once the burst miss was conclusive, the disposable run was stopped during its +ten-million-observation fill. Its dataset was removed by the fixture's normal +signal cleanup; timestamped stage logs and container exit evidence were +retained. Queries, outage replay, retention, and the final aggregate report +from that run are therefore not release evidence. Observatory will not lower +or relabel the published gate: durable-write grouping and projection work must +be measured, reviewed, and followed by a complete exact-candidate rerun. + +## August 18 query-boundary observation + +A later constrained candidate reached the complete ten-million-observation +primary corpus but stopped at the first required query. The indexed +`status >= 500` path still scanned and grouped the matching raw log rows, and +the ten-second execution budget expired before it produced the route/window +summary. No aggregate report was emitted, so that run is failure evidence—not +a capacity pass. + +The response is an additive, exact five-minute status/route projection. It is +updated in the same SQLite transaction as the primary projection, uses a +per-segment ledger for replay idempotence, backfills older projections once, +and expires with the hot-log projection. A non-aligned lower window boundary +still scans its exact raw fragment, bounded to less than five minutes. Queries +that need cold evidence or do not match the narrow typed shape stay on the raw +path. + +Before another full release campaign, the implementation completed the entire +development fixture and a separate one-million-observation diagnostic inside +the release cgroup boundary. The diagnostic used four CPUs and 8 GiB of memory; +the error summary completed five times with a 12.94 ms maximum sample, the +ordinary indexed item view with a 4.59 ms maximum, and the metric rollup with a +3.73 ms maximum. Maximum RSS was 205,545,472 bytes and the dataset was +1,005,353,808 bytes. Those measurements are a scale diagnostic only. They do +not replace the required one-hour, ten-million-observation exact-candidate +campaign or its three-second p95 gate. + +## August 18 asynchronous-projection diagnostic + +After durable acknowledgement was separated from grouped background +projection, a short local Linux/WSL2 development run exercised the same mixed +fixture with a two-second sustain, two-second burst, 100,000-observation query +corpus, four organizations, and a 500-millisecond visibility target. It was not +run inside the release cgroup and is not release evidence. + +The sustain reached about 2,530 observations per second with 31.92 ms p95 +ingest time and 201 ms p95 visibility. The burst reached about 10,787 +observations per second with 29.83 ms p95 ingest time and 404 ms p95 +visibility. The three required query shapes completed with p95 samples of +2.264 ms, 1.992 ms, and 1.512 ms. Maximum RSS was approximately 208 MB and the +dataset approximately 164 MB. + +This diagnostic is evidence that acknowledgement-path projection writes were +the measured burst bottleneck and that the revised shape is worth the full +campaign. It does not close the four-CPU/eight-GiB gate, the one-hour sustain, +the 60-second burst, the ten-million-observation corpus, outage replay, or the +exact packaged-candidate requirement. + +## August 18 exact-candidate boundary + +The exact asynchronous-projection candidate was then run in the release +four-CPU/eight-GiB container. It completed the one-hour sustain, burst, and +ten-million-observation fill without an out-of-memory condition. After a +ten-minute projection drain, 1,118 committed raw segments (922,788,512 bytes) +remained pending and the oldest projection lag was 12 minutes 22 seconds. The +campaign therefore failed closed before the query, outage-replay, retention, +and terminal aggregate stages. + +This is useful negative evidence: durable acceptance and bounded memory held, +but projection throughput at that synthetic scale is not yet a supported +claim. The result does not block lower-volume public dogfooding. It does block +calling the release-scale capacity gate complete, and it is the reason that +extended capacity remains a milestone campaign rather than the default +preview loop. + +The retained baseline comparisons, rejected multi-row and durability +experiments, and ordered lower-level case studies are recorded in +[PERFORMANCE.md](PERFORMANCE.md). Rejected experiments must not be repeated +without a materially changed premise and a new same-host baseline. diff --git a/docs/DASHBOARDS.md b/docs/DASHBOARDS.md new file mode 100644 index 0000000..4783394 --- /dev/null +++ b/docs/DASHBOARDS.md @@ -0,0 +1,72 @@ + + +# Saved queries and dashboards + +Observatory persists query definitions before building the web interface on +top of them. A saved query contains bounded human metadata, one organization, +an optional project/environment/service scope, the reviewed query text, and +the exact versioned typed AST produced from that text. Reads reparse the text, +hydrate its duration fields, validate the stored AST, and require the two +representations to match. + +Dashboards are versioned ordered collections of at most 16 panels. Each panel +selects one saved query and one deliberately small visualization contract: +`table`, `stat`, or `timeseries`. Composite foreign keys prevent a panel from +referencing another organization's query. Updates to saved queries and +dashboards require their current revision so concurrent edits fail instead of +silently overwriting one another. + +Identifiers are generated from cryptographic randomness; randomness failure +fails the write and never falls back to time or a predictable counter. + +## Source-control export + +`ExportDashboard` produces strict version-1 JSON containing the dashboard +definition and each referenced query once. It keeps the stable definition IDs +needed by panels, but omits organization IDs, operator user IDs, timestamps, +internal revisions, and redundant stored AST bytes. + +`observatory export` performs the same organization-authorized read from the +local server data directory. `observatory import` accepts exactly one strict, +bounded JSON bundle on standard input and requires the destination +organization twice: once as `--organization-id` and again as the exact +`--approve-organization` value. Import reparses every query, validates each +resource scope against the destination organization, replaces all portable +identifiers with cryptographically generated local identities, and commits +the queries, dashboard, and panels in one SQLite transaction. A bundle cannot +select its tenant or actor. + +Example: + +```sh +observatory export --organization-id organization-id \ + --actor-user-id user-id --dashboard operations >operations.json + +observatory import --organization-id organization-id \ + --approve-organization organization-id --actor-user-id user-id \ + + +# Tend deployment boundary + +Observatory pins Tend `v0.2.0-preview.2` as a checksum-verified Go tool and uses +its schema-2 singleton-candidate strategy on Linux. The +release configuration is [`release/tend.json`](../release/tend.json). It pins +the public origin, live and candidate loopback addresses, health paths, +application marker, release root, state path, shared host lock, and exact build +package. It contains no secret values. + +## Two deliberately different processes + +The installed service explicitly runs: + +```text +observatory server --config %d/server.json --systemd-credential-config +``` + +Systemd copies the root-owned mode-`0600` configuration and Web Push private +key into the service's private credential directory. Systemd 255 presents each +runtime copy as a root-owned mode-`0440` credential confined to the unit; +Observatory accepts that mode only when explicit systemd-credential validation +also proves the credential-directory boundary and owner. The static +`gamertan-observatory` service user can read those copies and write only its +state directory. The originals stay root-owned mode `0600` under +`/etc/gamertan-observatory/`. + +Local operator commands that read the server configuration use the same +explicit `--systemd-credential-config` flag and confined credential-directory +policy. This applies to bootstrap, resource and enrollment management, +retention and schema review, local queries, dashboard import/export, and +migrations; no operator workflow needs a less restrictive duplicate config. + +Tend executes a candidate binary without arguments and supplies only +`OBSERVATORY_TEND_CANDIDATE_LISTEN=127.0.0.1:18093`. That narrowly named mode: + +- accepts only a numeric loopback address and nonzero port; +- exposes only `/`, `/healthz`, and `/readyz` over plain loopback HTTP; +- returns a fixed marker and security headers; +- does not read configuration, open storage, acquire a process lock, run a + migration, evaluate alerts, send Web Push, access the network, or write a + file. + +The candidate proves that the exact binary starts and answers its bounded +contract. It does not pretend to prove a live data migration. After the +candidate passes, Tend stops it, atomically selects the immutable release, +restarts the installed service, probes the real loopback application, and +probes the routed HTTPS origin. A failure restores the previous release. + +The candidate listen variable must never appear in the shared Tend environment +file. Tend's server policy rejects that mistake. The live address remains in +the root-owned Observatory configuration. + +## Installed files + +The reviewed deployment inputs are: + +- `release/observatory.service`: hardened non-root service; +- `release/server.json`: non-secret dogfood server configuration; +- `release/observatory.env.example`: intentionally empty shared environment; +- `release/Caddyfile.observatory`: initial origin handler; +- `release/tend.json`: application-owned packaging and activation contract. + +The Caddy handler also defines a bounded, rotated JSON edge log. It strips the +entire query, request and response header maps, client addresses, and Caddy +user identity before writing. A late `log_append` copies only the bounded +application response request ID into a dedicated top-level field, allowing the +agent to correlate accepted edge and application records without retaining +cookies, credentials, referrers, user agents, or query values. The log file is +not an application authorization record and remains separate evidence. + +That installed handler is the privacy-minimized deployment policy, not a +platform prohibition. Applications that disclose and require richer traffic +evidence can use per-source `sensitive_fields` with a separately reviewed Caddy +filter, such as `examples/Caddyfile.sensitive-access-log`. The richer policy +still removes both header maps and never retains cookies or authorization +headers. Its query-key deny list is only a starting point: the application must +add every secret-bearing parameter it accepts before enabling query collection. + +On the host, their controlled destinations are: + +```text +/etc/gamertan-observatory/server.json root:root 0600 +/etc/gamertan-observatory/web-push.json root:root 0600 +/etc/tend/environment/observatory.env root:root 0600 +/etc/tend/services/observatory.json root:root 0644 +/etc/systemd/system/gamertan-observatory.service +/opt/gamertan-observatory/ +/var/lib/gamertan-observatory/ +``` + +The initial Caddy stanza is an explicit infrastructure change performed before +the first Tend activation. Tend validates and uses the existing routed origin; +it does not silently edit the host's top-level Caddyfile for a new service. + +## First installation and the first Tend maintenance release + +Tend intentionally bootstraps singleton state from an existing, validated +`current` release pointer. It does not invent a service account, Caddy origin, +root-owned configuration, or first live release. Observatory's one-time +installation therefore uses a twice-built, checksum-approved Tend package from +an exact clean pushed commit, exercises that binary in the stateless candidate +mode, installs its immutable release, creates the first `current` pointer, and +starts the reviewed service and Caddy origin under direct operator control. + +The next exact pushed commit became the real Tend maintenance campaign: +candidate health and application smoke, pointer activation, service restart, +routed public-origin smoke, state recording, rollback, and identical-artifact +redeployment. The observation below separates the one-time bootstrap from that +completed dogfood exercise. + +## Initial dogfood bootstrap observation + +On August 17, 2026, the initial live Observatory bootstrap used a Tend package +built twice from one clean pushed development source. Both package runs +produced the same SHA-256 digest: + +```text +42c21deba0519955ea7cbb140ea37321403cb26622d54d69c68bcfd19fe2764e +``` + +The installed `v0.0.1-preview.3` development binary reports Go 1.26.6. Tend +validated and safely extracted the immutable archive, and the stateless +candidate contract passed before the directly controlled first activation. +The resulting non-root systemd service remained ready with zero restarts while +the canonical HTTPS origin and the pre-existing EQL Helper, Gamertan, and +Sandwich Hime origins were rechecked. + +The bootstrap found two real integration defects before activation was +accepted: + +- systemd 255 presents a copied `LoadCredential` file as root-owned mode + `0440`, although the original remains root-owned mode `0600`; Observatory + originally rejected the runtime copy. The validator now permits `0440` only + when explicit systemd-credential mode also proves that the file is inside + the unit's credential directory and owned by root; +- an omitted Web Push notifier was stored as a typed nil pointer in a non-nil + interface, so the first incident evaluation panicked. Optional push + construction now returns a genuinely nil interface when no notifier is + configured, and its regression test exercises an incident with Web Push + disabled. + +Neither failure altered an existing application origin. Failed Observatory +artifacts were retained as diagnostic evidence, and the service was accepted +only after the corrected package passed its checks. Tend state remained +intentionally uninitialized until the first maintenance activation. + +## First Tend maintenance and rollback observation + +Trusted CI run 220 passed the exact pushed maintenance source. Tend built it +twice with Go 1.26.6 and module downloads disabled; both archives were +byte-identical at: + +```text +3afed7dcc719d98d9bc9a4e13790ce0a71036c339f1a354fa4e0d67abab3ccdb +``` + +On August 17, 2026, `tend push` sent that approved digest through the dedicated +forced-command deployment account. The host policy accepted only the +`observatory` service and validated the archive before Tend exercised the +stateless candidate, activated `v0.0.1-preview.4`, restarted the installed +unit, checked loopback health and readiness, probed the routed HTTPS origin, +and initialized deployment state with `v0.0.1-preview.3` as the previous +release. + +The recorded Tend rollback then restored the exact preview.3 release and +swapped preview.4 into the previous position. Loopback health, the public +Observatory origin, and EQL Helper continuity passed before the same approved +preview.4 archive was sent through the restricted transport again. Tend +revalidated and reactivated its existing content-addressed release. Final +state records preview.4 active and preview.3 previous. + +The candidate port and shared host lock were free after the exercise. The +service reported zero restarts and no warning-or-higher journal entries, and +the Observatory, EQL Helper, Gamertan, and Sandwich Hime origins all returned +success. This closes Observatory's initial Tend maintenance-and-rollback +dogfood claim; it does not close the separate medium-fleet, capacity, or public +preview release gates. + +## Startup-readiness dogfood finding + +On August 18, 2026, Tend correctly refused Observatory +`v0.0.1-preview.13` after the activated process did not bind its loopback +socket inside the 30-second post-activation window. Tend restored the exact +preview.12 release, preserved the earlier rollback release, recorded the failed +digest, and returned the service to active state with zero restarts. + +The package and stateless candidate had passed. The full server then performed +raw retention and identity-evidence pruning synchronously before opening its +listener. Those maintenance passes are bounded and valid, but their elapsed +time depends on accumulated data and cold filesystem state; they are not a safe +availability prerequisite. Observatory now completes mandatory raw recovery, +opens the configured listener, and only then runs retention and evidence +pruning through the same background path used for hourly maintenance. A +maintenance failure remains visible in bounded logs but does not prevent the +otherwise healthy server from answering readiness probes. + +This also records a Tend integration limitation: a stateless candidate proves +the packaged binary and candidate process boundary, not a stateful +application's complete startup path. Tend should continue to restore on a +missed activation window, but a future release should report this distinction +more explicitly and reject linked-worktree packaging before the build rather +than after Go omits `vcs.revision`. The trusted Gitea release workflow produced +and verified identical preview.13 bytes, but the Gitea Actions artifact API did +not list the successfully uploaded artifact; the independently reproduced +local digest and CI log therefore remained the review evidence for this +attempt. + +Preview.14 exposed a second, independent startup boundary on the accumulated +production archive. Mandatory recovery enumerated every hot raw segment by +checksum-verifying, decompressing, and retaining every decoded batch before it +compared the archive with the control catalog. With 20,550 small segments this +exceeded the service's 768 MiB memory limit before the listener opened. Tend +restored the recorded release, but the same archive-bound algorithm also +prevented that older binary from becoming ready; Tend therefore correctly +left its bounded stateless candidate routed for operator recovery instead of +claiming a successful rollback. + +Recovery now walks only private filesystem metadata and compares each object +with a prepared catalog lookup. It decodes one segment only when the catalog +is missing that committed object, and it replays unprojected catalog rows in +pages of 128. This fixes the availability defect rather than hiding it behind +a larger memory allocation. Full checksum verification remains part of +orphan admission, reads, cold archival, deletion, export, and explicit +projection rebuild. A deployment with production-scale retained evidence must +exercise the stateful startup path under its configured memory limit before +activation; the stateless package candidate remains a separate gate. + +## Preview.19 native-batch and stateful-Compose observation + +Trusted Gitea release-candidate run 380 passed exact source commit +`2b00be0984188b0c59d6a843921e03c9b182b881` with Go 1.26.6. Two independent +Tend package runs and the retained CI artifact produced the same archive: + +```text +6ce16931412f618fa4a3d0130fc0f69e7f96f994b1f12e6be3a66a0e49a9af95 +``` + +The public-preview capacity gate passed at 221 observations/second sustained, +1,315/second burst, approximately 60 ms visibility p95, approximately 2.1 ms +query p95, and approximately 173 MiB maximum RSS. These are bounded synthetic +gate results, not a medium-fleet or generic production-capacity claim. + +Before activation, the exact candidate was exercised against a copy of the +accumulated production control database, organization projection, and +immutable raw archive. The first migration failed closed while building the +new presence-only projection index: SQLite reported `database or disk is full` +with the service's 64 MiB `/tmp` tmpfs. The same image, data, limits, and +migration passed with a 512 MiB tmpfs, drained 986 pending raw segments to +zero, and finished with 27,788 retained segments. The database was not corrupt; +the index sort needed a larger bounded temporary working area. The reviewed +Compose definition now records that scratch budget explicitly. + +Observatory's production topology is a Docker Compose singleton, which Tend +does not yet model. The maintainer therefore paused the durable agent, verified +mode-`0600` SQLite backups and their SHA-256 digests, migrated the live data +offline, and started the exact candidate under the staged Compose definition. +Direct health and readiness passed approximately 3.25 seconds after the final +start; the routed public origin settled approximately one second later. + +The first activation script probed the public origin only once. That eager +probe observed a transient `503` and invoked its rollback path even though the +candidate had passed direct readiness. The rollback then demonstrated an +important second boundary: Preview.18 correctly rejected the newly migrated +control schema 11 and entered a fail-closed restart loop. The verified backups +remained intact, but restoring them would have discarded valid forward +migration work. The operator instead stopped the old agent, started the +schema-compatible Preview.19 candidate, waited through bounded direct and +public-origin readiness loops, and then started the matching Preview.19 agent. +An independent observer recorded a bounded `502`/`503` interruption during +the failed rollback and recovery. No continuous-delivery claim is made. + +At the immediate post-activation check on August 19, 2026: + +- the server and agent both reported the exact Preview.19 image, zero restarts, + no OOM event, and no warning-or-higher log entry; +- `/`, `/healthz`, and `/readyz` returned success through the public origin, + while `/app/` returned its expected authenticated redirect; +- `observatory check` returned `ok` against the live configuration and data; +- all 27,998 retained segments were projected with zero pending work; and +- the first stream advanced by the new agent had non-empty native-v2 batch and + encoded-body digests, proving the authenticated batch-identity path in the + live control catalogue. + +This is staging dogfood evidence for an unreleased development build. It does +not create a public tag, close the medium-fleet soak, or make the current +Compose handoff a supported Tend strategy. It does prove that authenticated +`(source, stream, sequence)` batch identity, exact digest replay protection, +raw-first durability, and overlapping-time-window semantics work together on +the live data plane without a global per-record deduplication index. + +## Secrets and administration + +No secret belongs in `tend.json`, the release manifest, process arguments, or +the shared environment file. Web Push keys and later credentials use systemd +credentials. Local operator bootstrap creates a one-time password in a new +private file and is performed while the application is stopped or through a +separately reviewed administrative procedure; it is not part of deployment +activation. The first browser or API session can reach only rotation, logout, +health, and immutable shell assets until the password is replaced. + +The first dogfood deployment is an unreleased development build. A public tag +requires the capacity campaign, rollback exercise, medium-fleet soak, reviewed +public snapshot, checksums, SBOM, signatures, and release gates in the roadmap. diff --git a/docs/INCIDENTS.md b/docs/INCIDENTS.md new file mode 100644 index 0000000..0bf9536 --- /dev/null +++ b/docs/INCIDENTS.md @@ -0,0 +1,40 @@ + + +# Incidents and alert rules + +Observatory alert rules reuse a stored, versioned query. They do not introduce +a second expression language, arbitrary SQL, shell execution, or a remote +plugin surface. + +Each rule defines: + +- one organization-owned saved query; +- a bounded minimum result-row count; +- one to ten required consecutive matching evaluations; +- a severity and a 15-second to 24-hour evaluation interval; and +- an enabled state. + +The evaluator claims at most 64 due rules per pass. Every query retains the +server's ordinary time, row, decoded-byte, and memory budgets. Rules execute +without sensitive-field permission; a query that requires sensitive or +unknown fields fails closed as `query_unavailable`. Query failure never clears +an existing incident. + +The durable lifecycle is: + +1. The first matching evaluation opens a `pending` incident. +2. The configured consecutive-match count promotes it to `firing`. +3. An authorized responder may `acknowledge`, `silence`, or `resolve` it. +4. A non-matching successful evaluation resolves the open incident. +5. An expired silence returns to `firing` if the rule still matches. + +Every transition appends a sequenced organization-scoped event. Events record +only the transition, actor identifier, and time. They do not copy query text, +query results, resource labels, or telemetry values into the control database. +The SSE channel carries only a generic refresh hint. + +PWA installation, explicit offline inbox access, application badge state, and +generic Web Push are progressive layers over this durable lifecycle. Web Push +is optional and best effort; it cannot create, update, resolve, or otherwise +replace an incident record. Its exact privacy boundary is documented in +[`PWA.md`](PWA.md). diff --git a/docs/INTERFACE.md b/docs/INTERFACE.md new file mode 100644 index 0000000..86252a5 --- /dev/null +++ b/docs/INTERFACE.md @@ -0,0 +1,76 @@ + + +# Sandwich Hime interface + +Observatory's web interface is server-rendered with Hime-san +`v1.0.0-beta.2`. Generated `.sando.go` files are committed and verified for +two-pass byte and modification-time stability. The production binary links +only `gamertan.com/sandwich-hime/sando@v1.0.0-beta.1`; the compiler does not +enter the service dependency graph. + +The current surface provides: + +- a public project explanation; +- local session sign-in and CSRF-protected sign-out; +- selection among the authenticated user's active organizations; +- bounded one-hour tables for logs, metrics, traces, and deployment evidence; +- organization-authorized creation of persisted queries through typed text or + the first assisted visual builder; +- one-panel dashboards, bounded panel execution, nonnegative numeric meter + summaries with complete table alternatives, and strict JSON export; +- a progressive SSE notification when newer observations are available; +- an organization-scoped incident inbox with durable lifecycle state and + ordinary acknowledge, silence, and resolve forms; and +- bounded alert-rule creation over an existing saved query; +- an installable manifest, deterministic service worker, and offline public + shell; and +- explicit opt-in storage of a read-only incident snapshot with badge state. + +Every organization selection is checked against both active membership and +the scoped `dashboards.read` grant. The query executor receives the authorized +organization separately from query text. No browser field can select a tenant +without that server-side decision. + +The SSE stream is deliberately an invalidation hint rather than a telemetry +transport. It emits fixed `ready` and `refresh` events with `{}` payloads, +coalesces pending refreshes, caps total and per-organization subscribers, and +is authorized independently when the connection opens. If JavaScript is +disabled, disconnected, or blocked, the same HTML tables and ordinary refresh +links remain usable. + +CSS and JavaScript are embedded, content-addressed, and served with immutable +cache policy. The application CSP permits only same-origin styles, scripts, +images, forms, and SSE connections; it does not use `unsafe-inline`. + +The assisted builder deliberately produces one optional comparison and one +optional aggregate/group/bucket stage from fixed controls. It serializes that +selection into ordinary query text and submits it to the same parser used by +the text editor; it does not maintain a second query language. More complex +queries remain available through typed text. + +Time-series panels render at most 48 finite, nonnegative points with native +`meter` elements. The complete result table is always rendered beside the +summary and remains the canonical accessible representation. Negative, +non-numeric, missing, or unsupported results simply omit the visual summary. + +Alert evaluation counts bounded saved-query rows, requires a configured number +of consecutive matches before promotion from pending to firing, and never +resolves an existing incident when query execution itself fails. The incident +event trail records system transitions and authorized human responses without +copying query results or telemetry values into control state. + +The service worker precaches only public content-addressed shell assets. It +stores no private response by default. An authorized user must activate the +offline control before the worker fetches a separate read-only incident page +and stores it under that organization's ordinary inbox URL. The snapshot has +no actions, CSRF material, query text, telemetry values, actors, or project, +environment, and service identifiers. Signing out asks the worker to delete +the complete private cache. Session expiry cannot itself reach an offline +browser, so this remains an explicit device-local privacy decision. + +Dashboard metadata and panel editing use optimistic revisions, and the bounded +assisted query builder shares the same parser as the complete typed editor. +The incident inbox provides an installable shell, explicit read-only offline +copy, badge state, and optional generic Web Push without making JavaScript an +authority for incident response. Richer visual editing remains future work, +not an alternate query authority or a hidden first-preview gate. diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md new file mode 100644 index 0000000..58f091b --- /dev/null +++ b/docs/PERFORMANCE.md @@ -0,0 +1,232 @@ + + +# Performance engineering ledger + +This document preserves measured optimization work, including changes that +were deliberately rejected. It prevents an attractive idea from being +reimplemented and rerun without a materially different premise. + +The capacity fixture is a synthetic engineering instrument. Its local results +are useful for comparing two exact implementations on the same host; they are +not production sizing claims and do not replace the constrained release +campaign in [CAPACITY.md](CAPACITY.md). + +## Current data path + +Observatory deliberately separates four jobs: + +1. the agent batches typed observations; +2. the server validates and durably acknowledges an immutable, checksummed, + zstd-compressed raw segment; +3. a background projector builds disposable per-organization SQLite read + models, typed indexes, and rollups; and +4. queries read authorized projections or budgeted cold evidence. + +Raw segments are the forensic truth. SQLite is currently a replaceable query +accelerator, not the only copy of an observation. A bounded in-memory layer may +accelerate the recent live window, but it must populate only after raw +acknowledgement and must remain safe to lose and rebuild. + +## Experiment ledger + +All August 18 local comparisons used the same Linux/WSL2 checkout based on +private commit `7c4f834aee8afcb2e9c3ae534b1b37b03259c9a3`, Go 1.26.6, the same synthetic +mixed workload, and an unconstrained 16-logical-CPU host. They are comparative +diagnostics, not release evidence. + +| Experiment | Corpus | Sustain / burst | Visibility p95 | Fill or query notes | Result | +| --- | ---: | ---: | ---: | --- | --- | +| Existing prepared single-row projection writes | 100,000 | 2,630.64/s / 13,084.50/s | 84.7 ms / 228.1 ms | about 198 MB peak RSS; about 176.9 MB dataset | Baseline retained. | +| 256-row multi-value SQLite inserts | 100,000 | 2,631.28/s / 13,015.34/s | 200.7 ms / 382.3 ms | about 197 MB peak RSS; about 205.6 MB dataset | Rejected and removed: no throughput gain, worse visibility and storage. | +| `synchronous=NORMAL` projection plus transaction receipts and recovery reconciliation | 100,000 | 2,630.13/s / 12,991.85/s | 175.35 ms / 230.76 ms | 199,081,984-byte peak RSS; 177,218,281-byte dataset | Rejected and removed: added recovery machinery without a measurable throughput gain. | +| Existing prepared single-row projection writes | 1,000,000 | 3,815.33/s / 18,901.41/s | 121.83 ms / 257.26 ms | fill 6.604 s; queries 6.226/1.839/1.557 ms; 1,090,726,860-byte dataset | Larger baseline retained. | +| `synchronous=NORMAL` projection plus receipts | 1,000,000 | 3,823.49/s / 18,766.63/s | 173.67 ms / 219.41 ms | fill 6.789 s; queries 6.637/1.721/1.586 ms; 1,092,327,298-byte dataset | Rejected and removed: within noise on throughput, slower fill, larger dataset. | + +Do not repeat either rejected experiment unless the transaction shape, schema, +SQLite version, storage medium, or workload has materially changed. Record the +new premise and a same-host baseline before doing so. + +## Native v2 exact-replay envelope + +The native v2 endpoint places bounded batch identity metadata in authenticated +request headers before the JSON record body. A retry of the current +acknowledged `(source, stream, sequence)` can therefore hash and discard the +complete bounded body, recheck the watermark under the source lock, and return +the retained acknowledgement without JSON decoding, zstd compression, +catalogue writes, or projection notification. New batches still follow the +complete validation and raw-first transaction. + +An August 18, 2026 Linux/amd64 Go 1.26.6 HTTP-handler microbenchmark used an +exact 500-record, approximately 60.6-KB JSON retry on an Intel i7-10700K. Five +independent benchmark samples recorded: + +| Path | Median time | Bytes allocated/op | Allocations/op | +| --- | ---: | ---: | ---: | +| Legacy native v1 exact retry | 4.105 ms | about 22.4 MB | about 14,760 | +| Framed native v2 exact retry | 0.275 ms | about 49.9 KB | 301 | + +For this one retry workload, v2 used about 14.9 times less wall time, 449 times +fewer allocated bytes, and 49 times fewer allocations. Reproduce with: + +```sh +go test -buildvcs=false ./internal/httpserver \ + -run '^$' -bench '^BenchmarkNativeExactReplay$' -benchmem -count=5 +``` + +This is not new-batch throughput, whole-agent delivery latency, or a production +capacity result. It proves only that the early exact-replay path removes the +intended duplicate work. The server still reads and hashes every replay byte; +transport integrity and the immutable retained segment remain authoritative. + +## First cost-centre diagnostic + +The capacity report now records projection backlog/drain time, raw/control/ +projection file bytes, and SQLite page use by schema object. The fields contain +only aggregate counts and checked-in schema names; they contain no telemetry +values, tenant identifiers, or local paths. + +An unconstrained local 100,000-primary/109,000-total observation diagnostic on +the same August 18 baseline recorded: + +- 19 pending segments and 14,936,883 decoded bytes at the start of drain; +- 4.580 seconds to drain that backlog; +- 446,217 bytes of compressed raw segments; +- 158,247,552 bytes across all organization projection SQLite files and + sidecars; +- 1,705,496 bytes of control SQLite files and sidecars; and +- 97,124,352 allocated SQLite page bytes in the primary projection, comprising + 28,409,856 table bytes, 68,702,208 index bytes, and 12,288 internal bytes. + +The synthetic observations compress unusually well, so the roughly 355:1 +projection-to-compressed-raw ratio must not be extrapolated to real telemetry. +The within-SQLite result is still actionable: indexes consumed about 2.42 +times the table bytes and about 70.7 percent of allocated primary-projection +pages. Index write amplification is therefore the first implementation case +study. This one short diagnostic is not a release capacity result. + +## Presence-only base-index candidate + +The first index case study replaced four full projection indexes with partial +indexes that contain only observations where the indexed value is present: +metric value, HTTP route, HTTP status, and request duration. Severity remains +a full index because its current storage representation uses an empty string +for absence. The migration is independently versioned and changes the old and +new index sets in one SQLite transaction; an interrupted migration therefore +retains the complete prior set and safely retries. + +Same-host comparisons against the cost-centre baseline recorded: + +| Corpus | Baseline index bytes | Candidate index bytes | All projection SQLite | Projection drain | Query p95 notes | +| ---: | ---: | ---: | ---: | ---: | --- | +| 100,000 primary observations | 68,702,208 | 57,135,104 (-16.8%) | 158,247,552 to 143,429,744 (-9.4%) | 4.580 s to 4.704 s | Same millisecond range; drain difference is within short-run noise. | +| 1,000,000 primary observations | 693,755,904 | 572,768,256 (-17.4%) | 1,087,731,688 to 954,606,472 (-12.2%) | 58.267 s to 54.535 s (-6.4%) | Baseline 6.157/1.908/1.659 ms; candidate 6.156/5.444/1.680 ms. The isolated recent-items increase remains only milliseconds and needs repetition before interpretation. | + +The million-observation candidate retained the identical 283,639,808 table +bytes and reduced the primary projection from 977,408,000 to 856,420,352 +allocated bytes. Synthetic compressed raw sizes were 4,011,063 and 4,031,350 +bytes respectively; that small corpus variance is not attributed to the index +change. + +This is a promising bounded schema improvement, not proof that continuous +universal projection is the final architecture. Migration and selective-query +tests must pass from legacy preview databases, followed by the full verifier +and trusted CI, before adoption. The adaptive raw-first design remains the +larger direction. + +## What the evidence says + +SQLite is not yet proven to be the limiting technology. The failed +ten-million-observation campaign proves that the current *projection shape* +cannot drain the synthetic fill inside the published window. It does not +separate SQLite's engine cost from Observatory's choices around row shape, +index count, JSON expression indexes, typed descriptor indexing, rollups, WAL +checkpoints, or single-organization serialization. + +The two rejected experiments also show that transaction syntax and one +durability pragma are not the dominant cost. The next work must measure cost +centres instead of swapping databases or adding cache infrastructure by +intuition. + +## Required cost-centre evidence + +Add aggregate, value-free timing and byte counters to the synthetic fixture +for: + +- native batch validation, JSON encoding, zstd compression, file write, + file `fsync`, directory `fsync`, and control-catalog commit; +- raw-segment open, checksum, decompression, and decode; +- base observation insertion, built-in index maintenance, reviewed custom + index maintenance, metric rollups, log rollups, WAL commit, and checkpoint; +- projector queue depth, oldest lag, rows and bytes per transaction, active + writer time, and time waiting for an organization lock; and +- query planning, rows/bytes scanned, SQLite execution, cold decode, and result + encoding. + +These counters belong only in the capacity fixture or bounded internal +instrumentation. They must not include telemetry values, local paths, +credentials, host names, or tenant identifiers. + +Projection backlog/drain and file/page accounting above are implemented. The +per-operation CPU/wall-time breakdown remains open. + +## Ordered technical case studies + +The larger lazy/eager partition design is specified in +[ADAPTIVE_PROJECTIONS.md](ADAPTIVE_PROJECTIONS.md). The ordered experiments +below are its evidence path, not independent promises to ship every idea. + +### 1. Index write amplification + +The base observation table currently maintains its primary key plus general +scope/time/name indexes and signal-specific severity, value, correlation, HTTP +JSON-expression, trace, and span access paths. Measure per-index bytes and +projection time. Compare only reviewed alternatives such as partial +signal-specific indexes or replacing a generic expression index with an exact +rollup already used by the supported query shape. Every candidate must rerun +the existing query and cold-evidence gates. + +### 2. Time and signal partitioning + +One organization currently has one SQLite writer. A partition per bounded time +window and/or signal could let logs, metrics, and traces project concurrently, +limit index working sets, and make retention a file-level operation. The cost +is a more complex planner, bounded multi-database queries, atomic partition +catalogue changes, and additional recovery cases. Prototype this only after +the index-cost campaign identifies single-database write amplification. + +### 3. Bounded recent-memory view + +A per-organization ring keyed by segment digest can make the latest live tail +visible immediately after durable raw acknowledgement while disk projection +continues. Queries would merge the bounded memory window with disk results and +deduplicate by segment/record identity. This improves live experience and +absorbs short projection bursts; it does not increase durable projector +throughput and must never become acknowledgement truth. + +### 4. Durable admission group commit + +Each accepted native batch currently creates and synchronizes its own raw file +and directory, then commits the control catalogue. A bounded group-commit +coordinator could acknowledge several independent streams after one directory +sync and one control transaction. It must preserve per-stream ordering, +organization quota checks, exact digest acknowledgements, cancellation, and a +small maximum wait. This is relevant only if cost-centre evidence places the +limit on durable admission rather than projection. + +### 5. Projection representation + +If row/index tuning and partitioning remain insufficient, compare a compact +typed projection representation against SQLite on identical raw segments. +Candidates must retain authorization-before-open, bounded queries, atomic +replacement, offline rebuild, corruption handling, and small-server operation. +An external database is not an optimization if it merely moves the same write +amplification into more infrastructure. + +## Database decision boundary + +Keep SQLite while it satisfies the measured small-server boundary with simpler +operations. Consider another embedded or external engine only after an exact +candidate demonstrates that the required workload remains blocked by SQLite +itself after index, partition, and batching work. Any replacement must beat the +same fixture while preserving tenant isolation, raw replay, deterministic +recovery, query budgets, deployment simplicity, and idle resource use. diff --git a/docs/PUBLIC_SNAPSHOT.md b/docs/PUBLIC_SNAPSHOT.md new file mode 100644 index 0000000..aa2f8aa --- /dev/null +++ b/docs/PUBLIC_SNAPSHOT.md @@ -0,0 +1,28 @@ + + +# Public snapshot boundary + +Observatory uses a private development repository and a separate canonical +public source repository. Public history is not a mirror of private history. +Each publication is a reviewed root snapshot of one exact, clean, pushed +private `main` commit. + +`scripts/public-snapshot.allow` is the complete public file boundary. The +exporter refuses workflow directories, absolute or parent-traversing paths, +missing tracked files, dirty worktrees, and a private HEAD that differs from +`origin/main`. It archives only the allowlisted paths from the committed Git +tree, byte-compares them with the worktree, scans the result for known private +material, and records the source commit, tree, date, and file count in +`PUBLIC-SNAPSHOT.json` with a SHA-256 sidecar. + +The generated snapshot has no `.git` directory. Publication tooling creates a +new reviewed public commit; it must never push private refs, tags, workflows, +reflogs, or historical objects. The canonical public Gitea commit and GitHub +discovery commit may have different Git identities, but their exported file +trees and snapshot manifest must be byte-identical. + +Before the first preview, the public snapshot, release archive, SBOM, +checksums, signature, canonical tag, and installed module must all be traced +back to the same reviewed private source commit. A public snapshot does not by +itself make an unreleased development build supported. + diff --git a/docs/PWA.md b/docs/PWA.md new file mode 100644 index 0000000..2fc90d5 --- /dev/null +++ b/docs/PWA.md @@ -0,0 +1,90 @@ + + +# Progressive web application boundary + +Observatory's PWA support adds availability without turning browser storage +into an implicit telemetry replica. + +The deterministic service worker precaches only: + +- the public offline explanation; +- content-addressed CSS and JavaScript; and +- the content-addressed application icon. + +It does not automatically cache authenticated navigation responses. On the +incident page, an authorized user may explicitly save a separate read-only +snapshot. The worker validates both same-origin URLs, requires one matching +organization selector, fetches with the current session, verifies a successful +HTML response, and stores that response under the ordinary inbox URL. + +The saved page contains the organization name plus current incident titles, +states, severities, and times. It excludes incident identifiers, response +forms, CSRF tokens, saved-query text, telemetry values, actor identifiers, and +project, environment, and service identifiers. The page is not an authority +for response actions; reconnection is required. + +Signing out sends a private-cache deletion request before submitting the +ordinary sign-out form. An expired server session cannot notify a disconnected +browser, so operators should treat saved incident snapshots like any other +explicitly downloaded sensitive material on that device. + +The Badging API receives only the current open-incident count when supported. + +## Optional Web Push + +Web Push follows [RFC 8291](https://www.rfc-editor.org/rfc/rfc8291.html) message +encryption and [RFC 8292](https://www.rfc-editor.org/rfc/rfc8292.html) VAPID +authentication. It is disabled unless the server configuration names a VAPID +private-key file, contact subject, bounded queue, and request timeout. Create a +new private key without exposing it in command arguments or output: + +```sh +observatory admin web-push generate-key \ + --output-file /etc/gamertan-observatory/web-push.json +``` + +The output is created exclusively as a mode-`0600` regular file. The installed +systemd unit copies that root-owned source into its private credential +directory as `web-push.json`, and the non-root server configuration references +only that runtime copy. Do not place the key value in configuration, process +arguments, deployment manifests, or source control: + +```json +"web_push": { + "private_key_file": "/run/credentials/gamertan-observatory.service/web-push.json", + "subject": "mailto:security@sandwichhime.com", + "queue_capacity": 64, + "request_timeout": "10s" +} +``` + +The source key remains `/etc/gamertan-observatory/web-push.json`, owned by +root with mode `0600`. `LoadCredential=` makes the service-specific runtime +copy readable without granting the static service account access to `/etc` or +weakening the original file. A differently named service must update both the +unit and the credential-directory path explicitly; Observatory does not search +for secrets. + +An authorized incident reader must press the browser control before +Observatory requests notification permission or creates a subscription. +One browser endpoint is owned by one user and may be mapped to several of that +user's organizations. Removing one organization keeps the browser subscribed +until its final mapping is removed. Authorization is checked again for every +delivery; revoked access retires only that organization's mapping. +Endpoints are accepted only as bounded HTTPS push-service URLs and delivery +uses a redirect-free client that rejects DNS results containing non-public IP +addresses. + +Only a transition into `firing` enqueues a notification. The bounded queue is +best effort and never blocks alert evaluation or incident persistence. The +encrypted payload is always exactly: + +> Gamertan Observatory needs your attention. + +The service worker ignores incoming payload content and renders that same +fixed sentence. It includes no organization, host, service, severity, rule, +incident identifier, count, or telemetry text. Activating it opens `/app/`, +where the user must have a valid authenticated session before seeing details. +Browser-vendor push relays remain an explicit metadata boundary: they can +observe delivery timing, endpoint identity, and the fixed payload size even +though they cannot read its encrypted content. diff --git a/docs/QUERY.md b/docs/QUERY.md new file mode 100644 index 0000000..0b87f60 --- /dev/null +++ b/docs/QUERY.md @@ -0,0 +1,130 @@ + + +# Query execution + +Observatory's text editor and assisted visual builder produce the same +versioned typed AST. The builder serializes its bounded controls to reviewable +query text and submits that text to the ordinary parser; it does not create a +parallel query language or bypass validation. Query text never selects or +overrides an organization: the HTTP server and local CLI authorize the +requested resource scope separately before planning or opening its projection. + +```text +logs +| where service == "eql" +| where status >= 500 +``` + +Use one `where` stage per comparison in the current preview: + +```text +logs +| where service == "eql" +| where status >= 500 +| window 24h +| summarize count(), p95(duration) by route, window(5m) +| sort count desc +| limit 50 +``` + +`window 24h` limits the lookback range. `window(5m)` creates five-minute +summary buckets; these are separate AST fields. Filters are typed from reviewed +field descriptors. Regular expressions use Go's bounded RE2 implementation, +not a backtracking engine. Arbitrary SQL is never accepted. + +The first assisted builder covers one optional comparison, one lookback, one +optional aggregate, one optional grouping field, one optional time bucket, and +a fixed row limit. Values are quoted before serialization and a literal +pipeline character is escaped before parsing, so form values cannot introduce +new stages. Multi-filter and multi-aggregate work remains in the text editor +until a richer builder can preserve the same explicit AST contract. + +Reviewed custom fields use the active per-organization descriptor registry. +An exact or range filter on an activated indexed field is correlated through +that version's typed index instead of casting arbitrary raw JSON. Values that +did not satisfy the reviewed type during the index build remain in raw truth +but do not silently become zero or another valid indexed value. + +Unknown fields remain raw-queryable only to principals with the separate +sensitive-telemetry permission. Results contain fixed safe identity columns +plus fields explicitly referenced by the query; sensitive bodies are never a +default result column. + +Every execution has independent duration, row, projected-byte, and memory +limits. `explain` reports the authorized projection, descriptors, index policy, +conservative scan estimate, cache eligibility, permissions, and budgets before +execution. + +## Storage scale is not query cost + +Observatory deliberately separates admission of evidence from execution of a +query. A large organization store, historical import, or forensic archive is +not itself a reason to reject new data or a small recent query. Ingestion and +import use their own explicit storage quotas, bounded batches, backpressure, +durable progress, and capacity checks. Query limits never act as an implicit +organization-size quota. + +Planning considers the authorized signal, resource scope, time window, +available indexes or rollups, and result limit. A projection file's total size +alone must not make an indexed, bounded recent-record query unavailable. The +executor still enforces actual duration, rows, logical bytes read, and memory; +summaries, alternate sorts, regular-expression filters, and cold forensic +reads retain conservative preflight because they may need to examine more of +the selected evidence. `explain` should make that distinction visible rather +than promising that a cheap result follows from a small output alone. + +A log summary with one canonical `status >= N` threshold, `count()`, a route +group, and either no bucket or a multiple-of-five-minute bucket uses an exact +five-minute projection. Its explain source ends in +`/rollup:http-status-route:5m`. The server still injects the organization and +optional project, environment, and service scope; the same duration, scan, +memory, and result limits apply. Missing routes remain distinct from explicitly +empty routes, and malformed stored statuses are excluded rather than cast to +zero. A lower time boundary inside a bucket reads only that raw partial bucket +and merges it with the complete projected buckets. Queries outside that exact +shape remain on the general executor instead of receiving a different +interpretation merely to reach a faster plan. + +Metric summaries with no per-sample value filter and a bucket of at least five +minutes can use the retained aggregate projection. Its explain source ends in +`/rollup:5m`. Counts, sums, minima, maxima, and averages remain exact; +percentiles use the bounded rollup histogram and set +`statistics.approximate=true`. Unknown, sensitive, high-cardinality, and +raw-only fields make the query use raw samples instead of silently reading an +incomplete rollup. + +Hot projection expiry does not make older evidence disappear. A query whose +lookback overlaps cold segments includes their catalogued uncompressed size in +the explain estimate, verifies and decompresses each exact zstd object, and +applies the same organization scope, sensitive-field permission, typed +filters, and execution budgets. This path deliberately accepts more latency +for forensic detail. If matching cold metric segments exist, Observatory uses +the exact raw path rather than combining them with an incomplete rollup. +The explain source adds `/cold:raw` when that tier participates. + +## Local authorized query + +The local command reads query text from standard input so values do not need to +appear in process arguments. Local operating-system access does not grant +telemetry access: `--actor-user-id` must still hold the scoped query grant. + +```sh +printf '%s\n' 'logs | where status >= 500 | window 1h | limit 50' | + observatory query \ + --actor-user-id USER_ID \ + --organization-id ORGANIZATION_ID +``` + +Use the optional project, environment, and service flags to narrow the +authorized scope. Output is a stable versioned JSON table: column descriptors +are ordered once and each row contains positional nullable string values in the +column's declared type and unit. + +## HTTP endpoints + +- `POST /api/v1/query/parse` validates text or builder AST input. +- `POST /api/v1/query/explain` requires a scoped query grant. +- `POST /api/v1/query` executes the same authorized plan. + +Session-backed endpoints require the canonical same origin. Error responses do +not echo query values or telemetry. diff --git a/docs/RECOVERY.md b/docs/RECOVERY.md new file mode 100644 index 0000000..df45212 --- /dev/null +++ b/docs/RECOVERY.md @@ -0,0 +1,110 @@ + + +# Recovery and projection rebuild + +Observatory treats immutable, checksummed raw segments as the replayable +telemetry truth. Per-organization SQLite projections, custom indexes, query +caches, and rollups are disposable products of that truth. + +## Ordinary startup recovery + +All three commands reconcile raw segments that reached durable storage before +their control record. `observatory server` then opens its listener and drains +catalogued but unprojected segments through the bounded background projector. +`observatory check` and `observatory migrate` wait for that projection work as +part of their offline verification. This closes both crash windows without +making source acknowledgement or server readiness depend on disposable query +state. Already projected segments are not rewritten. + +Discovery walks only private raw-object metadata: organization, source, +stream, sequence, path, encoded size, and the content digest carried in the +immutable filename. Observatory compares already catalogued objects with the +control database without reading their telemetry. Only an object missing from +the catalog is checksum-verified, decompressed, decoded, validated, and +admitted. Pending projections are replayed in bounded, organization-local +groups. This keeps recovery memory proportional to one fixed directory batch, +one bounded segment group, and one small projection page rather than to the +entire retained archive. + +That bounded startup path is not a substitute for forensic verification. +Checksums are still verified when an orphan is admitted, a segment is queried, +moved to cold storage, deleted, exported, or used for an explicit projection +rebuild. Operators should schedule those evidence checks and backups rather +than forcing every accepted raw object through memory before the listener can +open. + +When an explicit policy enables cold deletion, startup recovery also completes +a segment retirement that was interrupted +after its durable control-state transition. It removes any remaining raw +projection rows, verifies and removes the exact checksummed segment, and only +then clears its control record. A missing retired file is an idempotent state; +a changed file or path is a hard failure. + +Cold archival has an earlier independent recovery boundary. Observatory first +records the exact cold path, then atomically renames the verified zstd object, +then advances its catalog tier. Startup safely completes either a pending move +or a move that reached disk before the catalog update. It will not follow a +symlink, accept another path, or rebuild altered bytes. + +The shipped policy never starts the retirement transition. Cold raw evidence +therefore remains available until an organization or server operator has +deliberately enabled `delete_cold_raw`; projection expiry and compaction do not +silently imply evidence deletion. + +To run retention and SQLite compaction explicitly while the server is stopped: + +```sh +observatory migrate \ + --config /etc/gamertan-observatory/server.json \ + --apply-retention +``` + +The long-running server and ordinary local commands hold a shared lock on the +data directory. Offline migration and projection replacement require an +exclusive lock, so they fail closed while any participating Observatory +process is using that directory. + +## Rebuild one organization + +Stop the Observatory server, retain a filesystem-level backup, and run: + +```sh +observatory migrate \ + --config /etc/gamertan-observatory/server.json \ + --rebuild-organization organization-id \ + --approve-rebuild-organization organization-id +``` + +The two organization values must match exactly. The command: + +1. acquires exclusive ownership of the data directory; +2. refuses unknown organizations, symlinked projections, and unsafe SQLite + sidecars; +3. reads every registered **hot** raw segment for the organization through the + checksum-verifying segment store; cold evidence remains outside the fast + projection and available through the budgeted cold query path; +4. rebuilds the base projection and the organization's activated descriptor + version beside the live projection; +5. finalizes the replacement as a private standalone SQLite file; and +6. atomically replaces only that organization's projection and synchronizes + its directory. + +Validation or reconstruction failures before activation remove the temporary +files and preserve the existing projection. Other organizations are not +opened or rewritten. The JSON report contains the organization, raw segment +and observation counts, active projection version, and indexed-row count; it +contains no telemetry values. + +## Evidence boundary + +The rebuild proves that the currently registered checksummed hot segments can +recreate the fast disposable projection. Cold segments remain independently +checksummed forensic truth and are not silently promoted back into hot SQLite. +Neither mechanism replaces backups of the control database, raw/cold segments, +server configuration, identity state, or encryption keys. A missing or corrupt +segment is a hard failure, not a reason to silently accept partial history. + +No public Observatory preview exists yet, so there is not yet a supported +cross-preview migration promise. Every future public schema must add an +explicit migration fixture and rebuild campaign before its release can be +called compatible. diff --git a/docs/RETENTION.md b/docs/RETENTION.md new file mode 100644 index 0000000..3e3276d --- /dev/null +++ b/docs/RETENTION.md @@ -0,0 +1,148 @@ + + +# Retention and metric rollups + +Observatory enforces the server policy in `server.json` after startup recovery +and once per hour: + +- hot log projections: 30 days; +- hot trace projections: 30 days; +- hot raw metric projections: 14 days; +- immutable cold raw segments: preserved indefinitely by default; +- five-minute metric rollups: 400 days; and +- deployment events, resolved incidents, and security audit evidence: 400 days. + +All values are explicit configuration. The list above is the shipped example, +not a hidden fallback. `cold_raw_days` becomes a final raw-evidence cutoff only +when `delete_cold_raw` is explicitly enabled. It is not an additional window +after the hot cutoff. + +## Hot and cold evidence + +Raw batches are already independently Zstandard-compressed and addressed by +their SHA-256 digest. Observatory does not wrap them in a gzip tarball: that +would make selective verification and opening slower while usually saving +little space over already-compressed input. When the newest observation in a +segment leaves its hot window, Observatory atomically moves the exact object +from `raw/` to `cold/` and advances its catalog tier. No telemetry is decoded, +rewritten, or recompressed during that move. + +Cold queries are intentionally allowed to take longer. Their explain estimate +includes each selected segment's catalogued uncompressed size. Execution +validates the catalog path and time range, verifies the compressed checksum, +decodes a bounded segment, and applies the same tenant scope, sensitive-field +authorization, typed filters, scan limit, memory limit, row limit, and timeout +as a hot query. Matching cold metric segments force an exact raw query instead +of mixing incomplete history with aggregate rollups. +The explain plan adds `/cold:raw` whenever this tier participates. + +The default therefore preserves forensic raw detail without an automatic +destruction date. Projections and rollups still expire on schedule, so ordinary +queries stay compact while a deliberate cold query can recover exact older +evidence at an accepted additional cost. Full-disk encryption, protected +backups, quota planning, and filesystem capacity remain operator +responsibilities in the first preview. + +Forensic retention applies only **after** Observatory's collection boundary. +It does not mean collect everything: request and response bodies remain off by +default, known credential and header deny rules run before spooling, source +adapters use allowlists, and unknown fields remain sensitive and unindexed. +An operator must still choose lawful inputs and retention appropriate to the +people and systems represented by the evidence. + +Observatory does not currently combine cold objects into `tar.gz` archives. +Each raw batch is already zstd-compressed and content-addressed, so another +compression layer would usually save little while making selective reads and +independent checksum verification harder. A future high-inode-volume backend +may pack many unchanged zstd objects into an immutable checksummed container +with a separate bounded index. Such a pack must be completely written, +verified, and catalogued before its source objects can be retired; packing must +never change their logical digests or tenant boundaries. + +## What a rollup retains + +Every metric ingest updates its five-minute aggregate in the same SQLite +transaction as the raw projection. Each aggregate retains exact count, sum, +minimum, maximum, and the last sample value/timestamp plus a bounded deterministic +histogram. `p50`, `p95`, and `p99` over rollups are approximations and the +query result sets `statistics.approximate` to `true`. Count, sum, minimum, +maximum, and average remain exact for the retained samples. + +Longer retention never silently widens the data boundary. A dimension enters +a rollup only when its descriptor is: + +- reviewed or an Observatory built-in; +- marked with the `metric` retention class; +- public or internal, never sensitive; and +- low or medium cardinality. + +Unknown, sensitive, and high-cardinality attributes remain raw-only. The +explain plan names `/rollup:5m` whenever the aggregate projection is selected. +Queries that filter individual values, request sub-five-minute buckets, or +need fields unavailable in the aggregate continue to use raw samples. + +Metric values are bounded to an absolute value of `1e25`. This is far beyond +ordinary duration, byte, counter, and host measurements while keeping sums +and the fixed histogram domain finite under hostile input. + +## Organization policy + +An organization owner can shorten a policy: + +```sh +sudo observatory admin retention set \ + --config /etc/gamertan-observatory/server.json \ + --actor-user-id USER_ID \ + --organization-id ORGANIZATION_ID \ + --raw-logs-days 14 \ + --raw-traces-days 14 \ + --raw-metrics-days 7 \ + --cold-raw-days 180 \ + --delete-cold-raw \ + --metric-rollups-days 180 \ + --evidence-days 180 +``` + +`cold_raw_days` cannot be shorter than any hot raw/evidence window. Omit +`--delete-cold-raw` to preserve cold evidence indefinitely. Enabling deletion +is an explicit shortening policy. Extending any server default—including +turning off deletion when the server default enables it—additionally requires +the exact organization ID and a positive byte quota larger than the +organization's current stored data: + +```sh + --approve-extension ORGANIZATION_ID \ + --quota-bytes 10737418240 +``` + +The policy change and actor are recorded without telemetry values. A quota is +checked before accepting another raw segment, using current raw/projection +storage plus a conservative allowance for the new projection. Secret values +never enter the policy, audit summary, command output, or process arguments. + +## Deletion and recovery boundary + +Projected observations expire at their individual hot timestamps. Raw +segments are immutable, so a segment becomes cold only when its newest +observation crosses that signal's hot cutoff. A batch spanning multiple +timestamps can therefore keep its older members hot until the newest member +expires; agents should keep batches time-local. The pending archive path is +recorded first, the exact segment is verified and atomically moved, both +directory entries are synchronized, and the tier is advanced. Startup recovery +safely completes an interruption before or after the rename. + +Only when `delete_cold_raw` is true does the final cold cutoff separately mark +the segment retiring. Its +remaining projection and per-segment rollup ledger rows are removed, its +checksum/path are revalidated, its file and directory entry are synchronized, +and finally its control record is deleted. A missing file is accepted only for +the recorded interrupted-retirement state; changed bytes or another path stop +recovery. + +Metric rollups are disposable projections. Their expiration and SQLite +compaction do not alter hot or cold raw evidence. Projection rebuilds use only +hot segments so they do not turn the fast database back into an archive; the +query engine opens cold evidence directly when requested. Once the final cold +window expires under that explicit policy, rebuilding intentionally cannot +recreate those deleted samples. Backups must follow the same published +retention and deletion policy. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000..f6e39a8 --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,47 @@ + + +# Preview roadmap + +`v0.1.0-preview.1` remains blocked until all of these work together: + +- [x] Web Foundations organizations and scoped access are published and + integrated for generated one-time bootstrap, forced password rotation, + session revocation, and bounded query explanation. +- [x] Complete tail-cursor recovery and enrollment around the implemented + native collector adapters and durable 72-hour/5-GiB spool. +- [x] Caddy, requestlog, and Tend event file adapters plus authenticated, + bounded OTLP/HTTP protobuf ingestion for logs, metrics, and traces. +- [x] Linux host, cgroup v2, filesystem, network, and explicitly selected + process metrics in the durable unprivileged agent pipeline. +- [x] Logs, metrics, traces, and deployment events projected and queryable. +- [x] Add execution to the implemented unified builder/text AST, explain plan, + sensitive-field permission, and hard planning budgets. +- [x] Idempotent descriptive schema proposal persistence with organization- + scoped review permission and no observed values in proposal metadata. +- [x] Reviewed descriptor activation and atomic beside-current projection/index switching. +- [x] Sandwich Hime dashboards, accessible alternatives, SSE, durable alert + rules, an online incident inbox, installable shell, badge state, and an + explicit opt-in read-only offline incident snapshot. +- [x] Generic privacy-preserving Web Push with explicit browser opt-in, + fixed encrypted content, reauthorization at delivery, and a bounded + non-blocking queue. +- [x] Strict Tend activation and rollback annotations whose identity, event-file, + agent, network, or Observatory failures cannot control a release. +- [x] Enforced hot/cold raw and evidence retention, approved organization + overrides, serialized storage quotas, crash-recoverable archival and + indefinite forensic preservation by default, explicitly enabled final + retirement, budgeted forensic queries, projection compaction, and + five-minute metric rollups with privacy-safe dimensions. +- [ ] Security, corruption recovery, migration, and projection rebuild campaigns. +- [ ] 4-vCPU/8-GiB capacity campaign and medium-fleet soak. +- [ ] Sanitized public root snapshot, reproducible package, SBOM, and signed checksums. + +## Post-preview direction + +- [ ] Public-safe status dashboards backed by separately approved aggregate + projections, coarse buckets, minimum group sizes, publication delay, and + no unauthenticated path to internal or sensitive evidence. +- [ ] An additive structured-frame source path for byte-faithful preservation + of permitted producer records while retaining bounded framing, + compression, authentication, tenant scope, and asynchronous parsing. + Existing typed metrics and OTLP ingestion remain typed. diff --git a/docs/SCHEMA.md b/docs/SCHEMA.md new file mode 100644 index 0000000..1610e15 --- /dev/null +++ b/docs/SCHEMA.md @@ -0,0 +1,127 @@ + + +# Descriptive field schema + +Every reviewed Observatory field descriptor records its signal, type, unit, +meaning, sensitivity, cardinality budget, index policy, retention class, and +projection version. Built-in fields such as normalized HTTP route and status +already have explicit descriptors. + +Unknown attribute keys remain queryable only with the sensitive-telemetry +permission. They default to: + +- sensitive; +- high cardinality; +- no index; +- raw retention. + +This includes client addresses, raw queries, referrers, user agents, and +anonymous session identifiers selected through an agent source's explicit +`sensitive_fields` policy. Collection and classification are separate review +steps: opting a value into durable raw evidence does not make it public, +low-cardinality, or indexed. + +The reviewed sensitivity classes are deliberately small: + +- `public`: eligible for an explicitly approved public-safe aggregate; +- `internal`: ordinary authenticated organization telemetry; and +- `sensitive`: separately authorized evidence such as client addresses, + queries, referrers, user agents, session correlations, or unreviewed fields. + +`public` is an eligibility label, not an automatic publication instruction. +A future unauthenticated status view must compile a separate allowlisted +aggregate projection with coarse buckets, minimum group sizes, a publication +delay, and explicit organization approval. It must never expose the +authenticated dashboard, raw records, identifiers, small groups, or query +text. This public-safe projection is roadmap work rather than a capability of +the current private preview. + +`raw`, `metric`, and `evidence` are distinct lifecycle classes. `raw` survives +the hot projection cutoff in the checksummed cold archive indefinitely by +default. An organization can opt into an explicit final raw deletion window. +Only reviewed, +non-sensitive, bounded-cardinality metric dimensions marked `metric` enter the +five-minute rollup projection. Selecting `metric` is therefore a retention +decision as well as a query optimization; see +[`RETENTION.md`](RETENTION.md). + +After a raw segment is committed, Observatory records one aggregate proposal +summary per unknown field in that segment. The segment digest, organization, +and field form an idempotency key, so ingestion recovery cannot double count +proposal evidence. The proposal contains occurrence count, estimated encoded +bytes, an inferred type, first/last seen times, and a generic query using the +field name. It never stores an observed value or telemetry body. +The batch validator rejects more than 1,024 distinct attribute keys before raw +segment commit so source-controlled field names cannot create unbounded schema +work. + +Organization owners can inspect pending proposals locally: + +```sh +observatory admin descriptors list \ + --actor-user-id USER_ID \ + --organization-id ORGANIZATION_ID +``` + +Operating the host or holding the platform-operator role does not grant this +permission. Descriptor review uses the organization-scoped `schema.manage` +grant. + +## Review and activation + +The list output supplies the inferred descriptor, occurrence count, estimated +encoded bytes, first/last observation time, and generic example query. Review +those values and create a complete descriptor JSON file. The file is an +administrative input, so production requires an absolute, root-owned, +mode-`0600`, regular non-symlink file; JSON is capped at 64 KiB and unknown +properties fail closed. + +```json +{ + "version": 1, + "signal": "metrics", + "field": "workshop.queue_depth", + "type": "integer", + "unit": "items", + "meaning": "Number of work items waiting in the selected service queue.", + "sensitivity": "internal", + "cardinality": "low", + "index": "range", + "retention": "raw", + "projection_version": 1 +} +``` + +The input projection version is a placeholder; Observatory assigns the next +organization-local version during activation. + +```sh +observatory admin descriptors activate \ + --actor-user-id USER_ID \ + --organization-id ORGANIZATION_ID \ + --descriptor-file /root/reviews/workshop.queue_depth.json +``` + +Activation copies every currently active descriptor into a new immutable +version, builds the complete typed custom-field index beside the current one, +and changes the active version inside the same organization-database +transaction. Invalid historical values stay in raw evidence but are not +coerced into the index. New ingestion writes to the active version. Old index +tables remain available for recovery, and retry repairs an interrupted +control-database acknowledgement without rebuilding an already active, +identical descriptor. + +A proposal that should not become part of the reviewed schema can be rejected +idempotently: + +```sh +observatory admin descriptors reject \ + --actor-user-id USER_ID \ + --organization-id ORGANIZATION_ID \ + --signal metrics \ + --field workshop.queue_depth +``` + +An active descriptor cannot be rejected. Revision of an already active +descriptor requires a future explicit review-proposal workflow; the current +command will not silently reinterpret an active field. diff --git a/docs/SECURITY_CAMPAIGN.md b/docs/SECURITY_CAMPAIGN.md new file mode 100644 index 0000000..1e6d7cb --- /dev/null +++ b/docs/SECURITY_CAMPAIGN.md @@ -0,0 +1,147 @@ + + +# Security and recovery campaign + +This document maps the current maintainer self-assessment to executable +evidence. It is not an independent audit or a claim that the public preview is +ready. The release commit, packaged binary, browser campaign, capacity run, +and fleet soak must receive separate evidence before the preview tag. + +## Current executable evidence + +| Boundary | Executable evidence | +| --- | --- | +| Organization isolation and scoped queries | `TestBootstrapSeparatesPlatformAndOrganizationAccess`, `TestTeamsInvitationsRevocationAndBreakGlassRemainOrganizationScoped`, `TestPlanRejectsCrossTenantScopeAndScanBudget`, `TestCredentialScopeCannotBeOverridden` | +| Invitations, teams, revocation, and break-glass audit/expiry | `TestTeamsInvitationsRevocationAndBreakGlassRemainOrganizationScoped` plus the pinned Web Foundations package tests | +| Generated bootstrap credential, forced first-login rotation, and local recovery | `TestAdminBootstrapGeneratesExclusiveOneTimeCredential`, `TestAdminBootstrapSupportsConfinedSystemdCredentials`, `TestAdminHierarchyAndEnrollmentSupportConfinedSystemdCredentials`, `TestAdminUserResetPasswordGeneratesPrivateCredentialAndRevokesSessions`, `TestTemporaryOperatorMustRotatePasswordBeforeUsingApplication`, `TestAPITemporaryOperatorReceivesScopedRotationToken`, plus pinned Web Foundations password-change and atomic recovery tests | +| Token-bound HTML forms with privacy-browser compatibility and cross-site rejection | `TestHTMLLoginUsesTokenWhenBrowserOmitsOrObscuresOriginMetadata`, `TestHTMLLoginFailsClosed`, `TestTemporaryOperatorMustRotatePasswordBeforeUsingApplication`, `TestDashboardManagementIsScopedCSRFProtectedAndExportable`, `TestIncidentRulesEvaluationInboxAndResponseAreScoped` | +| Replay, exact logical-batch acknowledgement, duplicate batches, sequence gaps, enrollment, and source revocation | `TestScopedIngestionDeduplicationAndReplay`, `TestNativeEnvelopeReplayUsesBatchIdentityAndAllowsOverlappingTime`, `TestFramedNativeIngestAcknowledgesExactReplayAndOverlappingTime`, `TestSendAcceptsRealServerBatchDigestRatherThanPrivateSegmentDigest`, `TestRunnerPreservesSpoolOnMismatchedAcknowledgement`, `TestEnrollmentIsScopedExpiringAndSingleUse`, `TestAgentEnrollmentIsSingleUseAndCredentialCanSelfRevoke` | +| Native envelope and compressed or malformed OTLP framing | `TestEnvelopeHeadersRoundTripAndRejectAmbiguity`, `FuzzParseEnvelopeHeaders`, `TestOTLPHTTPIngestionIsAuthenticatedBoundedAndCompressed`, `TestDecodeRejectsMalformedAndNonFiniteData`, `FuzzDecode` | +| Cardinality, timestamp, field, query, scan, memory, and result bounds | `TestBatchRejectsDistinctFieldCardinalityAbuse`, `TestBatchClockSkewAndRetentionWindowsFailClosed`, `TestQueryEnforcesSensitiveAndExecutionBudgets`, `FuzzParse` | +| Secret minimization and safe error output | `TestCaddyCollectorDropsSecretsAndQuery`, `TestRequestLogCollectorUsesWhitelist`, `TestDecodeLogsDropsCredentialsAndPreservesTelemetry`, `TestIngestDoesNotExposeValuesInErrors` | +| Filesystem and interrupted-write boundaries | `TestStorageRejectsSymlinkedSQLiteFilesAndProjectionDirectories`, `TestStateAtomicRoundTripAndSymlinkRefusal`, `TestSpoolRejectsQuotaAndSymlink`, `TestTailerPreservesPartialLineAndRecoversRotation`, `TestInterruptedTemporarySegmentIsIgnoredUntilAtomicCommit` | +| Raw corruption and crash recovery | `TestCommitReadAndCorruption`, `TestRecoveryIndexesRawSegmentMissingFromControlDatabase`, `TestProjectionRebuildFailurePreservesLiveProjection` | +| Complete projection reconstruction | `TestProjectionRebuildRestoresRawTruthAndActivatedDescriptorsAtomically`, `TestProjectionRebuildRejectsUnknownOrganization`, `TestProjectionRebuildRefusesSymlinkProjectionOrSidecar` | +| Migration locking and schema fixtures | `TestProcessLockSeparatesLiveServerFromOfflineMigration`, `TestOfflineMigrationRefusesLiveDataDirectory`, `TestControlSchemaFourMigratesToIncidentSchema`, `TestControlSchemaFiveMigratesToPushSchema` | +| PWA, SSE, incident scope, and generic push content | `TestManifestAndServiceWorkerUseExactContentAddressedShell`, `TestLiveRefreshStreamIsAuthorizedAndCarriesNoTelemetry`, `TestIncidentRulesEvaluationInboxAndResponseAreScoped`, `TestSenderUsesEncryptedGenericPayloadAndValidVAPID` | +| Tend input and non-authoritative deployment evidence | `TestTendCollectorIsStrict`, `FuzzTendCollector`; producer activation/rollback failure injection remains authoritative in Tend's own repository | + +The migration fixtures cover every internal control schema that predates this +candidate and can reach its current schema. Observatory has not published a +preview, so no public-version migration claim exists yet. Each future public +preview must retain a fixture and a raw-projection rebuild path. + +## Reproduction + +The ordinary verifier pins Go and Hime-san, checks generated-source freshness +and no-op determinism, then runs tests, the race detector, vet, a trimmed +build, and the production dependency boundary: + +```sh +GOCACHE=/tmp/observatory-go-cache ./scripts/verify.sh +``` + +The bounded adversarial campaign adds query, OTLP, and Tend-event fuzzing and +reruns the security-critical packages under the race detector: + +```sh +OBSERVATORY_FUZZ_TIME=30s \ +GOCACHE=/tmp/observatory-go-cache \ +./scripts/security-campaign.sh +``` + +On August 17, 2026, a 10-second-per-target development run completed without +an invariant failure: approximately 2.45 million query-parser cases, 670,000 +OTLP cases, and 166,000 strict Tend-event cases, followed by the uncached race +matrix. Counts are observations from one run, not minimum performance claims. + +Trusted Gitea assurance run 232 then exercised source commit +`14125387d13947eb4a523e95c2e441ff89abcc1a` (tree +`f378872450054e7ff6f5499c61a5db0e4a3d3da7`) with the pinned Go 1.26.6 +toolchain and `govulncheck` v1.6.0. It completed 1,092,559 query-parser, +398,595 OTLP-decoder, and 324,020 Tend-event fuzz executions in three separate +30-second targets, then passed the uncached security-package race matrix and +the ordinary deterministic verifier. The checkout remained unchanged. + +The same scan reported zero reachable vulnerabilities and zero +vulnerabilities in imported packages. It also reported GO-2026-5932 in the +required `golang.org/x/crypto` module because its legacy `openpgp` package is +unmaintained. Observatory and its Web Foundations dependency do not import +that package, and the advisory has no fixed module version. This is a recorded +dependency boundary, not a claim that the module-only advisory was repaired. + +The resource-limited ingestion, query, outage-replay, and retention gate is +defined separately in [`CAPACITY.md`](CAPACITY.md). Its short development mode +is suitable for implementation feedback; only the exact one-hour release mode +can close the capacity item below. + +## Open release evidence + +- Exercise one granted browser-vendor Web Push delivery and OS notification + activation in a headed supported browser. The networkless Chromium campaign + below proves the application and worker boundaries without contacting a push + relay or depending on a desktop notification service. +- Complete the specified 4-vCPU/8-GiB capacity campaign, outage replay, + retention, compaction, and concurrent-organization workloads. The + August 17 constrained run passed its one-hour 2,000-observation/second + sustain boundary but missed the 10,000-observation/second burst boundary; + [`CAPACITY.md`](CAPACITY.md) records the measured result and remaining work. +- Complete the medium-fleet dogfood soak. Observatory's first real Tend + activation, rollback, and identical-artifact redeployment passed on + August 17; [`DEPLOYMENT.md`](DEPLOYMENT.md) records that bounded result. +- Re-run this campaign against the exact packaged release commit and record + its immutable artifact digest, SBOM, checksum, and signature. + +## Real-browser campaign + +`scripts/browser-campaign.sh` builds a build-tagged, disposable HTTPS fixture +and drives Chromium through pinned Playwright 1.62.1. The approved Linux image +is `mcr.microsoft.com/playwright@sha256:dcc5531e97840b9b5e794f2814476b21571c5124a3fca2267d73041f56e7580e`. +The fixture and browser use loopback only. The campaign rejects every HTTP +request to another origin and verifies manifest installability, the public +offline shell, explicit private-inbox caching and sign-out deletion, SSE +reconnection, app badging, generic notification content and activation, +keyboard entry, landmarks, forced colors, reduced motion, and 320-pixel +overflow. + +On August 17, 2026, the campaign completed twice in pinned headless Chromium +inside a disposable, networkless Linux container limited to four CPUs and +8 GiB. Both runs reported every evidence field true and zero external HTTP +requests. They exercised a real service worker, cache storage, EventSource, +session, incident inbox, application badge, and the production push and +notification-click handlers. Headless Chromium denied the intentional +user-triggered notification permission request, so the worker's fixed-shape +push event and activation path were invoked directly; the prohibited OS focus +side effect was replaced with an in-worker client test double. This proves the +payload and activation contract, not end-to-end browser-vendor delivery. The +exact npm lockfile audit reported zero known vulnerabilities at all severities. + +The disposable fixture uses a one-hour self-signed `localhost` certificate and +passes only its ephemeral SHA-256 subject-public-key fingerprint to Chromium's +SPKI allowlist. Chromium serializes `Origin: null` on form submissions under +that synthetic certificate while Fetch Metadata still reports `same-origin`. +The fixture does not rewrite those headers: the campaign exercises the same +token-bound HTML form policy used in production and asserts the observed +metadata. The token remains mandatory, and explicit cross-site metadata still +fails closed. This is not a general TLS-origin acceptance test. + +To test native EventSource recovery deterministically, the build-tagged +fixture cancels the real authenticated `/app/events` request after 1.25 +seconds. The browser must reconnect and receive a later generic refresh event. +The production handler, authorization, event format, and connection lifetime +are not changed. + +Install the exact development dependency without downloading a browser when a +pinned Playwright container supplies Chromium: + +```sh +cd test/browser +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm ci --ignore-scripts +cd ../.. +./scripts/browser-campaign.sh +``` + +The resulting JSON contains pass/fail booleans and no machine, user, token, +organization, incident, path, or telemetry identifiers. A development pass is +not release evidence until repeated against the packaged release commit and +the pinned container digest is recorded. diff --git a/docs/TEND.md b/docs/TEND.md new file mode 100644 index 0000000..751054f --- /dev/null +++ b/docs/TEND.md @@ -0,0 +1,76 @@ + + +# Tend deployment evidence + +Observatory consumes Tend's bounded version-1 deployment-event JSONL as an +unprivileged, read-only file source. Tend does not call Observatory and never +waits for an Observatory acknowledgement. Identity, entropy, event-file, agent, +network, and server failures therefore cannot decide whether Tend activates or +rolls back an application. + +Each accepted record has exactly these fields: + +- a cryptographically random operation ID; +- service, approved artifact SHA-256, source commit, and release version; +- phase, optional slot, elapsed milliseconds, outcome, and UTC observation + time. + +The adapter rejects unknown fields, trailing JSON, negative durations, +malformed identifiers, unsafe values, invalid timestamps, and records larger +than Tend's 4-KiB producer limit. It discards the source line after converting +only this fixed field set into a deployment observation. No environment value, +secret path, HTTP body, command output, or arbitrary process output belongs to +the contract. + +## Agent configuration + +Grant the dedicated Observatory agent account read access only to the selected +service's event file, then configure an explicit local source: + +```json +{ + "kind": "tend_events_jsonl", + "path": "/opt/example/deployment-events.jsonl", + "stream_id": "tend-deployments" +} +``` + +The source's organization, project, environment, and service scope comes from +its server-side enrollment. Neither the file nor a deployment event may choose +tenant scope. The normal durable cursor and spool rules apply during rotation, +agent restarts, or an Observatory outage. + +## Querying + +Activation and rollback attempts share the `tend.deployment` observation name +and correlate through `deployment.operation_id`: + +```text +deployments +| where service.name == "example-site" +| sort deployment.duration_ms desc +| limit 50 +``` + +Keep candidate, activation, and rollback phases together when reconstructing a +release operation. A missing event means evidence was unavailable; it must not +be interpreted as proof that a deployment did not occur. Tend's own state and +the application's active release remain authoritative for deployment control. + +The producer contract is maintained by [Gamertan Tend](https://gitea.speelman.ca/gamertan/tend). + +## Production dogfood observation + +On August 18, 2026, one checksum-approved Tend candidate exercised candidate +validation, activation, explicit rollback, and reactivation for Gamertan, +Sandwich Hime, and Observatory. The production agent then projected all three +bounded event files through separate streams: ten Gamertan events and six for +each of the two singleton services. Gamertan's stream intentionally includes +two failed pre-activation candidates that never received public traffic; those +records helped identify a hardened-umask extraction defect before the corrected +candidate completed all three service cycles. + +This observation demonstrates the producer-to-agent-to-projection path and its +failure evidence. It is not a substitute for Tend state, active-release +inspection, public health checks, or the remaining Observatory fleet and +capacity release gates. diff --git a/examples/Caddyfile.sensitive-access-log b/examples/Caddyfile.sensitive-access-log new file mode 100644 index 0000000..1752c8f --- /dev/null +++ b/examples/Caddyfile.sensitive-access-log @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: 0BSD +# +# Explicit richer-evidence example. Replace the origin, upstream, paths, and +# query-key deny list only after reviewing the application's privacy policy. + +example.com { + log application_sensitive_access { + output file /var/log/caddy/example-sensitive-access.jsonl { + mode 0640 + roll_size 100mb + roll_keep 10 + roll_keep_for 720h + } + format filter { + request>uri query { + delete access_token + delete api_key + delete authorization + delete code + delete credential + delete key + delete password + delete secret + delete session + delete session_id + delete token + } + client_ip ip_mask 24 56 + request>remote_ip delete + request>remote_port delete + request>client_ip delete + request>headers delete + resp_headers delete + user_id delete + wrap json + } + } + log_append request_id {http.response.header.X-Request-ID} + log_append client_ip {http.request.client_ip} + log_append referrer {http.request.header.Referer} + log_append user_agent {http.request.header.User-Agent} + reverse_proxy 127.0.0.1:8080 +} diff --git a/examples/LICENSE b/examples/LICENSE new file mode 100644 index 0000000..540369f --- /dev/null +++ b/examples/LICENSE @@ -0,0 +1,12 @@ +Zero-Clause BSD + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..eec6336 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,23 @@ + + +# Examples + +Copy these files into your own deployment repository and adapt them. Production configuration belongs at `/etc/gamertan-observatory/`, must be root-owned mode `0600`, and should reference secrets stored in separate root-owned files. The example systemd unit uses `LoadCredential=` so the agent can consume runtime copies while remaining an unprivileged service. + +The optional `web_push` block expects a private key created on the server with +`observatory admin web-push generate-key --output-file /etc/gamertan-observatory/web-push.json`. +Remove the block when browser notifications are not wanted. Never commit the +generated file. + +`agent.json` demonstrates the privacy-minimized collection default. If an +application's privacy policy and operating purpose permit collection of client +addresses, queries, referrers, user agents, or anonymous session identifiers, +select those names explicitly in that source's `sensitive_fields` array. The +agent will not retain them merely because they exist in the producer log. + +`Caddyfile.sensitive-access-log` is an intentionally richer edge-log example. +It still deletes request and response header maps, filters common credential +query keys, masks client addresses, and keeps the application request ID as a +correlation field. Its deny list cannot know an application's vocabulary: +review and extend it before use. Treat the resulting file as sensitive source +evidence with narrow read permissions and a documented retention period. diff --git a/examples/agent-credential.json b/examples/agent-credential.json new file mode 100644 index 0000000..666a5eb --- /dev/null +++ b/examples/agent-credential.json @@ -0,0 +1,4 @@ +{ + "credential": "replace-with-the-single-enrollment-result" +} + diff --git a/examples/agent-credential.json.license b/examples/agent-credential.json.license new file mode 100644 index 0000000..3227c7d --- /dev/null +++ b/examples/agent-credential.json.license @@ -0,0 +1 @@ +SPDX-License-Identifier: 0BSD diff --git a/examples/agent-enrollment.json b/examples/agent-enrollment.json new file mode 100644 index 0000000..95a408f --- /dev/null +++ b/examples/agent-enrollment.json @@ -0,0 +1,3 @@ +{ + "enrollment_token": "replace-with-the-single-use-enrollment-token" +} diff --git a/examples/agent-enrollment.json.license b/examples/agent-enrollment.json.license new file mode 100644 index 0000000..3227c7d --- /dev/null +++ b/examples/agent-enrollment.json.license @@ -0,0 +1 @@ +SPDX-License-Identifier: 0BSD diff --git a/examples/agent.json b/examples/agent.json new file mode 100644 index 0000000..95d3783 --- /dev/null +++ b/examples/agent.json @@ -0,0 +1,42 @@ +{ + "schema": 1, + "server_url": "https://observatory.example.com", + "credential_file": "/etc/gamertan-observatory/agent-credential.json", + "spool_dir": "/var/lib/gamertan-observatory-agent", + "state_file": "/var/lib/gamertan-observatory-agent/state.json", + "max_spool_bytes": 5368709120, + "max_spool_age": "72h", + "batch_records": 5000, + "flush_interval": "1s", + "sources": [ + { + "kind": "caddy_json", + "path": "/var/log/caddy/example-access.jsonl", + "stream_id": "caddy-access" + }, + { + "kind": "requestlog_jsonl", + "path": "/var/log/example/request.jsonl", + "stream_id": "application-request" + }, + { + "kind": "tend_events_jsonl", + "path": "/opt/example/deployment-events.jsonl", + "stream_id": "tend-deployments" + }, + { + "kind": "linux_metrics", + "stream_id": "host-metrics", + "linux_metrics": { + "proc_root": "/proc", + "cgroup_root": "/sys/fs/cgroup", + "filesystems": [ + { "name": "root", "path": "/" } + ], + "cgroups": [ + { "name": "example-service", "path": "system.slice/example.service" } + ] + } + } + ] +} diff --git a/examples/agent.json.license b/examples/agent.json.license new file mode 100644 index 0000000..3227c7d --- /dev/null +++ b/examples/agent.json.license @@ -0,0 +1 @@ +SPDX-License-Identifier: 0BSD diff --git a/examples/observatory-agent.service b/examples/observatory-agent.service new file mode 100644 index 0000000..da6308e --- /dev/null +++ b/examples/observatory-agent.service @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: 0BSD + +[Unit] +Description=Gamertan Observatory agent +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=observatory-agent +Group=observatory-agent +LoadCredential=agent.json:/etc/gamertan-observatory/agent.json +LoadCredential=agent-credential.json:/etc/gamertan-observatory/agent-credential.json +ExecStart=/usr/local/bin/observatory agent --systemd-credentials --config %d/agent.json --credential-file %d/agent-credential.json +Restart=on-failure +RestartSec=5s +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectControlGroups=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +ReadWritePaths=/var/lib/gamertan-observatory-agent + +[Install] +WantedBy=multi-user.target diff --git a/examples/server.json b/examples/server.json new file mode 100644 index 0000000..a0d0f05 --- /dev/null +++ b/examples/server.json @@ -0,0 +1,30 @@ +{ + "schema": 1, + "listen": "127.0.0.1:9010", + "public_url": "https://observatory.example.com", + "data_dir": "/var/lib/gamertan-observatory", + "max_body_bytes": 33554432, + "max_concurrent_ingest": 8, + "session_lifetime": "12h", + "query": { + "max_duration": "5s", + "max_rows": 10000, + "max_scanned_bytes": 268435456, + "max_memory_bytes": 134217728 + }, + "retention": { + "raw_logs_days": 30, + "raw_traces_days": 30, + "raw_metrics_days": 14, + "cold_raw_days": 400, + "delete_cold_raw": false, + "metric_rollups_days": 400, + "evidence_days": 400 + }, + "web_push": { + "private_key_file": "/etc/gamertan-observatory/web-push.json", + "subject": "mailto:security@example.com", + "queue_capacity": 64, + "request_timeout": "10s" + } +} diff --git a/examples/server.json.license b/examples/server.json.license new file mode 100644 index 0000000..3227c7d --- /dev/null +++ b/examples/server.json.license @@ -0,0 +1 @@ +SPDX-License-Identifier: 0BSD diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1306a7c --- /dev/null +++ b/go.mod @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +module gamertan.com/observatory + +go 1.26 + +require ( + gamertan.com/sandwich-hime/sando v1.0.0-beta.1 + gamertan.com/web v0.1.0-preview.4 + github.com/klauspost/compress v1.18.7 + go.opentelemetry.io/proto/otlp v1.11.0 + google.golang.org/protobuf v1.36.11 + modernc.org/sqlite v1.56.0 +) + +require ( + gamertan.com/tend v0.2.0-preview.2 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) + +tool gamertan.com/tend/cmd/tend diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..621dd14 --- /dev/null +++ b/go.sum @@ -0,0 +1,66 @@ +gamertan.com/sandwich-hime/sando v1.0.0-beta.1 h1:WHe6LnVKvOz7CQr9OIWI78pCzIFIAGgoXlGfjYVAVoU= +gamertan.com/sandwich-hime/sando v1.0.0-beta.1/go.mod h1:awOXo4t2zNxxgH60qkTpMOdhYnultm9AfpjlN9gOEis= +gamertan.com/tend v0.2.0-preview.2 h1:1bcu+0cHEtbw3aqnBX0PaS0WWQf3v88Mw3wwrZZwXVA= +gamertan.com/tend v0.2.0-preview.2/go.mod h1:JEzonyOsZsG+jcPDCE6LkTw6E8rHg+Vp5TFgBz5sEVE= +gamertan.com/web v0.1.0-preview.4 h1:n5s0njTVgBe+5jlWIv6u2wnKZGU/hN/UfgBrJU9MpJM= +gamertan.com/web v0.1.0-preview.4/go.mod h1:KOxki9zGWgNb9+1H6OsvnShBijLvw+ZjurBR8D8FZW0= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/agent/agent.go b/internal/agent/agent.go new file mode 100644 index 0000000..b9462e4 --- /dev/null +++ b/internal/agent/agent.go @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package agent + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "gamertan.com/observatory/internal/agentclient" + "gamertan.com/observatory/internal/agentstate" + "gamertan.com/observatory/internal/config" + "gamertan.com/observatory/internal/edgealert" + "gamertan.com/observatory/internal/hostmetrics" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/spool" + "gamertan.com/observatory/internal/storage" + "gamertan.com/observatory/internal/tailer" +) + +type Sender interface { + Send(context.Context, model.Batch) (storage.Ack, error) + SendAlertTransition(context.Context, model.AlertTransition) (storage.SourceAlertTransitionAck, error) +} + +type Runner struct { + configuration config.Agent + sourceID string + stateStore *agentstate.Store + state agentstate.State + spool *spool.Spool + sender Sender + agentEpoch string +} + +func Open(configuration config.Agent, credential string, transport http.RoundTripper) (*Runner, error) { + sourceID, err := sourceIDFromCredential(credential) + if err != nil { + return nil, err + } + stateStore, state, err := agentstate.Open(configuration.StateFile) + if err != nil { + return nil, err + } + queue, err := spool.Open(configuration.SpoolDir, configuration.MaxSpoolBytes, configuration.MaxSpoolAge) + if err != nil { + return nil, err + } + client, err := agentclient.New(configuration.ServerURL, credential, transport) + if err != nil { + return nil, err + } + return newRunner(configuration, sourceID, stateStore, state, queue, client, agentEpoch(credential)) +} + +func New(configuration config.Agent, sourceID string, stateStore *agentstate.Store, state agentstate.State, queue *spool.Spool, sender Sender) (*Runner, error) { + return newRunner(configuration, sourceID, stateStore, state, queue, sender, agentEpoch(sourceID)) +} + +func newRunner(configuration config.Agent, sourceID string, stateStore *agentstate.Store, state agentstate.State, queue *spool.Spool, sender Sender, epoch string) (*Runner, error) { + if err := model.ValidateSourceID(sourceID); err != nil || stateStore == nil || queue == nil || sender == nil { + return nil, errors.New("agent runtime dependencies are invalid") + } + if err := state.Validate(); err != nil { + return nil, err + } + if len(epoch) != 32 { + return nil, errors.New("agent epoch is invalid") + } + return &Runner{configuration: configuration, sourceID: sourceID, stateStore: stateStore, state: state, spool: queue, sender: sender, agentEpoch: epoch}, nil +} + +func (runner *Runner) RunOnce(ctx context.Context, now time.Time) error { + if now.IsZero() { + return errors.New("agent cycle time is required") + } + var cycleErrors []error + if err := runner.recoverCheckpoints(now); err != nil { + return err + } + deliveryFailed := false + if err := runner.deliver(ctx, now); err != nil { + cycleErrors = append(cycleErrors, err) + deliveryFailed = true + } + for _, source := range runner.configuration.Sources { + cursor := runner.state.Streams[source.StreamID] + if source.Kind == "linux_metrics" { + observations, collectErr := hostmetrics.Collect(*source.LinuxMetrics, now) + if len(observations) == 0 { + if collectErr != nil { + cycleErrors = append(cycleErrors, fmt.Errorf("collect stream %s: %w", source.StreamID, collectErr)) + } + continue + } + for start := 0; start < len(observations); start += runner.configuration.BatchRecords { + end := min(start+runner.configuration.BatchRecords, len(observations)) + cursor.Sequence++ + spooled, err := runner.spoolObservations(source.StreamID, cursor, model.SignalMetrics, observations[start:end], now) + if err != nil { + if spooled { + return err + } + cycleErrors = append(cycleErrors, fmt.Errorf("spool stream %s: %w", source.StreamID, err)) + break + } + } + if collectErr != nil { + cycleErrors = append(cycleErrors, fmt.Errorf("collect stream %s: %w", source.StreamID, collectErr)) + } + continue + } + result, err := tailer.Read(source, cursor, runner.configuration.BatchRecords, now) + if err != nil { + cycleErrors = append(cycleErrors, fmt.Errorf("collect stream %s: %w", source.StreamID, err)) + continue + } + if len(result.Observations) == 0 { + if result.Cursor != cursor { + runner.state.Streams[source.StreamID] = result.Cursor + if err = runner.stateStore.Save(runner.state); err != nil { + return err + } + } + continue + } + result.Cursor.Sequence = cursor.Sequence + 1 + spooled, err := runner.spoolObservations(source.StreamID, result.Cursor, result.Signal, result.Observations, now) + if err != nil { + if spooled { + return err + } + cycleErrors = append(cycleErrors, fmt.Errorf("spool stream %s: %w", source.StreamID, err)) + continue + } + } + if !deliveryFailed { + if err := runner.deliver(ctx, now); err != nil { + cycleErrors = append(cycleErrors, err) + } + } + return errors.Join(cycleErrors...) +} + +func (runner *Runner) spoolObservations(streamID string, cursor agentstate.Cursor, signal model.Signal, observations []model.Observation, now time.Time) (bool, error) { + batch := model.Batch{Version: model.BatchVersion, SourceID: runner.sourceID, StreamID: streamID, Sequence: cursor.Sequence, ObservedAt: now.UTC(), Signal: signal, Records: observations} + checkpoint, err := json.Marshal(cursor) + if err != nil { + return false, err + } + if _, err = runner.spool.PutWithCheckpoint(batch, checkpoint, now); err != nil { + return false, err + } + runner.state.Streams[streamID] = cursor + return true, runner.stateStore.Save(runner.state) +} + +func (runner *Runner) recoverCheckpoints(now time.Time) error { + entries, err := runner.spool.List(now) + if err != nil { + return err + } + changed := false + for _, entry := range entries { + _, checkpoint, err := runner.spool.ReadWithCheckpoint(entry) + if err != nil { + return err + } + if len(checkpoint) == 0 { + return errors.New("pending spool batch lacks a cursor checkpoint") + } + cursor, err := decodeCursor(checkpoint) + if err != nil || cursor.Sequence != entry.Sequence { + return errors.New("pending spool checkpoint is invalid") + } + current := runner.state.Streams[entry.StreamID] + if cursor.Sequence < current.Sequence { + continue + } + if cursor.Sequence == current.Sequence { + if cursor != current { + return errors.New("pending spool checkpoint conflicts with agent state") + } + continue + } + if cursor.Sequence != current.Sequence+1 { + return errors.New("pending spool checkpoint has a sequence gap") + } + runner.state.Streams[entry.StreamID] = cursor + changed = true + } + if changed { + return runner.stateStore.Save(runner.state) + } + return nil +} + +func (runner *Runner) deliver(ctx context.Context, now time.Time) error { + entries, err := runner.spool.List(now) + if err != nil { + return err + } + blocked := map[string]bool{} + var deliveryErrors []error + for _, entry := range entries { + if blocked[entry.StreamID] { + continue + } + batch, _, err := runner.spool.ReadWithCheckpoint(entry) + if err != nil { + return err + } + ack, sendErr := runner.sender.Send(ctx, batch) + if sendErr != nil { + blocked[entry.StreamID] = true + deliveryErrors = append(deliveryErrors, fmt.Errorf("deliver stream %s sequence %d: %w", entry.StreamID, entry.Sequence, sendErr)) + continue + } + batchDigest, digestErr := batch.Digest() + if digestErr != nil || ack.SourceID != batch.SourceID || ack.StreamID != batch.StreamID || ack.Sequence != batch.Sequence || ack.BatchDigest != batchDigest { + blocked[entry.StreamID] = true + deliveryErrors = append(deliveryErrors, fmt.Errorf("deliver stream %s sequence %d: acknowledgement does not match exact batch", entry.StreamID, entry.Sequence)) + continue + } + transitionFailed := false + for _, rule := range runner.configuration.AlertRules { + if rule.StreamID != batch.StreamID { + continue + } + evaluation, evaluationErr := edgealert.Evaluate(rule, batch) + if evaluationErr != nil { + blocked[entry.StreamID] = true + deliveryErrors = append(deliveryErrors, fmt.Errorf("evaluate alert rule %s for stream %s sequence %d: %w", rule.ID, entry.StreamID, entry.Sequence, evaluationErr)) + transitionFailed = true + break + } + transition := model.AlertTransition{Version: model.AlertTransitionVersion, RuleID: rule.ID, RuleRevision: rule.Revision, AgentEpoch: runner.agentEpoch, Sequence: batch.Sequence, StreamID: batch.StreamID, BatchSequence: batch.Sequence, SegmentDigest: ack.Digest, WindowStart: evaluation.WindowStart, WindowEnd: evaluation.WindowEnd, State: evaluation.State, ObservedAt: evaluation.ObservedAt} + transitionAck, transitionErr := runner.sender.SendAlertTransition(ctx, transition) + expectedDigest, digestErr := transition.Digest() + if transitionErr != nil || digestErr != nil || transitionAck.SourceID != batch.SourceID || transitionAck.RuleID != transition.RuleID || transitionAck.RuleRevision != transition.RuleRevision || transitionAck.AgentEpoch != transition.AgentEpoch || transitionAck.Sequence != transition.Sequence || transitionAck.Digest != expectedDigest { + blocked[entry.StreamID] = true + if transitionErr == nil { + transitionErr = errors.New("acknowledgement does not match exact transition") + } + deliveryErrors = append(deliveryErrors, fmt.Errorf("deliver alert rule %s for stream %s sequence %d: %w", rule.ID, entry.StreamID, entry.Sequence, transitionErr)) + transitionFailed = true + break + } + } + if transitionFailed { + continue + } + if err = runner.spool.Acknowledge(entry, entry.Digest); err != nil { + return err + } + } + return errors.Join(deliveryErrors...) +} + +func agentEpoch(sourceID string) string { + digest := sha256.Sum256([]byte("observatory-agent-epoch-v1\x00" + sourceID)) + return hex.EncodeToString(digest[:16]) +} + +func decodeCursor(body []byte) (agentstate.Cursor, error) { + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + var cursor agentstate.Cursor + if err := decoder.Decode(&cursor); err != nil { + return agentstate.Cursor{}, err + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return agentstate.Cursor{}, errors.New("cursor checkpoint has trailing data") + } + if cursor.Offset < 0 || (cursor.Device == 0) != (cursor.Inode == 0) || cursor.Sequence == 0 { + return agentstate.Cursor{}, errors.New("cursor checkpoint is invalid") + } + return cursor, nil +} + +func sourceIDFromCredential(credential string) (string, error) { + if !strings.HasPrefix(credential, "obs1.") || strings.ContainsAny(credential, " \t\r\n") { + return "", errors.New("source credential is invalid") + } + remainder := strings.TrimPrefix(credential, "obs1.") + separator := strings.LastIndexByte(remainder, '.') + if separator < 1 || separator == len(remainder)-1 { + return "", errors.New("source credential is invalid") + } + sourceID := remainder[:separator] + if err := model.ValidateSourceID(sourceID); err != nil { + return "", errors.New("source credential is invalid") + } + return sourceID, nil +} diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go new file mode 100644 index 0000000..3b44e75 --- /dev/null +++ b/internal/agent/agent_test.go @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package agent + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/agentstate" + "gamertan.com/observatory/internal/config" + "gamertan.com/observatory/internal/hostmetrics" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/spool" + "gamertan.com/observatory/internal/storage" +) + +type fakeSender struct { + fail bool + failAlerts bool + batchDigests map[uint64]string + sent []uint64 + alerts []model.AlertTransition +} + +func (sender *fakeSender) SendAlertTransition(_ context.Context, transition model.AlertTransition) (storage.SourceAlertTransitionAck, error) { + if sender.failAlerts { + return storage.SourceAlertTransitionAck{}, errors.New("alert endpoint unavailable") + } + sender.alerts = append(sender.alerts, transition) + digest, err := transition.Digest() + if err != nil { + return storage.SourceAlertTransitionAck{}, err + } + return storage.SourceAlertTransitionAck{SourceID: "source", RuleID: transition.RuleID, RuleRevision: transition.RuleRevision, AgentEpoch: transition.AgentEpoch, Sequence: transition.Sequence, Digest: digest}, nil +} + +func (sender *fakeSender) Send(_ context.Context, batch model.Batch) (storage.Ack, error) { + if sender.fail { + return storage.Ack{}, errors.New("server unavailable") + } + sender.sent = append(sender.sent, batch.Sequence) + batchDigest, err := batch.Digest() + if err != nil { + return storage.Ack{}, err + } + if override := sender.batchDigests[batch.Sequence]; override != "" { + batchDigest = override + } + return storage.Ack{SourceID: batch.SourceID, StreamID: batch.StreamID, Sequence: batch.Sequence, Digest: strings.Repeat("d", 64), BatchDigest: batchDigest}, nil +} + +func TestRunnerCollectsWhileOfflineAndRecoversCheckpoint(t *testing.T) { + root := t.TempDir() + logPath := filepath.Join(root, "request.jsonl") + if err := os.WriteFile(logPath, []byte(requestLine("one")+"\n"), 0o640); err != nil { + t.Fatal(err) + } + spoolRoot := filepath.Join(root, "spool") + configuration := config.Agent{BatchRecords: 1, MaxSpoolBytes: 1 << 20, MaxSpoolAge: 72 * time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}} + stateStore, state, err := agentstate.Open(configuration.StateFile) + if err != nil { + t.Fatal(err) + } + queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge) + if err != nil { + t.Fatal(err) + } + sender := &fakeSender{fail: true} + runner, err := New(configuration, "source", stateStore, state, queue, sender) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC) + if err = runner.RunOnce(context.Background(), now); err == nil { + t.Fatal("offline delivery unexpectedly succeeded") + } + file, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err = file.WriteString(requestLine("two") + "\n"); err != nil { + t.Fatal(err) + } + file.Close() + if err = runner.RunOnce(context.Background(), now.Add(time.Second)); err == nil { + t.Fatal("offline delivery unexpectedly succeeded") + } + entries, err := queue.List(now.Add(time.Second)) + if err != nil || len(entries) != 2 || entries[0].Sequence != 1 || entries[1].Sequence != 2 { + t.Fatalf("entries=%+v err=%v", entries, err) + } + // Simulate a crash after both durable spool commits but before either cursor + // checkpoint reached the state file. Recovery must advance state from the + // spool envelopes before it reads the source again. + empty := agentstate.State{Version: agentstate.Version, Streams: map[string]agentstate.Cursor{}} + if err = stateStore.Save(empty); err != nil { + t.Fatal(err) + } + _, recoveredState, err := agentstate.Open(configuration.StateFile) + if err != nil { + t.Fatal(err) + } + sender.fail = false + restarted, err := New(configuration, "source", stateStore, recoveredState, queue, sender) + if err != nil { + t.Fatal(err) + } + if err = restarted.RunOnce(context.Background(), now.Add(2*time.Second)); err != nil { + t.Fatal(err) + } + if len(sender.sent) != 2 || sender.sent[0] != 1 || sender.sent[1] != 2 { + t.Fatalf("sent=%v", sender.sent) + } + if entries, err = queue.List(now.Add(2 * time.Second)); err != nil || len(entries) != 0 { + t.Fatalf("remaining=%+v err=%v", entries, err) + } + _, finalState, err := agentstate.Open(configuration.StateFile) + if err != nil || finalState.Streams["request"].Sequence != 2 { + t.Fatalf("state=%+v err=%v", finalState, err) + } +} + +func TestRunnerPreservesSpoolOnMismatchedAcknowledgement(t *testing.T) { + root := t.TempDir() + logPath := filepath.Join(root, "request.jsonl") + if err := os.WriteFile(logPath, []byte(requestLine("one")+"\n"), 0o640); err != nil { + t.Fatal(err) + } + spoolRoot := filepath.Join(root, "spool") + configuration := config.Agent{BatchRecords: 1, MaxSpoolBytes: 1 << 20, MaxSpoolAge: 72 * time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}} + stateStore, state, err := agentstate.Open(configuration.StateFile) + if err != nil { + t.Fatal(err) + } + queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge) + if err != nil { + t.Fatal(err) + } + sender := &fakeSender{batchDigests: map[uint64]string{1: strings.Repeat("a", 64)}} + runner, err := New(configuration, "source", stateStore, state, queue, sender) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC) + if err = runner.RunOnce(context.Background(), now); err == nil || !strings.Contains(err.Error(), "acknowledgement does not match exact batch") { + t.Fatalf("expected exact acknowledgement rejection, got %v", err) + } + entries, err := queue.List(now) + if err != nil || len(entries) != 1 { + t.Fatalf("entries=%+v err=%v", entries, err) + } +} + +func TestRunnerDeliversLocallyEvaluatedAlertAfterRawBatchAcknowledgement(t *testing.T) { + root := t.TempDir() + logPath := filepath.Join(root, "request.jsonl") + if err := os.WriteFile(logPath, []byte(requestLineWithStatus("failed", 503)+"\n"), 0o640); err != nil { + t.Fatal(err) + } + spoolRoot := filepath.Join(root, "spool") + ast, err := query.Parse("logs | where status >= 500 | limit 10", model.MaxRecords) + if err != nil { + t.Fatal(err) + } + configuration := config.Agent{BatchRecords: 10, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}, AlertRules: []config.AgentAlertRule{{Version: 1, ID: "http-failures", Revision: 1, StreamID: "request", Query: "logs | where status >= 500 | limit 10", MinimumMatches: 1, AST: ast}}} + stateStore, state, err := agentstate.Open(configuration.StateFile) + if err != nil { + t.Fatal(err) + } + queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge) + if err != nil { + t.Fatal(err) + } + sender := &fakeSender{} + runner, err := New(configuration, "source", stateStore, state, queue, sender) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC) + if err = runner.RunOnce(context.Background(), now); err != nil { + t.Fatal(err) + } + if len(sender.sent) != 1 || len(sender.alerts) != 1 || sender.alerts[0].State != "matched" || sender.alerts[0].Sequence != 1 || sender.alerts[0].SegmentDigest != strings.Repeat("d", 64) || len(sender.alerts[0].AgentEpoch) != 32 { + t.Fatalf("sent=%v alerts=%+v", sender.sent, sender.alerts) + } + if entries, listErr := queue.List(now); listErr != nil || len(entries) != 0 { + t.Fatalf("entries=%+v err=%v", entries, listErr) + } +} + +func TestRunnerRetainsRawBatchUntilAlertTransitionIsAcknowledged(t *testing.T) { + root := t.TempDir() + logPath := filepath.Join(root, "request.jsonl") + if err := os.WriteFile(logPath, []byte(requestLineWithStatus("failed", 503)+"\n"), 0o640); err != nil { + t.Fatal(err) + } + spoolRoot := filepath.Join(root, "spool") + ast, err := query.Parse("logs | where status >= 500 | limit 10", model.MaxRecords) + if err != nil { + t.Fatal(err) + } + configuration := config.Agent{BatchRecords: 10, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}, AlertRules: []config.AgentAlertRule{{Version: 1, ID: "http-failures", Revision: 1, StreamID: "request", Query: "logs | where status >= 500 | limit 10", MinimumMatches: 1, AST: ast}}} + stateStore, state, err := agentstate.Open(configuration.StateFile) + if err != nil { + t.Fatal(err) + } + queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge) + if err != nil { + t.Fatal(err) + } + sender := &fakeSender{failAlerts: true} + runner, err := New(configuration, "source", stateStore, state, queue, sender) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC) + if err = runner.RunOnce(context.Background(), now); err == nil || !strings.Contains(err.Error(), "alert endpoint unavailable") { + t.Fatalf("err=%v", err) + } + if entries, listErr := queue.List(now); listErr != nil || len(entries) != 1 { + t.Fatalf("entries=%+v err=%v", entries, listErr) + } + sender.failAlerts = false + if err = runner.RunOnce(context.Background(), now.Add(time.Second)); err != nil { + t.Fatal(err) + } + if len(sender.sent) != 2 || len(sender.alerts) != 1 { + t.Fatalf("sent=%v alerts=%+v", sender.sent, sender.alerts) + } +} + +func TestSourceIDParsingKeepsDotsInsideIdentifier(t *testing.T) { + id, err := sourceIDFromCredential("obs1.node.example.abcdef") + if err != nil || id != "node.example" { + t.Fatalf("id=%q err=%v", id, err) + } + for _, invalid := range []string{"token", "obs1..secret", "obs1.bad value.secret"} { + if _, err = sourceIDFromCredential(invalid); err == nil { + t.Fatalf("accepted %q", invalid) + } + } +} + +func TestRunnerSpoolsLinuxMetricsWhileServerIsOffline(t *testing.T) { + root := t.TempDir() + proc := filepath.Join(root, "proc") + if err := os.MkdirAll(filepath.Join(proc, "net"), 0o700); err != nil { + t.Fatal(err) + } + files := map[string]string{ + "stat": "cpu 1 2 3 4\n", "uptime": "10 5\n", + "meminfo": "MemTotal: 10 kB\nMemAvailable: 8 kB\nSwapTotal: 2 kB\nSwapFree: 1 kB\n", + "loadavg": "0.1 0.2 0.3 1/1 1\n", filepath.Join("net", "dev"): "Inter-| Receive | Transmit\n", + } + for name, body := range files { + if err := os.WriteFile(filepath.Join(proc, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + spoolRoot := filepath.Join(root, "spool") + configuration := config.Agent{BatchRecords: 3, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "linux_metrics", StreamID: "host-metrics", LinuxMetrics: &hostmetrics.Config{ProcRoot: proc}}}} + stateStore, state, err := agentstate.Open(configuration.StateFile) + if err != nil { + t.Fatal(err) + } + queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge) + if err != nil { + t.Fatal(err) + } + runner, err := New(configuration, "source", stateStore, state, queue, &fakeSender{fail: true}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 6, 0, 0, 0, time.UTC) + if err = runner.RunOnce(context.Background(), now); err == nil { + t.Fatal("offline delivery unexpectedly succeeded") + } + entries, err := queue.List(now) + if err != nil || len(entries) < 2 { + t.Fatalf("entries=%+v err=%v", entries, err) + } + for index, entry := range entries { + batch, _, readErr := queue.ReadWithCheckpoint(entry) + if readErr != nil { + t.Fatal(readErr) + } + if batch.Signal != model.SignalMetrics || batch.StreamID != "host-metrics" || len(batch.Records) == 0 || len(batch.Records) > configuration.BatchRecords || batch.Sequence != uint64(index+1) { + t.Fatalf("batch=%+v", batch) + } + } +} + +func requestLine(route string) string { + return `{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/` + route + `","status":200,"bytes":12,"duration_ns":1000,"request_id":"request-1"}` +} + +func requestLineWithStatus(route string, status int) string { + return fmt.Sprintf(`{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/%s","status":%d,"bytes":12,"duration_ns":1000,"request_id":"request-1"}`, route, status) +} diff --git a/internal/agentclient/client.go b/internal/agentclient/client.go new file mode 100644 index 0000000..b8effe9 --- /dev/null +++ b/internal/agentclient/client.go @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package agentclient + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/nativeprotocol" + "gamertan.com/observatory/internal/storage" +) + +type Client struct { + endpoint string + alertEndpoint string + credential string + sourceID string + http *http.Client +} + +type EnrollmentResult struct { + SourceID string `json:"source_id"` + Credential string `json:"credential"` +} + +func Enroll(ctx context.Context, serverURL, enrollmentToken string, transport http.RoundTripper) (EnrollmentResult, error) { + u, err := url.Parse(serverURL) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" { + return EnrollmentResult{}, errors.New("server URL must be an absolute HTTPS origin") + } + if len(enrollmentToken) != len("obse1.")+64 || !strings.HasPrefix(enrollmentToken, "obse1.") || strings.ContainsAny(enrollmentToken, " \t\r\n") { + return EnrollmentResult{}, errors.New("invalid enrollment token") + } + if transport == nil { + transport = http.DefaultTransport + } + client := &http.Client{Transport: transport, Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect refused") }} + request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimSuffix(serverURL, "/")+"/api/v1/agent/enroll", http.NoBody) + if err != nil { + return EnrollmentResult{}, errors.New("create enrollment request") + } + request.Header.Set("Authorization", "Bearer "+enrollmentToken) + request.Header.Set("Accept", "application/json") + response, err := client.Do(request) + if err != nil { + return EnrollmentResult{}, errors.New("enrollment request failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusCreated { + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 64<<10)) + return EnrollmentResult{}, fmt.Errorf("enrollment returned HTTP %d", response.StatusCode) + } + decoder := json.NewDecoder(io.LimitReader(response.Body, 64<<10)) + decoder.DisallowUnknownFields() + var result EnrollmentResult + if err = decoder.Decode(&result); err != nil { + return EnrollmentResult{}, errors.New("invalid enrollment response") + } + if err = decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return EnrollmentResult{}, errors.New("enrollment response has trailing data") + } + if err = validateCredential(result.SourceID, result.Credential); err != nil { + return EnrollmentResult{}, err + } + return result, nil +} + +func RevokeSource(ctx context.Context, serverURL, credential string, transport http.RoundTripper) error { + u, err := url.Parse(serverURL) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" { + return errors.New("server URL must be an absolute HTTPS origin") + } + if len(credential) < 48 || len(credential) > 512 || !strings.HasPrefix(credential, "obs1.") || strings.ContainsAny(credential, " \t\r\n") { + return errors.New("invalid source credential") + } + if transport == nil { + transport = http.DefaultTransport + } + client := &http.Client{Transport: transport, Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect refused") }} + request, err := http.NewRequestWithContext(ctx, http.MethodDelete, strings.TrimSuffix(serverURL, "/")+"/api/v1/agent/source", http.NoBody) + if err != nil { + return errors.New("create source revocation request") + } + request.Header.Set("Authorization", "Bearer "+credential) + response, err := client.Do(request) + if err != nil { + return errors.New("source revocation request failed") + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 64<<10)) + if response.StatusCode != http.StatusNoContent { + return fmt.Errorf("source revocation returned HTTP %d", response.StatusCode) + } + return nil +} + +func New(serverURL, credential string, transport http.RoundTripper) (*Client, error) { + u, err := url.Parse(serverURL) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" { + return nil, errors.New("server URL must be an absolute HTTPS origin") + } + sourceID, credentialErr := credentialSourceID(credential) + if credentialErr != nil { + return nil, errors.New("invalid source credential") + } + if transport == nil { + transport = http.DefaultTransport + } + httpClient := &http.Client{Transport: transport, Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect refused") }} + base := strings.TrimSuffix(serverURL, "/") + return &Client{endpoint: base + "/api/v2/ingest/native", alertEndpoint: base + "/api/v1/agent/alert-transition", credential: credential, sourceID: sourceID, http: httpClient}, nil +} + +func (c *Client) SendAlertTransition(ctx context.Context, transition model.AlertTransition) (storage.SourceAlertTransitionAck, error) { + b, err := json.Marshal(transition) + if err != nil { + return storage.SourceAlertTransitionAck{}, errors.New("encode source alert transition") + } + expectedDigest, err := transition.Digest() + if err != nil { + return storage.SourceAlertTransitionAck{}, err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.alertEndpoint, bytes.NewReader(b)) + if err != nil { + return storage.SourceAlertTransitionAck{}, errors.New("create source alert transition request") + } + request.Header.Set("Authorization", "Bearer "+c.credential) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Accept", "application/json") + response, err := c.http.Do(request) + if err != nil { + return storage.SourceAlertTransitionAck{}, errors.New("source alert transition request failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusAccepted { + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 64<<10)) + return storage.SourceAlertTransitionAck{}, fmt.Errorf("source alert transition returned HTTP %d", response.StatusCode) + } + decoder := json.NewDecoder(io.LimitReader(response.Body, 64<<10)) + decoder.DisallowUnknownFields() + var ack storage.SourceAlertTransitionAck + if err = decoder.Decode(&ack); err != nil { + return storage.SourceAlertTransitionAck{}, errors.New("invalid source alert transition acknowledgement") + } + if err = decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return storage.SourceAlertTransitionAck{}, errors.New("source alert transition acknowledgement has trailing data") + } + if ack.SourceID != c.sourceID || ack.RuleID != transition.RuleID || ack.RuleRevision != transition.RuleRevision || ack.AgentEpoch != transition.AgentEpoch || ack.Sequence != transition.Sequence || ack.Digest != expectedDigest { + return storage.SourceAlertTransitionAck{}, errors.New("source alert transition acknowledgement does not match transition") + } + return ack, nil +} + +func credentialSourceID(credential string) (string, error) { + if len(credential) < 48 || len(credential) > 512 || !strings.HasPrefix(credential, "obs1.") || strings.ContainsAny(credential, " \t\r\n") { + return "", errors.New("invalid source credential") + } + remainder := strings.TrimPrefix(credential, "obs1.") + separator := strings.LastIndexByte(remainder, '.') + if separator < 1 || separator == len(remainder)-1 { + return "", errors.New("invalid source credential") + } + sourceID := remainder[:separator] + if model.ValidateSourceID(sourceID) != nil { + return "", errors.New("invalid source credential") + } + return sourceID, nil +} + +func validateCredential(sourceID, credential string) error { + if model.ValidateSourceID(sourceID) != nil || len(credential) < 48 || len(credential) > 512 || !strings.HasPrefix(credential, "obs1."+sourceID+".") || strings.ContainsAny(credential, " \t\r\n") { + return errors.New("invalid source credential") + } + return nil +} + +func (c *Client) Send(ctx context.Context, batch model.Batch) (storage.Ack, error) { + b, err := json.Marshal(batch) + if err != nil { + return storage.Ack{}, errors.New("encode native batch") + } + envelope, err := batch.Envelope(b) + if err != nil { + return storage.Ack{}, errors.New("encode native batch envelope") + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(b)) + if err != nil { + return storage.Ack{}, errors.New("create ingestion request") + } + request.Header.Set("Authorization", "Bearer "+c.credential) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Accept", "application/json") + nativeprotocol.SetHeaders(request.Header, envelope) + response, err := c.http.Do(request) + if err != nil { + return storage.Ack{}, errors.New("ingestion request failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusAccepted { + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 64<<10)) + return storage.Ack{}, fmt.Errorf("ingestion returned HTTP %d", response.StatusCode) + } + decoder := json.NewDecoder(io.LimitReader(response.Body, 64<<10)) + decoder.DisallowUnknownFields() + var ack storage.Ack + if err := decoder.Decode(&ack); err != nil { + return storage.Ack{}, errors.New("invalid ingestion acknowledgement") + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return storage.Ack{}, errors.New("ingestion acknowledgement has trailing data") + } + decodedDigest, digestErr := hex.DecodeString(ack.Digest) + decodedBatchDigest, batchDigestErr := hex.DecodeString(ack.BatchDigest) + if ack.SourceID != batch.SourceID || ack.StreamID != batch.StreamID || ack.Sequence != batch.Sequence || digestErr != nil || len(decodedDigest) != sha256.Size || batchDigestErr != nil || len(decodedBatchDigest) != sha256.Size || ack.BatchDigest != envelope.BatchDigest { + return storage.Ack{}, errors.New("ingestion acknowledgement does not match batch") + } + return ack, nil +} diff --git a/internal/agentclient/client_test.go b/internal/agentclient/client_test.go new file mode 100644 index 0000000..d9977e8 --- /dev/null +++ b/internal/agentclient/client_test.go @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package agentclient + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/httpserver" + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/nativeprotocol" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/storage" +) + +type transportFunc func(*http.Request) (*http.Response, error) + +func (f transportFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) } + +func TestSendRequiresMatchingAcknowledgementAndDoesNotRedirect(t *testing.T) { + credential := "obs1.source." + strings.Repeat("a", 64) + var authorization, path string + client, err := New("https://observatory.example", credential, transportFunc(func(request *http.Request) (*http.Response, error) { + authorization = request.Header.Get("Authorization") + path = request.URL.Path + body, readErr := io.ReadAll(request.Body) + if readErr != nil { + t.Fatal(readErr) + } + digest := sha256.Sum256(body) + if request.Header.Get(nativeprotocol.WireDigestHeader) != hex.EncodeToString(digest[:]) || request.Header.Get(nativeprotocol.StreamHeader) != "access" || request.Header.Get(nativeprotocol.SequenceHeader) != "1" { + t.Fatalf("missing framed batch metadata: %v", request.Header) + } + response := fmt.Sprintf(`{"source_id":"source","stream_id":"access","sequence":1,"digest":"%s","batch_digest":"%s","duplicate":false}`, strings.Repeat("a", 64), hex.EncodeToString(digest[:])) + return &http.Response{StatusCode: http.StatusAccepted, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(response))}, nil + })) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}} + if _, err := client.Send(context.Background(), batch); err != nil { + t.Fatal(err) + } + if authorization != "Bearer "+credential || path != "/api/v2/ingest/native" { + t.Fatalf("authorization=%q path=%q", authorization, path) + } +} + +func TestSendAcceptsRealServerBatchDigestRatherThanPrivateSegmentDigest(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + identities, err := identity.Open(root) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + token, err := store.CreateSource(context.Background(), "source", model.Scope{OrganizationID: "organization", ProjectID: "project", EnvironmentID: "production", ServiceID: "service"}) + if err != nil { + t.Fatal(err) + } + server, err := httpserver.New(store, identities, httpserver.Options{ + PublicOrigin: "https://observatory.example", MaxBodyBytes: 1 << 20, MaxQueryRows: 100, + QueryBudget: query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, SessionLifetime: time.Hour, + }) + if err != nil { + t.Fatal(err) + } + client, err := New("https://observatory.example", token, transportFunc(func(request *http.Request) (*http.Response, error) { + request.RemoteAddr = "127.0.0.1:40000" + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + return response.Result(), nil + })) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}} + ack, err := client.Send(context.Background(), batch) + if err != nil { + t.Fatal(err) + } + expected, err := batch.Digest() + if err != nil { + t.Fatal(err) + } + if ack.BatchDigest != expected || ack.Digest == ack.BatchDigest { + t.Fatalf("ack=%+v expected batch digest=%s", ack, expected) + } +} + +func TestSendRejectsMismatchedAcknowledgementWithoutEchoingBody(t *testing.T) { + credential := "obs1.source." + strings.Repeat("a", 64) + client, err := New("https://observatory.example", credential, transportFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusAccepted, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"source_id":"source","stream_id":"access","sequence":1,"digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","batch_digest":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}`))}, nil + })) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request", Body: "do-not-echo"}}} + _, err = client.Send(context.Background(), batch) + if err == nil || strings.Contains(err.Error(), "do-not-echo") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestSendAlertTransitionUsesExactEndpointAndRejectsMismatchedAcknowledgement(t *testing.T) { + credential := "obs1.source." + strings.Repeat("a", 64) + now := time.Date(2026, 8, 18, 23, 50, 0, 0, time.UTC) + transition := model.AlertTransition{Version: model.AlertTransitionVersion, RuleID: "rule-a", RuleRevision: 2, AgentEpoch: strings.Repeat("b", 32), Sequence: 3, StreamID: "requests", BatchSequence: 8, SegmentDigest: strings.Repeat("c", 64), WindowStart: now.Add(-time.Minute), WindowEnd: now, State: "matched", ObservedAt: now} + digest, err := transition.Digest() + if err != nil { + t.Fatal(err) + } + var path, authorization string + client, err := New("https://observatory.example", credential, transportFunc(func(request *http.Request) (*http.Response, error) { + path = request.URL.Path + authorization = request.Header.Get("Authorization") + body, readErr := io.ReadAll(request.Body) + if readErr != nil || !strings.Contains(string(body), `"segment_digest":"`+transition.SegmentDigest+`"`) { + t.Fatalf("body=%q err=%v", string(body), readErr) + } + response := fmt.Sprintf(`{"source_id":"source","rule_id":"rule-a","rule_revision":2,"agent_epoch":"%s","sequence":3,"digest":"%s","duplicate":false}`, transition.AgentEpoch, digest) + return &http.Response{StatusCode: http.StatusAccepted, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(response))}, nil + })) + if err != nil { + t.Fatal(err) + } + ack, err := client.SendAlertTransition(context.Background(), transition) + if err != nil || ack.Digest != digest || path != "/api/v1/agent/alert-transition" || authorization != "Bearer "+credential { + t.Fatalf("ack=%+v path=%q authorization=%q err=%v", ack, path, authorization, err) + } + + mismatch, err := New("https://observatory.example", credential, transportFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusAccepted, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(fmt.Sprintf(`{"source_id":"source","rule_id":"rule-a","rule_revision":2,"agent_epoch":"%s","sequence":4,"digest":"%s","duplicate":false}`, transition.AgentEpoch, digest)))}, nil + })) + if err != nil { + t.Fatal(err) + } + if _, err = mismatch.SendAlertTransition(context.Background(), transition); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("mismatch err=%v", err) + } + wrongSource, err := New("https://observatory.example", credential, transportFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusAccepted, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(fmt.Sprintf(`{"source_id":"other","rule_id":"rule-a","rule_revision":2,"agent_epoch":"%s","sequence":3,"digest":"%s","duplicate":false}`, transition.AgentEpoch, digest)))}, nil + })) + if err != nil { + t.Fatal(err) + } + if _, err = wrongSource.SendAlertTransition(context.Background(), transition); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("wrong source err=%v", err) + } +} + +func TestEnrollAndRevokeUseExactHTTPSEndpoints(t *testing.T) { + enrollment := "obse1." + strings.Repeat("e", 64) + credential := "obs1.source." + strings.Repeat("a", 64) + var methods []string + transport := transportFunc(func(request *http.Request) (*http.Response, error) { + methods = append(methods, request.Method+" "+request.URL.Path) + switch request.URL.Path { + case "/api/v1/agent/enroll": + if request.Header.Get("Authorization") != "Bearer "+enrollment { + t.Fatal("enrollment authorization missing") + } + return &http.Response{StatusCode: http.StatusCreated, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"source_id":"source","credential":"` + credential + `"}`))}, nil + case "/api/v1/agent/source": + if request.Header.Get("Authorization") != "Bearer "+credential { + t.Fatal("source authorization missing") + } + return &http.Response{StatusCode: http.StatusNoContent, Header: make(http.Header), Body: http.NoBody}, nil + default: + t.Fatalf("unexpected endpoint %s", request.URL.Path) + return nil, nil + } + }) + result, err := Enroll(context.Background(), "https://observatory.example", enrollment, transport) + if err != nil || result.SourceID != "source" || result.Credential != credential { + t.Fatalf("result=%+v err=%v", result, err) + } + if err = RevokeSource(context.Background(), "https://observatory.example", credential, transport); err != nil { + t.Fatal(err) + } + if len(methods) != 2 || methods[0] != "POST /api/v1/agent/enroll" || methods[1] != "DELETE /api/v1/agent/source" { + t.Fatalf("methods=%v", methods) + } +} diff --git a/internal/agentstate/state.go b/internal/agentstate/state.go new file mode 100644 index 0000000..1eff23a --- /dev/null +++ b/internal/agentstate/state.go @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package agentstate + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +const Version = 1 + +type Cursor struct { + Device uint64 `json:"device"` + Inode uint64 `json:"inode"` + Offset int64 `json:"offset"` + Sequence uint64 `json:"sequence"` + DiscardingLine bool `json:"discarding_line"` + DroppedRecords uint64 `json:"dropped_records"` + Discontinuities uint64 `json:"discontinuities"` +} + +type State struct { + Version int `json:"version"` + Streams map[string]Cursor `json:"streams"` +} + +type Store struct{ path string } + +func Open(path string) (*Store, State, error) { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return nil, State{}, errors.New("agent state path must be absolute and clean") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, State{}, fmt.Errorf("create agent state directory: %w", err) + } + parent, err := os.Lstat(filepath.Dir(path)) + if err != nil || !parent.IsDir() || parent.Mode()&os.ModeSymlink != 0 || parent.Mode().Perm()&0o077 != 0 { + return nil, State{}, errors.New("agent state directory must be private and must not be a symlink") + } + store := &Store{path: path} + state, err := store.Load() + return store, state, err +} + +func (store *Store) Load() (State, error) { + info, err := os.Lstat(store.path) + if errors.Is(err, os.ErrNotExist) { + return State{Version: Version, Streams: map[string]Cursor{}}, nil + } + if err != nil { + return State{}, fmt.Errorf("inspect agent state: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm() != 0o600 { + return State{}, errors.New("agent state must be a mode-0600 regular non-symlink file") + } + body, err := os.ReadFile(store.path) + if err != nil { + return State{}, fmt.Errorf("read agent state: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + var state State + if err = decoder.Decode(&state); err != nil { + return State{}, errors.New("decode agent state") + } + if err = decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return State{}, errors.New("agent state contains trailing data") + } + if err = state.Validate(); err != nil { + return State{}, err + } + return state, nil +} + +func (store *Store) Save(state State) error { + if err := state.Validate(); err != nil { + return err + } + body, err := json.Marshal(state) + if err != nil { + return err + } + body = append(body, '\n') + dir := filepath.Dir(store.path) + temporary, err := os.CreateTemp(dir, ".state-*") + if err != nil { + return err + } + name := temporary.Name() + defer os.Remove(name) + if err = temporary.Chmod(0o600); err == nil { + _, err = temporary.Write(body) + } + if err == nil { + err = temporary.Sync() + } + if closeErr := temporary.Close(); err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("write agent state: %w", err) + } + if existing, inspectErr := os.Lstat(store.path); inspectErr == nil { + if !existing.Mode().IsRegular() || existing.Mode()&os.ModeSymlink != 0 { + return errors.New("agent state destination is not a regular file") + } + } else if !errors.Is(inspectErr, os.ErrNotExist) { + return inspectErr + } + if err = os.Rename(name, store.path); err != nil { + return err + } + directory, err := os.Open(dir) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} + +func (state State) Validate() error { + if state.Version != Version || state.Streams == nil || len(state.Streams) > 64 { + return errors.New("agent state identity is invalid") + } + for stream, cursor := range state.Streams { + if !safeID(stream) || cursor.Offset < 0 || (cursor.Device == 0) != (cursor.Inode == 0) { + return errors.New("agent state cursor is invalid") + } + } + return nil +} + +func safeID(value string) bool { + if value == "" || len(value) > 128 { + return false + } + for _, character := range value { + if !(character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || strings.ContainsRune("._-", character)) { + return false + } + } + return true +} diff --git a/internal/agentstate/state_test.go b/internal/agentstate/state_test.go new file mode 100644 index 0000000..3e0d3bc --- /dev/null +++ b/internal/agentstate/state_test.go @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package agentstate + +import ( + "os" + "path/filepath" + "testing" +) + +func TestStateAtomicRoundTripAndSymlinkRefusal(t *testing.T) { + root := filepath.Join(t.TempDir(), "state") + path := filepath.Join(root, "state.json") + store, state, err := Open(path) + if err != nil { + t.Fatal(err) + } + state.Streams["access"] = Cursor{Device: 1, Inode: 2, Offset: 99, Sequence: 3, DroppedRecords: 4} + if err = store.Save(state); err != nil { + t.Fatal(err) + } + _, loaded, err := Open(path) + if err != nil || loaded.Streams["access"] != state.Streams["access"] { + t.Fatalf("loaded=%+v err=%v", loaded, err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("state info=%v err=%v", info, err) + } + if err = os.Remove(path); err != nil { + t.Fatal(err) + } + outside := filepath.Join(t.TempDir(), "outside") + if err = os.WriteFile(outside, []byte(`{"version":1,"streams":{}}`), 0o600); err != nil { + t.Fatal(err) + } + if err = os.Symlink(outside, path); err != nil { + t.Skip(err) + } + if _, _, err = Open(path); err == nil { + t.Fatal("symlinked state accepted") + } +} diff --git a/internal/browsertest/main.go b/internal/browsertest/main.go new file mode 100644 index 0000000..96b952f --- /dev/null +++ b/internal/browsertest/main.go @@ -0,0 +1,253 @@ +//go:build observatory_browser_fixture + +// SPDX-License-Identifier: AGPL-3.0-only + +// Command browsertest serves a disposable HTTPS Observatory instance for the +// real-browser verification campaign. It is excluded from ordinary builds. +package main + +import ( + "bytes" + "context" + "crypto/ecdh" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "gamertan.com/observatory/internal/httpserver" + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/storage" +) + +const ( + fixtureUsername = "browser-operator" + fixturePassword = "browser-fixture-password" +) + +type pushDispatcher struct{} + +func (pushDispatcher) Enqueue(string) bool { return true } + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "browser fixture failed:", err) + os.Exit(1) + } +} + +func run() error { + root, err := os.MkdirTemp("", "observatory-browser-fixture-") + if err != nil { + return errors.New("create browser fixture root") + } + defer os.RemoveAll(root) + if err = os.Chmod(root, 0o700); err != nil { + return errors.New("protect browser fixture root") + } + + certificate, spki, err := localCertificate() + if err != nil { + return err + } + listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS13}) + if err != nil { + return errors.New("listen for browser fixture") + } + defer listener.Close() + _, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + return errors.New("read browser fixture listener") + } + origin := "https://localhost:" + port + + store, err := storage.Open(filepath.Join(root, "data")) + if err != nil { + return err + } + defer store.Close() + identities, err := identity.Open(filepath.Join(root, "data")) + if err != nil { + return err + } + defer identities.Close() + bootstrap, err := identities.Bootstrap(context.Background(), identity.BootstrapInput{ + Username: fixtureUsername, Email: "browser@example.test", DisplayName: "Browser Operator", Password: fixturePassword, + }) + if err != nil { + return err + } + + now := time.Now().UTC() + scope := model.Scope{OrganizationID: bootstrap.Organization.ID, ProjectID: "browser-project", EnvironmentID: "test", ServiceID: "browser-service"} + token, err := store.CreateSource(context.Background(), "browser-source", scope) + if err != nil { + return err + } + if _, err = store.Ingest(context.Background(), token, logBatch(1, now), now); err != nil { + return err + } + if err = store.Recover(context.Background()); err != nil { + return err + } + saved, err := store.SaveQuery(context.Background(), storage.SavedQueryInput{ + OrganizationID: scope.OrganizationID, Name: "Browser fixture evidence", Description: "Exact browser-campaign evidence.", + Query: "logs | window 1h | limit 10", ActorUserID: bootstrap.User.ID, MaxRows: 100, + }, now) + if err != nil { + return err + } + if _, err = store.SaveAlertRule(context.Background(), storage.AlertRuleInput{ + OrganizationID: scope.OrganizationID, Name: "Browser fixture incident", Description: "Exercises the private offline incident boundary.", + SavedQueryID: saved.ID, Severity: "critical", MinimumMatches: 1, RequiredConsecutive: 1, + EvaluationInterval: 15 * time.Second, Enabled: true, ActorUserID: bootstrap.User.ID, + }, now); err != nil { + return err + } + + vapid, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + return errors.New("create browser fixture Web Push key") + } + server, err := httpserver.New(store, identities, httpserver.Options{ + PublicOrigin: origin, MaxBodyBytes: 1 << 20, MaxQueryRows: 100, + QueryBudget: query.Budget{MaxDuration: 2 * time.Second, MaxRows: 100, MaxScannedBytes: 32 << 20, MaxMemoryBytes: 16 << 20}, + SessionLifetime: time.Hour, PushPublicKey: base64.RawURLEncoding.EncodeToString(vapid.PublicKey().Bytes()), PushDispatcher: pushDispatcher{}, + }) + if err != nil { + return err + } + if _, err = server.EvaluateAlerts(context.Background()); err != nil { + return err + } + handler := browserFixture(server.Handler()) + httpServer := &http.Server{Handler: handler, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 30 * time.Second} + serveErrors := make(chan error, 1) + go func() { serveErrors <- httpServer.Serve(listener) }() + + updatesDone := make(chan struct{}) + defer close(updatesDone) + go publishUpdates(handler, origin, token, updatesDone) + + fmt.Println(origin) + fmt.Println(spki) + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(signals) + select { + case <-signals: + shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return httpServer.Shutdown(shutdown) + case serveErr := <-serveErrors: + if errors.Is(serveErr, http.ErrServerClosed) { + return nil + } + return serveErr + } +} + +// browserFixture bounds the fixture EventSource connection so the browser +// campaign can prove native reconnection. It does not alter request origin +// metadata; form submissions exercise the production token-bound policy. +func browserFixture(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/app/events" { + ctx, cancel := context.WithTimeout(r.Context(), 1250*time.Millisecond) + defer cancel() + r = r.WithContext(ctx) + } + next.ServeHTTP(w, r) + }) +} + +func logBatch(sequence uint64, observed time.Time) model.Batch { + return model.Batch{Version: model.BatchVersion, SourceID: "browser-source", StreamID: "browser-stream", Sequence: sequence, ObservedAt: observed, Signal: model.SignalLogs, Records: []model.Observation{{ + Timestamp: observed, Name: "browser.fixture", Severity: "information", Body: "bounded browser fixture observation", + Attributes: map[string]string{"http.route": "/browser-fixture", "http.status_code": "200"}, + }}} +} + +func publishUpdates(handler http.Handler, origin, token string, done <-chan struct{}) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + sequence := uint64(2) + for { + select { + case <-done: + return + case observed := <-ticker.C: + body, err := json.Marshal(logBatch(sequence, observed.UTC())) + if err != nil { + return + } + request := httptest.NewRequest(http.MethodPost, origin+"/api/v1/ingest/native", bytes.NewReader(body)) + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code == http.StatusAccepted { + sequence++ + } + } + } +} + +func localCertificate() (tls.Certificate, string, error) { + private, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, "", errors.New("create local TLS key") + } + maximum := new(big.Int).Lsh(big.NewInt(1), 128) + serial, err := rand.Int(rand.Reader, maximum) + if err != nil { + return tls.Certificate{}, "", errors.New("create local TLS serial") + } + now := time.Now().UTC() + template := x509.Certificate{ + SerialNumber: serial, Subject: pkix.Name{CommonName: "Observatory browser fixture"}, + NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"localhost"}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &private.PublicKey, private) + if err != nil { + return tls.Certificate{}, "", errors.New("create local TLS certificate") + } + encodedKey, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + return tls.Certificate{}, "", errors.New("encode local TLS key") + } + encodedPublic, err := x509.MarshalPKIXPublicKey(&private.PublicKey) + if err != nil { + return tls.Certificate{}, "", errors.New("encode local TLS public key") + } + digest := sha256.Sum256(encodedPublic) + certificate, err := tls.X509KeyPair( + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: encodedKey}), + ) + if err != nil { + return tls.Certificate{}, "", err + } + return certificate, base64.StdEncoding.EncodeToString(digest[:]), nil +} diff --git a/internal/capacitytest/main.go b/internal/capacitytest/main.go new file mode 100644 index 0000000..407cf96 --- /dev/null +++ b/internal/capacitytest/main.go @@ -0,0 +1,920 @@ +//go:build observatory_capacity_fixture && linux + +// SPDX-License-Identifier: AGPL-3.0-only + +// Command capacitytest runs the bounded Observatory capacity and recovery +// campaign. It is excluded from ordinary builds and emits only aggregate, +// synthetic evidence. +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "flag" + "fmt" + "math" + "net/url" + "os" + "os/signal" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/spool" + "gamertan.com/observatory/internal/storage" +) + +const ( + reportVersion = 2 + visibilityObservationTimeout = 30 * time.Second +) + +type settings struct { + sustainRate int + sustainDuration time.Duration + burstRate int + burstDuration time.Duration + minimumPrimary int64 + queryIterations int + organizations int + primaryWeight int + batchInterval time.Duration + requireCgroup bool + expectedCPUs int + expectedMemory int64 +} + +type phaseReport struct { + TargetRate int `json:"target_rate_per_second"` + DurationSeconds float64 `json:"target_duration_seconds"` + Observations int64 `json:"observations"` + ElapsedSeconds float64 `json:"elapsed_seconds"` + AchievedRate float64 `json:"achieved_rate_per_second"` + IngestP50Milliseconds float64 `json:"ingest_p50_milliseconds"` + IngestP95Milliseconds float64 `json:"ingest_p95_milliseconds"` + IngestP99Milliseconds float64 `json:"ingest_p99_milliseconds"` + VisibleP50Milliseconds float64 `json:"visible_p50_milliseconds"` + VisibleP95Milliseconds float64 `json:"visible_p95_milliseconds"` + VisibleP99Milliseconds float64 `json:"visible_p99_milliseconds"` +} + +type queryReport struct { + Name string `json:"name"` + Iterations int `json:"iterations"` + P50Milliseconds float64 `json:"p50_milliseconds"` + P95Milliseconds float64 `json:"p95_milliseconds"` + P99Milliseconds float64 `json:"p99_milliseconds"` + MaximumScannedRows int64 `json:"maximum_scanned_rows"` + MaximumScannedBytes int64 `json:"maximum_scanned_bytes"` +} + +type spoolReport struct { + Batches int `json:"batches"` + Observations int64 `json:"observations"` + OldestAgeHours float64 `json:"oldest_age_hours"` + Replayed int64 `json:"replayed_observations"` + RemainingAfterAck int `json:"remaining_after_ack"` + DuplicateRecognized bool `json:"duplicate_recognized"` +} + +type retentionEvidence struct { + ArchivedSegments int `json:"archived_segments"` + ArchivedBytes int64 `json:"archived_bytes"` + RemovedSegments int `json:"removed_segments"` + RemovedBytes int64 `json:"removed_bytes"` + ProjectionRowsRemoved int64 `json:"projection_rows_removed"` + ColdQueryRows int `json:"cold_query_rows"` +} + +type resourceReport struct { + GOMAXPROCS int `json:"gomaxprocs"` + CPUQuota float64 `json:"cpu_quota"` + MemoryLimitBytes int64 `json:"memory_limit_bytes"` + MaximumRSSBytes int64 `json:"maximum_rss_bytes"` + DatasetBytes int64 `json:"dataset_bytes"` +} + +type projectionDrainReport struct { + PendingSegments int `json:"pending_segments_at_start"` + PendingBytes int64 `json:"pending_bytes_at_start"` + OldestLagSeconds float64 `json:"oldest_pending_lag_seconds_at_start"` + ElapsedSeconds float64 `json:"elapsed_seconds"` +} + +type storageClass struct { + Files int `json:"files"` + Bytes int64 `json:"bytes"` +} + +type storageBreakdown struct { + Raw storageClass `json:"raw_segments"` + Projection storageClass `json:"projection_sqlite"` + Control storageClass `json:"control_sqlite"` + Other storageClass `json:"other"` + Total storageClass `json:"total"` + SQLitePageClasses sqlitePageBreakdown `json:"primary_projection_sqlite_page_classes"` + SQLiteBytes map[string]int64 `json:"primary_projection_sqlite_objects"` +} + +type sqlitePageBreakdown struct { + Tables int64 `json:"tables_bytes"` + Indexes int64 `json:"indexes_bytes"` + Internal int64 `json:"internal_bytes"` + Total int64 `json:"total_bytes"` +} + +type campaignReport struct { + Version int `json:"version"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + Organizations int `json:"organizations"` + PrimaryPhaseWeight int `json:"primary_phase_weight"` + TotalObservations int64 `json:"total_observations"` + PrimaryObservations int64 `json:"primary_observations"` + Sustain phaseReport `json:"sustain"` + Burst phaseReport `json:"burst"` + FillSeconds float64 `json:"fill_seconds"` + ProjectionDrain projectionDrainReport `json:"projection_drain"` + PrimaryStorage storageBreakdown `json:"primary_storage"` + Queries []queryReport `json:"queries"` + Spool spoolReport `json:"spool"` + Retention retentionEvidence `json:"retention"` + Resources resourceReport `json:"resources"` + Pass bool `json:"pass"` +} + +type sourceState struct { + index int + weight int64 + organizationID string + sourceID string + token string + sequences map[model.Signal]uint64 + count atomic.Int64 +} + +type measurements struct { + mu sync.Mutex + ingest []time.Duration + visibility []time.Duration +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "capacity campaign failed:", err) + os.Exit(1) + } +} + +func run() error { + configuration := settings{} + flag.IntVar(&configuration.sustainRate, "sustain-rate", 2_000, "sustained mixed observations per second") + flag.DurationVar(&configuration.sustainDuration, "sustain-duration", time.Hour, "sustained campaign duration") + flag.IntVar(&configuration.burstRate, "burst-rate", 10_000, "burst observations per second") + flag.DurationVar(&configuration.burstDuration, "burst-duration", time.Minute, "burst campaign duration") + flag.Int64Var(&configuration.minimumPrimary, "minimum-primary-observations", 10_000_000, "minimum observations in the primary organization") + flag.IntVar(&configuration.queryIterations, "query-iterations", 20, "iterations per common query") + flag.IntVar(&configuration.organizations, "organizations", 4, "concurrent organizations") + flag.IntVar(&configuration.primaryWeight, "primary-phase-weight", 1, "relative phase weight of the primary organization") + flag.DurationVar(&configuration.batchInterval, "batch-interval", 500*time.Millisecond, "batch scheduling interval") + flag.BoolVar(&configuration.requireCgroup, "require-cgroup", false, "require exact CPU and memory cgroup limits") + flag.IntVar(&configuration.expectedCPUs, "expected-cpus", 4, "required CPU quota") + flag.Int64Var(&configuration.expectedMemory, "expected-memory-bytes", 8<<30, "required memory limit") + flag.Parse() + if flag.NArg() != 0 { + return errors.New("capacity campaign accepts no positional arguments") + } + if err := configuration.validate(); err != nil { + return err + } + cpuQuota, memoryLimit, err := cgroupLimits() + if err != nil && configuration.requireCgroup { + return err + } + if configuration.requireCgroup && (math.Abs(cpuQuota-float64(configuration.expectedCPUs)) > 0.001 || memoryLimit != configuration.expectedMemory || runtime.GOMAXPROCS(0) != configuration.expectedCPUs) { + return fmt.Errorf("resource boundary differs: cpu=%.3f memory=%d gomaxprocs=%d", cpuQuota, memoryLimit, runtime.GOMAXPROCS(0)) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + root, err := os.MkdirTemp("", "observatory-capacity-") + if err != nil { + return errors.New("create capacity workspace") + } + defer os.RemoveAll(root) + if err = os.Chmod(root, 0o700); err != nil { + return errors.New("protect capacity workspace") + } + store, err := storage.Open(filepath.Join(root, "dataset")) + if err != nil { + return err + } + defer store.Close() + projectorContext, stopProjector := context.WithCancel(ctx) + projectorDone := make(chan struct{}) + projectorErrors := make(chan error, 1) + go func() { + defer close(projectorDone) + store.RunProjector(projectorContext, 100*time.Millisecond, func(projectErr error) { + select { + case projectorErrors <- projectErr: + default: + } + }) + }() + defer func() { + stopProjector() + <-projectorDone + }() + states, err := createSources(ctx, store, configuration.organizations, configuration.primaryWeight) + if err != nil { + return err + } + report := campaignReport{Version: reportVersion, StartedAt: time.Now().UTC(), Organizations: len(states), PrimaryPhaseWeight: configuration.primaryWeight} + seed := &atomic.Uint64{} + fmt.Fprintln(os.Stderr, "stage=sustain") + report.Sustain, err = runPhase(ctx, store, states, seed, configuration.sustainRate, configuration.sustainDuration, configuration.batchInterval) + if err != nil { + return err + } + fmt.Fprintln(os.Stderr, "stage=burst") + report.Burst, err = runPhase(ctx, store, states, seed, configuration.burstRate, configuration.burstDuration, configuration.batchInterval) + if err != nil { + return err + } + fmt.Fprintln(os.Stderr, "stage=fill") + fillStarted := time.Now() + if err = fillPrimary(ctx, store, states[0], seed, configuration.minimumPrimary); err != nil { + return err + } + report.FillSeconds = time.Since(fillStarted).Seconds() + report.PrimaryObservations = states[0].count.Load() + for _, state := range states { + report.TotalObservations += state.count.Load() + } + if report.PrimaryObservations < configuration.minimumPrimary { + return errors.New("primary dataset did not reach the required observation count") + } + drainStarted := time.Now() + drainStart, statusErr := store.ProjectionStatus(ctx, drainStarted.UTC()) + if statusErr != nil { + return statusErr + } + report.ProjectionDrain = projectionDrainReport{ + PendingSegments: drainStart.PendingSegments, + PendingBytes: drainStart.PendingBytes, + OldestLagSeconds: drainStart.OldestPendingLag.Seconds(), + } + if err = waitForProjection(ctx, store, projectorErrors, 10*time.Minute); err != nil { + return err + } + report.ProjectionDrain.ElapsedSeconds = time.Since(drainStarted).Seconds() + fmt.Fprintln(os.Stderr, "stage=queries") + report.Queries, err = runQueries(ctx, store, states[0].organizationID, configuration.queryIterations) + if err != nil { + return err + } + report.PrimaryStorage, err = measureStorage(filepath.Join(root, "dataset"), states[0].organizationID) + if err != nil { + return err + } + fmt.Fprintln(os.Stderr, "stage=spool") + report.Spool, err = runSpoolReplay(ctx, filepath.Join(root, "outage")) + if err != nil { + return err + } + fmt.Fprintln(os.Stderr, "stage=retention") + report.Retention, err = runRetention(ctx, filepath.Join(root, "retention")) + if err != nil { + return err + } + report.Resources = resourceReport{GOMAXPROCS: runtime.GOMAXPROCS(0), CPUQuota: cpuQuota, MemoryLimitBytes: memoryLimit} + report.Resources.MaximumRSSBytes = maximumRSS() + report.Resources.DatasetBytes, err = directoryBytes(root) + if err != nil { + return err + } + var campaignErrors []error + if report.Sustain.AchievedRate < float64(configuration.sustainRate)*0.99 || report.Burst.AchievedRate < float64(configuration.burstRate)*0.99 { + campaignErrors = append(campaignErrors, errors.New("target observation rate was not sustained")) + } + if report.Sustain.VisibleP95Milliseconds >= 2_000 || report.Burst.VisibleP95Milliseconds >= 2_000 { + campaignErrors = append(campaignErrors, errors.New("p95 ingestion-to-query visibility exceeded two seconds")) + } + for _, result := range report.Queries { + if result.P95Milliseconds >= 3_000 { + campaignErrors = append(campaignErrors, fmt.Errorf("query %s exceeded the three-second p95 boundary", result.Name)) + } + } + if configuration.requireCgroup && report.Resources.MaximumRSSBytes >= configuration.expectedMemory { + campaignErrors = append(campaignErrors, errors.New("maximum RSS reached the cgroup memory boundary")) + } + report.CompletedAt = time.Now().UTC() + report.Pass = len(campaignErrors) == 0 + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err = encoder.Encode(report); err != nil { + campaignErrors = append(campaignErrors, fmt.Errorf("encode capacity report: %w", err)) + } + return errors.Join(campaignErrors...) +} + +func (configuration settings) validate() error { + if configuration.sustainRate < 1 || configuration.burstRate < configuration.sustainRate || configuration.sustainDuration < time.Second || configuration.burstDuration < time.Second || configuration.minimumPrimary < 1 || configuration.queryIterations < 1 || configuration.queryIterations > 100 || configuration.organizations != 4 || configuration.primaryWeight < 1 || configuration.primaryWeight > 32 || configuration.batchInterval < 100*time.Millisecond || configuration.batchInterval > time.Second || configuration.expectedCPUs < 1 || configuration.expectedMemory < 1<<30 { + return errors.New("capacity campaign settings are invalid") + } + primaryShare := float64(configuration.primaryWeight) / float64(configuration.primaryWeight+configuration.organizations-1) + for _, rate := range []int{configuration.sustainRate, configuration.burstRate} { + if int(math.Ceil(float64(rate)*configuration.batchInterval.Seconds()*primaryShare)) > model.MaxRecords { + return errors.New("weighted batch would exceed the model record limit") + } + } + return nil +} + +func createSources(ctx context.Context, store *storage.Store, count, primaryWeight int) ([]*sourceState, error) { + states := make([]*sourceState, 0, count) + for index := 0; index < count; index++ { + state := &sourceState{index: index, weight: 1, organizationID: fmt.Sprintf("capacity-org-%d", index+1), sourceID: fmt.Sprintf("capacity-source-%d", index+1), sequences: map[model.Signal]uint64{}} + if index == 0 { + state.weight = int64(primaryWeight) + } + var err error + state.token, err = store.CreateSource(ctx, state.sourceID, model.Scope{OrganizationID: state.organizationID, ProjectID: "observatory", EnvironmentID: "capacity", ServiceID: "server"}) + if err != nil { + return nil, err + } + states = append(states, state) + } + return states, nil +} + +func runPhase(ctx context.Context, store *storage.Store, states []*sourceState, seed *atomic.Uint64, rate int, duration, interval time.Duration) (phaseReport, error) { + target := int64(math.Round(float64(rate) * duration.Seconds())) + batches := int(math.Ceil(float64(duration) / float64(interval))) + started := time.Now() + measure := &measurements{} + errorsChannel := make(chan error, len(states)) + visibilityExpected := make(chan time.Time, batches) + visibilityErrors := make(chan error, 1) + visibilityDone := make(chan struct{}) + go func() { + defer close(visibilityDone) + for expected := range visibilityExpected { + visible, err := latestVisibility(ctx, store, states[0].organizationID, expected) + if err != nil { + select { + case visibilityErrors <- err: + default: + } + return + } + measure.mu.Lock() + measure.visibility = append(measure.visibility, visible) + measure.mu.Unlock() + } + }() + var wait sync.WaitGroup + remaining := target + weightTotal := int64(0) + for _, state := range states { + weightTotal += state.weight + } + for index, state := range states { + stateTarget := target * state.weight / weightTotal + if index == 0 { + allocated := int64(0) + for _, candidate := range states { + allocated += target * candidate.weight / weightTotal + } + stateTarget += target - allocated + } + remaining -= stateTarget + wait.Add(1) + go func(current *sourceState, count int64) { + defer wait.Done() + if err := runScheduledSource(ctx, store, current, seed, count, batches, interval, started, measure, visibilityExpected); err != nil { + errorsChannel <- err + } + }(state, stateTarget) + } + if remaining != 0 { + return phaseReport{}, errors.New("phase allocation did not preserve target") + } + wait.Wait() + elapsed := time.Since(started) + close(visibilityExpected) + <-visibilityDone + close(errorsChannel) + for phaseErr := range errorsChannel { + if phaseErr != nil { + return phaseReport{}, phaseErr + } + } + select { + case visibilityErr := <-visibilityErrors: + return phaseReport{}, visibilityErr + default: + } + measure.mu.Lock() + ingest := append([]time.Duration(nil), measure.ingest...) + visibility := append([]time.Duration(nil), measure.visibility...) + measure.mu.Unlock() + if len(ingest) == 0 || len(visibility) == 0 { + return phaseReport{}, errors.New("phase produced no latency evidence") + } + return phaseReport{ + TargetRate: rate, DurationSeconds: duration.Seconds(), Observations: target, ElapsedSeconds: elapsed.Seconds(), AchievedRate: float64(target) / elapsed.Seconds(), + IngestP50Milliseconds: milliseconds(percentile(ingest, 0.50)), IngestP95Milliseconds: milliseconds(percentile(ingest, 0.95)), IngestP99Milliseconds: milliseconds(percentile(ingest, 0.99)), + VisibleP50Milliseconds: milliseconds(percentile(visibility, 0.50)), VisibleP95Milliseconds: milliseconds(percentile(visibility, 0.95)), VisibleP99Milliseconds: milliseconds(percentile(visibility, 0.99)), + }, nil +} + +func runScheduledSource(ctx context.Context, store *storage.Store, state *sourceState, seed *atomic.Uint64, target int64, batches int, interval time.Duration, phaseStart time.Time, measurements *measurements, visibilityExpected chan<- time.Time) error { + base, remainder := target/int64(batches), target%int64(batches) + for batchIndex := 0; batchIndex < batches; batchIndex++ { + planned := phaseStart.Add(time.Duration(batchIndex) * interval) + if delay := time.Until(planned); delay > 0 { + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + count := base + if int64(batchIndex) < remainder { + count++ + } + if count == 0 { + continue + } + signal := []model.Signal{model.SignalLogs, model.SignalMetrics, model.SignalTraces}[batchIndex%3] + observed := time.Now().UTC() + records := syntheticRecords(int(count), signal, observed, seed) + state.sequences[signal]++ + batch := model.Batch{Version: model.BatchVersion, SourceID: state.sourceID, StreamID: string(signal), Sequence: state.sequences[signal], ObservedAt: observed, Signal: signal, Records: records} + ingestStarted := time.Now() + ack, err := store.Ingest(ctx, state.token, batch, observed) + latency := time.Since(ingestStarted) + if err != nil { + return err + } + expected, err := batch.Digest() + if err != nil || ack.BatchDigest != expected || ack.Duplicate { + return errors.New("ingestion acknowledgement did not bind the scheduled batch") + } + state.count.Add(count) + measurements.mu.Lock() + measurements.ingest = append(measurements.ingest, latency) + measurements.mu.Unlock() + if state.index == 0 && signal == model.SignalLogs { + visibilityExpected <- observed + } + } + return nil +} + +func syntheticRecords(count int, signal model.Signal, observed time.Time, seed *atomic.Uint64) []model.Observation { + records := make([]model.Observation, count) + routes := []string{"/", "/items", "/search", "/healthz"} + for index := range records { + id := seed.Add(1) + timestamp := observed.Add(time.Duration(index) * time.Nanosecond) + switch signal { + case model.SignalLogs: + status := "200" + if id%50 == 0 { + status = "503" + } + records[index] = model.Observation{Timestamp: timestamp, Name: "http.server.request", Severity: "information", CorrelationID: fmt.Sprintf("capacity-%d", id), Attributes: map[string]string{"http.route": routes[id%uint64(len(routes))], "http.status_code": status, "duration_ns": strconv.FormatUint(100_000+id%5_000_000, 10)}} + case model.SignalMetrics: + value := float64(id%10_000) / 100 + records[index] = model.Observation{Timestamp: timestamp, Name: "system.cpu.utilization", Value: &value} + case model.SignalTraces: + records[index] = model.Observation{Timestamp: timestamp, Name: "http.server", TraceID: fmt.Sprintf("%032x", id), SpanID: fmt.Sprintf("%016x", id), Attributes: map[string]string{"http.route": routes[id%uint64(len(routes))]}} + } + } + return records +} + +func latestVisibility(ctx context.Context, store *storage.Store, organizationID string, expected time.Time) (time.Duration, error) { + ast, err := query.Parse("logs | sort timestamp desc | limit 1", 10) + if err != nil { + return 0, err + } + // The release boundary is the aggregate p95 below two seconds, not a + // zero-outlier maximum. Keep observing a slow sample long enough to retain + // the phase evidence; the report gate below still fails an excessive p95. + deadline := time.Now().Add(visibilityObservationTimeout) + for { + result, queryErr := store.Query(ctx, ast, query.Scope{OrganizationID: organizationID}, capacityBudget(10), time.Now().UTC()) + if queryErr == nil && len(result.Rows) == 1 { + var timestamp string + for index, column := range result.Columns { + if column.Field == "timestamp" && result.Rows[0].Values[index] != nil { + timestamp = *result.Rows[0].Values[index] + } + } + visibleAt, parseErr := time.Parse(time.RFC3339Nano, timestamp) + if parseErr == nil && !visibleAt.Before(expected) { + return time.Since(expected), nil + } + } + if time.Now().After(deadline) { + return 0, fmt.Errorf("latest visibility query remained stale for %s", visibilityObservationTimeout) + } + timer := time.NewTimer(10 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return 0, ctx.Err() + case <-timer.C: + } + } +} + +func waitForProjection(ctx context.Context, store *storage.Store, projectorErrors <-chan error, maximum time.Duration) error { + deadline := time.Now().Add(maximum) + for { + select { + case err := <-projectorErrors: + return fmt.Errorf("background projection failed: %w", err) + default: + } + status, err := store.ProjectionStatus(ctx, time.Now().UTC()) + if err != nil { + return err + } + if status.PendingSegments == 0 { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("projection backlog remained after %s: segments=%d bytes=%d lag=%s", maximum, status.PendingSegments, status.PendingBytes, status.OldestPendingLag) + } + timer := time.NewTimer(100 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +func fillPrimary(ctx context.Context, store *storage.Store, state *sourceState, seed *atomic.Uint64, minimum int64) error { + batchIndex := 0 + for state.count.Load() < minimum { + remaining := minimum - state.count.Load() + count := int64(model.MaxRecords) + if remaining < count { + count = remaining + } + signal := []model.Signal{model.SignalLogs, model.SignalMetrics, model.SignalTraces}[batchIndex%3] + observed := time.Now().UTC() + state.sequences[signal]++ + batch := model.Batch{Version: model.BatchVersion, SourceID: state.sourceID, StreamID: string(signal), Sequence: state.sequences[signal], ObservedAt: observed, Signal: signal, Records: syntheticRecords(int(count), signal, observed, seed)} + ack, err := store.Ingest(ctx, state.token, batch, observed) + if err != nil { + return err + } + expected, _ := batch.Digest() + if ack.BatchDigest != expected || ack.Duplicate { + return errors.New("fill acknowledgement did not bind the batch") + } + state.count.Add(count) + batchIndex++ + if batchIndex%100 == 0 { + fmt.Fprintf(os.Stderr, "stage=fill observations=%d\n", state.count.Load()) + } + } + return nil +} + +func runQueries(ctx context.Context, store *storage.Store, organizationID string, iterations int) ([]queryReport, error) { + definitions := []struct{ name, text string }{ + {"recent-errors-by-route", `logs | where status >= 500 | window 24h | summarize count() by route, window(5m) | sort count desc | limit 50`}, + {"recent-items", `logs | where route == "/items" | window 24h | sort timestamp desc | limit 50`}, + {"metric-rollup", `metrics | window 24h | summarize count(), p95(value) by name, window(5m) | sort count desc | limit 50`}, + } + reports := make([]queryReport, 0, len(definitions)) + for _, definition := range definitions { + ast, err := query.Parse(definition.text, 50) + if err != nil { + return nil, err + } + var samples []time.Duration + var maximumRows, maximumBytes int64 + for iteration := 0; iteration < iterations; iteration++ { + started := time.Now() + result, queryErr := store.Query(ctx, ast, query.Scope{OrganizationID: organizationID}, capacityBudget(50), time.Now().UTC()) + samples = append(samples, time.Since(started)) + if queryErr != nil { + return nil, fmt.Errorf("capacity query %s failed: %w", definition.name, queryErr) + } + if len(result.Rows) == 0 { + return nil, fmt.Errorf("capacity query %s returned no rows", definition.name) + } + maximumRows = max(maximumRows, int64(result.Stats.ScannedRows)) + maximumBytes = max(maximumBytes, result.Stats.ScannedBytes) + } + reports = append(reports, queryReport{Name: definition.name, Iterations: iterations, P50Milliseconds: milliseconds(percentile(samples, 0.50)), P95Milliseconds: milliseconds(percentile(samples, 0.95)), P99Milliseconds: milliseconds(percentile(samples, 0.99)), MaximumScannedRows: maximumRows, MaximumScannedBytes: maximumBytes}) + } + return reports, nil +} + +func runSpoolReplay(ctx context.Context, root string) (spoolReport, error) { + now := time.Now().UTC() + queue, err := spool.Open(filepath.Join(root, "spool"), 1<<30, 72*time.Hour) + if err != nil { + return spoolReport{}, err + } + store, err := storage.Open(filepath.Join(root, "server")) + if err != nil { + return spoolReport{}, err + } + defer store.Close() + token, err := store.CreateSource(ctx, "outage-source", model.Scope{OrganizationID: "outage-org", ProjectID: "observatory", EnvironmentID: "capacity", ServiceID: "agent"}) + if err != nil { + return spoolReport{}, err + } + const batches, recordsPerBatch = 72, 100 + for index := 0; index < batches; index++ { + observed := now.Add(-time.Duration(batches-index) * time.Hour).Add(time.Minute) + batch := model.Batch{Version: model.BatchVersion, SourceID: "outage-source", StreamID: "logs", Sequence: uint64(index + 1), ObservedAt: observed, Signal: model.SignalLogs, Records: syntheticRecords(recordsPerBatch, model.SignalLogs, observed, &atomic.Uint64{})} + entry, putErr := queue.Put(batch, observed) + if putErr != nil { + return spoolReport{}, putErr + } + if err = os.Chtimes(entry.Path, observed, observed); err != nil { + return spoolReport{}, err + } + } + entries, err := queue.List(now) + if err != nil || len(entries) != batches { + return spoolReport{}, errors.New("72-hour spool did not preserve every batch") + } + report := spoolReport{Batches: len(entries), Observations: batches * recordsPerBatch, OldestAgeHours: now.Sub(entries[0].ModTime).Hours()} + for index, entry := range entries { + batch, readErr := queue.Read(entry) + if readErr != nil { + return spoolReport{}, readErr + } + ack, ingestErr := store.Ingest(ctx, token, batch, now) + if ingestErr != nil { + return spoolReport{}, ingestErr + } + expected, _ := batch.Digest() + if ack.BatchDigest != expected { + return spoolReport{}, errors.New("outage replay acknowledgement mismatch") + } + if index == 0 { + duplicate, duplicateErr := store.Ingest(ctx, token, batch, now) + if duplicateErr != nil || !duplicate.Duplicate || duplicate.BatchDigest != expected { + return spoolReport{}, errors.New("outage replay duplicate was not recognized") + } + report.DuplicateRecognized = true + } + if err = queue.Acknowledge(entry, entry.Digest); err != nil { + return spoolReport{}, err + } + report.Replayed += int64(len(batch.Records)) + } + remaining, err := queue.List(now) + if err != nil { + return spoolReport{}, err + } + report.RemainingAfterAck = len(remaining) + if report.Replayed != report.Observations || report.RemainingAfterAck != 0 || report.OldestAgeHours < 71.9 { + return spoolReport{}, errors.New("outage replay evidence is incomplete") + } + return report, nil +} + +func runRetention(ctx context.Context, root string) (retentionEvidence, error) { + now := time.Now().UTC() + store, err := storage.Open(root) + if err != nil { + return retentionEvidence{}, err + } + defer store.Close() + token, err := store.CreateSource(ctx, "retention-source", model.Scope{OrganizationID: "retention-org", ProjectID: "observatory", EnvironmentID: "capacity", ServiceID: "server"}) + if err != nil { + return retentionEvidence{}, err + } + tests := []struct { + signal model.Signal + age time.Duration + }{ + {model.SignalLogs, 31 * 24 * time.Hour}, + {model.SignalTraces, 31 * 24 * time.Hour}, + {model.SignalMetrics, 15 * 24 * time.Hour}, + {model.SignalDeployments, 399 * 24 * time.Hour}, + } + for index, item := range tests { + observed := now.Add(-item.age) + batch := model.Batch{Version: model.BatchVersion, SourceID: "retention-source", StreamID: string(item.signal), Sequence: 1, ObservedAt: now, Signal: item.signal, Records: syntheticRecords(1, item.signal, observed, &atomic.Uint64{})} + if item.signal == model.SignalDeployments { + batch.Records[0] = model.Observation{Timestamp: observed, Name: "deployment", Attributes: map[string]string{"outcome": "success"}} + } + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + return retentionEvidence{}, fmt.Errorf("retention fixture %d: %w", index, err) + } + } + if err = store.Recover(ctx); err != nil { + return retentionEvidence{}, err + } + policy := storage.RetentionPolicy{RawLogsDays: 30, RawTracesDays: 30, RawMetricsDays: 14, ColdRawDays: 400, DeleteColdRaw: true, MetricRollupsDays: 400, EvidenceDays: 400} + report, err := store.ApplyRetention(ctx, policy, now.Add(2*24*time.Hour)) + if err != nil { + return retentionEvidence{}, err + } + ast, err := query.Parse("logs | window 960h | limit 10", 10) + if err != nil { + return retentionEvidence{}, err + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "retention-org", Sensitive: true}, capacityBudget(10), now.Add(2*24*time.Hour)) + if err != nil { + return retentionEvidence{}, err + } + evidence := retentionEvidence{ArchivedSegments: report.RawSegmentsArchived, ArchivedBytes: report.RawBytesArchived, RemovedSegments: report.RawSegmentsRemoved, RemovedBytes: report.RawBytesRemoved, ProjectionRowsRemoved: report.ProjectedObservationsRemoved, ColdQueryRows: len(result.Rows)} + if evidence.ArchivedSegments != 4 || evidence.RemovedSegments != 1 || evidence.ProjectionRowsRemoved != 4 || evidence.ColdQueryRows != 1 { + return retentionEvidence{}, fmt.Errorf("retention lifecycle mismatch: %+v", evidence) + } + return evidence, nil +} + +func capacityBudget(rows int) query.Budget { + return query.Budget{MaxDuration: 10 * time.Second, MaxRows: rows, MaxScannedBytes: 64 << 30, MaxMemoryBytes: 1 << 30} +} + +func percentile(values []time.Duration, fraction float64) time.Duration { + copyOfValues := append([]time.Duration(nil), values...) + sort.Slice(copyOfValues, func(left, right int) bool { return copyOfValues[left] < copyOfValues[right] }) + index := int(math.Ceil(float64(len(copyOfValues))*fraction)) - 1 + if index < 0 { + index = 0 + } + return copyOfValues[index] +} + +func milliseconds(value time.Duration) float64 { return float64(value) / float64(time.Millisecond) } + +func cgroupLimits() (float64, int64, error) { + cpuBody, err := os.ReadFile("/sys/fs/cgroup/cpu.max") + if err != nil { + return 0, 0, errors.New("read cgroup CPU limit") + } + parts := strings.Fields(string(cpuBody)) + if len(parts) != 2 || parts[0] == "max" { + return 0, 0, errors.New("cgroup CPU quota is not finite") + } + quota, quotaErr := strconv.ParseFloat(parts[0], 64) + period, periodErr := strconv.ParseFloat(parts[1], 64) + memoryBody, memoryErr := os.ReadFile("/sys/fs/cgroup/memory.max") + memory, parseMemoryErr := strconv.ParseInt(strings.TrimSpace(string(memoryBody)), 10, 64) + if quotaErr != nil || periodErr != nil || period <= 0 || memoryErr != nil || parseMemoryErr != nil { + return 0, 0, errors.New("cgroup resource limit is invalid") + } + return quota / period, memory, nil +} + +func maximumRSS() int64 { + var usage syscall.Rusage + if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) != nil { + return 0 + } + return usage.Maxrss * 1024 +} + +func directoryBytes(root string) (int64, error) { + var total int64 + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.Type()&os.ModeSymlink != 0 { + return errors.New("capacity workspace contains a symlink") + } + if entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil || !info.Mode().IsRegular() { + return errors.New("capacity workspace contains a non-regular file") + } + if info.Size() > math.MaxInt64-total { + return errors.New("capacity workspace size overflow") + } + total += info.Size() + return nil + }) + return total, err +} + +func measureStorage(root, primaryOrganizationID string) (storageBreakdown, error) { + var report storageBreakdown + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.Type()&os.ModeSymlink != 0 { + return errors.New("capacity dataset contains a symlink") + } + if entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil || !info.Mode().IsRegular() { + return errors.New("capacity dataset contains a non-regular file") + } + relative, err := filepath.Rel(root, path) + if err != nil || relative == "." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) { + return errors.New("capacity dataset path is invalid") + } + class := &report.Other + switch { + case relative == "control.sqlite" || strings.HasPrefix(relative, "control.sqlite-"): + class = &report.Control + case strings.HasPrefix(relative, "raw"+string(os.PathSeparator)) || strings.HasPrefix(relative, "cold"+string(os.PathSeparator)): + class = &report.Raw + case strings.HasPrefix(relative, "organizations"+string(os.PathSeparator)): + class = &report.Projection + } + if info.Size() > math.MaxInt64-class.Bytes || info.Size() > math.MaxInt64-report.Total.Bytes { + return errors.New("capacity storage class size overflow") + } + class.Files++ + class.Bytes += info.Size() + report.Total.Files++ + report.Total.Bytes += info.Size() + return nil + }) + if err != nil { + return storageBreakdown{}, err + } + projectionPath := filepath.Join(root, "organizations", primaryOrganizationID, "projection.sqlite") + report.SQLiteBytes, report.SQLitePageClasses, err = sqliteObjectBytes(projectionPath) + if err != nil { + return storageBreakdown{}, err + } + return report, nil +} + +func sqliteObjectBytes(path string) (map[string]int64, sqlitePageBreakdown, error) { + dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, sqlitePageBreakdown{}, errors.New("open capacity projection diagnostics") + } + defer db.Close() + db.SetMaxOpenConns(1) + rows, err := db.Query(`SELECT d.name,COALESCE(m.type,'internal'),COALESCE(SUM(d.pgsize),0) FROM dbstat AS d LEFT JOIN sqlite_schema AS m ON m.name=d.name GROUP BY d.name,m.type ORDER BY d.name`) + if err != nil { + return nil, sqlitePageBreakdown{}, errors.New("read capacity projection page accounting") + } + defer rows.Close() + report := map[string]int64{} + var classes sqlitePageBreakdown + for rows.Next() { + var name, objectType string + var bytes int64 + if err = rows.Scan(&name, &objectType, &bytes); err != nil || name == "" || bytes < 0 { + return nil, sqlitePageBreakdown{}, errors.New("read capacity projection page accounting") + } + report[name] = bytes + if bytes > math.MaxInt64-classes.Total { + return nil, sqlitePageBreakdown{}, errors.New("capacity projection page accounting overflow") + } + classes.Total += bytes + switch objectType { + case "table": + classes.Tables += bytes + case "index": + classes.Indexes += bytes + default: + classes.Internal += bytes + } + } + if err = rows.Err(); err != nil { + return nil, sqlitePageBreakdown{}, errors.New("read capacity projection page accounting") + } + if len(report) == 0 { + return nil, sqlitePageBreakdown{}, errors.New("capacity projection page accounting is empty") + } + if classes.Total != classes.Tables+classes.Indexes+classes.Internal { + return nil, sqlitePageBreakdown{}, errors.New("capacity projection page accounting total mismatch") + } + return report, classes, nil +} diff --git a/internal/capacitytest/main_test.go b/internal/capacitytest/main_test.go new file mode 100644 index 0000000..85e68f1 --- /dev/null +++ b/internal/capacitytest/main_test.go @@ -0,0 +1,80 @@ +//go:build observatory_capacity_fixture && linux + +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "context" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/storage" +) + +func TestMeasureStorageClassifiesFilesAndSQLiteObjects(t *testing.T) { + root := filepath.Join(t.TempDir(), "dataset") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + scope := model.Scope{OrganizationID: "capacity-org-1", ProjectID: "observatory", EnvironmentID: "capacity", ServiceID: "server"} + token, err := store.CreateSource(t.Context(), "capacity-source-1", scope) + if err != nil { + store.Close() + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "capacity-source-1", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: syntheticRecords(2, model.SignalLogs, now, new(atomic.Uint64))} + if _, err = store.Ingest(context.Background(), token, batch, now); err != nil { + store.Close() + t.Fatal(err) + } + if _, err = store.ProjectPending(t.Context()); err != nil { + store.Close() + t.Fatal(err) + } + report, err := measureStorage(root, scope.OrganizationID) + if closeErr := store.Close(); err == nil && closeErr != nil { + err = closeErr + } + if err != nil { + t.Fatal(err) + } + if report.Raw.Files == 0 || report.Raw.Bytes == 0 || report.Projection.Files == 0 || report.Projection.Bytes == 0 || report.Control.Files == 0 || report.Control.Bytes == 0 { + t.Fatalf("incomplete breakdown: %+v", report) + } + if report.Total.Files != report.Raw.Files+report.Projection.Files+report.Control.Files+report.Other.Files { + t.Fatalf("file total mismatch: %+v", report) + } + if report.Total.Bytes != report.Raw.Bytes+report.Projection.Bytes+report.Control.Bytes+report.Other.Bytes { + t.Fatalf("byte total mismatch: %+v", report) + } + if report.SQLiteBytes["observations"] == 0 || report.SQLiteBytes["observations_signal_time"] == 0 { + t.Fatalf("SQLite object accounting missing: %+v", report.SQLiteBytes) + } + if report.SQLitePageClasses.Tables == 0 || report.SQLitePageClasses.Indexes == 0 || report.SQLitePageClasses.Total != report.SQLitePageClasses.Tables+report.SQLitePageClasses.Indexes+report.SQLitePageClasses.Internal { + t.Fatalf("SQLite page classes incomplete: %+v", report.SQLitePageClasses) + } +} + +func TestMeasureStorageRejectsSymlink(t *testing.T) { + root := t.TempDir() + target := filepath.Join(t.TempDir(), "target") + if err := os.WriteFile(target, []byte("unsafe"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(root, "linked")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + if _, err := measureStorage(root, "capacity-org-1"); err == nil { + t.Fatal("symlinked capacity evidence was accepted") + } +} diff --git a/internal/collector/collector.go b/internal/collector/collector.go new file mode 100644 index 0000000..46c46ad --- /dev/null +++ b/internal/collector/collector.go @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package collector + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/netip" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" + + "gamertan.com/observatory/internal/model" +) + +const MaxLineBytes = 1 << 20 + +var tendSafeValue = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,256}$`) +var tendOperationID = regexp.MustCompile(`^[0-9a-f]{32}$`) +var tendArtifactDigest = regexp.MustCompile(`^[0-9a-f]{64}$`) +var tendCommit = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func Parse(kind string, line []byte, fallback time.Time, sensitiveFields ...string) (model.Signal, model.Observation, error) { + if len(line) == 0 || len(line) > MaxLineBytes || !utf8.Valid(line) || bytes.IndexByte(line, 0) >= 0 { + return "", model.Observation{}, errors.New("collector line is outside accepted bounds") + } + switch kind { + case "caddy_json": + observation, err := parseCaddy(line, fallback, sensitiveFields) + return model.SignalLogs, observation, err + case "requestlog_jsonl": + observation, err := parseRequestLog(line, fallback, sensitiveFields) + return model.SignalLogs, observation, err + case "tend_events_jsonl": + observation, err := parseTend(line, fallback) + return model.SignalDeployments, observation, err + default: + return "", model.Observation{}, errors.New("unsupported collector kind") + } +} + +func parseCaddy(line []byte, fallback time.Time, sensitiveFields []string) (model.Observation, error) { + var record struct { + Timestamp float64 `json:"ts"` + Status int `json:"status"` + Size int64 `json:"size"` + Duration float64 `json:"duration"` + RequestID string `json:"request_id"` + ClientIP string `json:"client_ip"` + Referrer string `json:"referrer"` + UserAgent string `json:"user_agent"` + Request struct { + Method string `json:"method"` + URI string `json:"uri"` + Headers map[string][]string `json:"headers"` + } `json:"request"` + } + if err := json.Unmarshal(line, &record); err != nil { + return model.Observation{}, errors.New("invalid Caddy JSON record") + } + timestamp := fallback + if record.Timestamp > 0 { + seconds, fraction := mathModf(record.Timestamp) + timestamp = time.Unix(seconds, int64(fraction*1e9)).UTC() + } + path := "/" + parsed, parseErr := url.ParseRequestURI(record.Request.URI) + if parseErr == nil && parsed.Path != "" { + path = parsed.EscapedPath() + } + attributes := map[string]string{ + "http.method": bounded(record.Request.Method, 32), + "http.path": bounded(path, 4096), + "http.status_code": strconv.Itoa(record.Status), + "http.response_bytes": strconv.FormatInt(record.Size, 10), + "duration_ns": strconv.FormatInt(int64(record.Duration*1e9), 10), + } + requestID := record.RequestID + if !safeIdentifier(requestID) { + requestID = firstHeader(record.Request.Headers, "X-Request-Id", "X-Request-ID") + } + if safeIdentifier(requestID) { + attributes["request.id"] = requestID + } + selected := sensitiveFieldSet(sensitiveFields) + if selected["client_ip"] { + if address, err := netip.ParseAddr(record.ClientIP); err == nil && address.IsValid() && address.Zone() == "" { + attributes["client.address"] = address.String() + } + } + if selected["query"] && parseErr == nil && parsed.RawQuery != "" { + attributes["url.query"] = bounded(parsed.RawQuery, 4096) + } + if selected["referrer"] { + if value := boundedSensitive(record.Referrer, 4096); value != "" { + attributes["http.request.referrer"] = value + } + } + if selected["user_agent"] { + if value := boundedSensitive(record.UserAgent, 1024); value != "" { + attributes["user_agent.original"] = value + } + } + return model.Observation{Timestamp: timestamp, Name: "caddy.http.request", CorrelationID: attributes["request.id"], Attributes: attributes}, nil +} + +func parseRequestLog(line []byte, fallback time.Time, sensitiveFields []string) (model.Observation, error) { + var raw map[string]json.RawMessage + if err := json.Unmarshal(line, &raw); err != nil { + return model.Observation{}, errors.New("invalid requestlog JSON record") + } + timestamp := parseTimeFields(raw, fallback, "observed_at", "timestamp", "time") + attributes := map[string]string{} + copyString(raw, attributes, "method", "http.method", 32) + copyString(raw, attributes, "route", "http.route", 4096) + copyNumber(raw, attributes, "status", "http.status_code") + copyNumber(raw, attributes, "bytes", "http.response_bytes") + copyNumber(raw, attributes, "duration_ns", "duration_ns") + copyString(raw, attributes, "authorization_outcome", "auth.outcome", 128) + requestID := rawString(raw["request_id"], 128) + if safeIdentifier(requestID) { + attributes["request.id"] = requestID + } + selected := sensitiveFieldSet(sensitiveFields) + if selected["client_ip"] { + if address, err := netip.ParseAddr(rawString(raw["client_ip"], 64)); err == nil && address.IsValid() && address.Zone() == "" { + attributes["client.address"] = address.String() + } + } + if selected["query"] { + copyString(raw, attributes, "query", "url.query", 4096) + } + if selected["referrer"] { + copyString(raw, attributes, "referer", "http.request.referrer", 2048) + } + if selected["user_agent"] { + copyString(raw, attributes, "user_agent", "user_agent.original", 1024) + } + if selected["session_id"] { + copyString(raw, attributes, "session_id", "session.id", 256) + } + return model.Observation{Timestamp: timestamp, Name: "application.http.request", CorrelationID: requestID, Attributes: attributes}, nil +} + +func sensitiveFieldSet(fields []string) map[string]bool { + selected := make(map[string]bool, len(fields)) + for _, field := range fields { + selected[field] = true + } + return selected +} + +func boundedSensitive(value string, maximum int) string { + if value == "" || len(value) > maximum || !utf8.ValidString(value) || strings.ContainsAny(value, "\x00\r\n") { + return "" + } + return value +} + +func parseTend(line []byte, _ time.Time) (model.Observation, error) { + if len(line) > 4096 { + return model.Observation{}, errors.New("invalid Tend deployment event") + } + var record tendEvent + decoder := json.NewDecoder(bytes.NewReader(line)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&record); err != nil || record.validate() != nil { + return model.Observation{}, errors.New("invalid Tend deployment event") + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return model.Observation{}, errors.New("invalid Tend deployment event") + } + timestamp, _ := time.Parse(time.RFC3339Nano, record.ObservedAt) + attributes := map[string]string{ + "deployment.operation_id": record.OperationID, + "service.name": record.Service, + "deployment.artifact": record.ArtifactDigest, + "deployment.commit": record.Commit, + "deployment.version": record.ReleaseVersion, + "deployment.phase": record.Phase, + "deployment.duration_ms": strconv.FormatInt(record.DurationMillis, 10), + "deployment.outcome": record.Outcome, + } + if record.Slot != "" { + attributes["deployment.slot"] = record.Slot + } + return model.Observation{Timestamp: timestamp.UTC(), Name: "tend.deployment", CorrelationID: record.OperationID, Attributes: attributes}, nil +} + +type tendEvent struct { + Version int `json:"version"` + OperationID string `json:"operation_id"` + Service string `json:"service"` + ArtifactDigest string `json:"artifact_digest"` + Commit string `json:"commit"` + ReleaseVersion string `json:"release_version"` + Phase string `json:"phase"` + Slot string `json:"slot"` + DurationMillis int64 `json:"duration_ms"` + Outcome string `json:"outcome"` + ObservedAt string `json:"observed_at"` +} + +func (event tendEvent) validate() error { + if event.Version != 1 || !tendOperationID.MatchString(event.OperationID) || !tendArtifactDigest.MatchString(event.ArtifactDigest) || !tendCommit.MatchString(event.Commit) { + return errors.New("identity or provenance is invalid") + } + for _, value := range []string{event.Service, event.ArtifactDigest, event.Commit, event.ReleaseVersion, event.Phase, event.Outcome} { + if !tendSafeValue.MatchString(value) || strings.ContainsRune(value, '\x00') { + return errors.New("value is invalid") + } + } + if event.Slot != "" && !tendSafeValue.MatchString(event.Slot) { + return errors.New("slot is invalid") + } + if event.DurationMillis < 0 { + return errors.New("duration is invalid") + } + if _, err := time.Parse(time.RFC3339Nano, event.ObservedAt); err != nil { + return errors.New("timestamp is invalid") + } + return nil +} + +func copyString(raw map[string]json.RawMessage, attributes map[string]string, source, target string, maximum int) { + if value := rawString(raw[source], maximum); value != "" { + attributes[target] = value + } +} + +func copyNumber(raw map[string]json.RawMessage, attributes map[string]string, source, target string) { + value := strings.TrimSpace(string(raw[source])) + if value == "" || len(value) > 64 { + return + } + if _, err := strconv.ParseFloat(value, 64); err == nil { + attributes[target] = value + } +} + +func rawString(raw json.RawMessage, maximum int) string { + var value string + if len(raw) == 0 || json.Unmarshal(raw, &value) != nil || len(value) > maximum || !utf8.ValidString(value) || strings.ContainsAny(value, "\x00\r\n") { + return "" + } + return value +} + +func parseTimeFields(raw map[string]json.RawMessage, fallback time.Time, names ...string) time.Time { + for _, name := range names { + value := rawString(raw[name], 128) + if value == "" { + continue + } + if parsed, err := time.Parse(time.RFC3339Nano, value); err == nil { + return parsed.UTC() + } + } + return fallback.UTC() +} + +func firstHeader(headers map[string][]string, names ...string) string { + for _, name := range names { + for key, values := range headers { + if strings.EqualFold(key, name) && len(values) == 1 { + return values[0] + } + } + } + return "" +} + +func safeIdentifier(value string) bool { + if len(value) < 1 || len(value) > 128 { + return false + } + for _, r := range value { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("._:-", r)) { + return false + } + } + return true +} + +func bounded(value string, maximum int) string { + if len(value) > maximum { + for maximum > 0 && !utf8.ValidString(value[:maximum]) { + maximum-- + } + return value[:maximum] + } + return value +} + +func mathModf(value float64) (int64, float64) { + seconds := int64(value) + return seconds, value - float64(seconds) +} diff --git a/internal/collector/collector_test.go b/internal/collector/collector_test.go new file mode 100644 index 0000000..b5d3149 --- /dev/null +++ b/internal/collector/collector_test.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package collector + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestCaddyCollectorDropsSecretsAndQuery(t *testing.T) { + line := []byte(`{"ts":1720000000.25,"request":{"method":"GET","uri":"/search?q=secret","remote_ip":"192.0.2.10","headers":{"Cookie":["session=secret"],"Authorization":["Bearer secret"],"X-Request-Id":["req-123"]}},"status":200,"size":42,"duration":0.001}`) + _, observation, err := Parse("caddy_json", line, time.Unix(1, 0)) + if err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(observation) + for _, forbidden := range []string{"q=secret", "session=secret", "Bearer secret", "192.0.2.10", "Authorization", "Cookie"} { + if strings.Contains(string(b), forbidden) { + t.Fatalf("secret escaped into observation: %s", b) + } + } + if observation.Attributes["http.path"] != "/search" || observation.CorrelationID != "req-123" { + t.Fatalf("observation=%+v", observation) + } +} + +func TestCaddyCollectorUsesFilteredTopLevelRequestID(t *testing.T) { + line := []byte(`{"ts":1720000000.25,"request_id":"response-request-123","client_ip":"192.0.2.45","referrer":"https://example.test/start?private=yes","user_agent":"example-agent","request":{"method":"GET","uri":"/items?view=private"},"status":200,"size":42,"duration":0.001}`) + _, observation, err := Parse("caddy_json", line, time.Unix(1, 0)) + if err != nil { + t.Fatal(err) + } + if observation.CorrelationID != "response-request-123" || observation.Attributes["request.id"] != "response-request-123" || observation.Attributes["http.path"] != "/items" { + t.Fatalf("observation=%+v", observation) + } + for _, forbidden := range []string{"client.address", "url.query", "http.request.referrer", "user_agent.original"} { + if _, ok := observation.Attributes[forbidden]; ok { + t.Fatalf("default collection retained sensitive field %q", forbidden) + } + } + _, observation, err = Parse("caddy_json", line, time.Unix(1, 0), "client_ip", "query", "referrer", "user_agent") + if err != nil || observation.Attributes["client.address"] != "192.0.2.45" || observation.Attributes["url.query"] != "view=private" || observation.Attributes["http.request.referrer"] != "https://example.test/start?private=yes" || observation.Attributes["user_agent.original"] != "example-agent" { + t.Fatalf("sensitive observation=%+v err=%v", observation, err) + } + + line = []byte(`{"request_id":"invalid request id","request":{"method":"GET","uri":"/items","headers":{"X-Request-ID":["compatible-header-id"]}},"status":200}`) + _, observation, err = Parse("caddy_json", line, time.Unix(1, 0)) + if err != nil || observation.CorrelationID != "compatible-header-id" { + t.Fatalf("fallback observation=%+v err=%v", observation, err) + } +} + +func TestRequestLogCollectorUsesWhitelist(t *testing.T) { + line := []byte(`{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/items/{id}","status":200,"request_id":"request-1","query":"view=full","client_ip":"192.0.2.2","referer":"https://example.test/start","user_agent":"example-agent","session_id":"anonymous-session","extra_secret":"nope"}`) + _, observation, err := Parse("requestlog_jsonl", line, time.Unix(1, 0)) + if err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(observation) + if strings.Contains(string(b), "192.0.2.2") || strings.Contains(string(b), "view=full") || strings.Contains(string(b), "anonymous-session") || observation.Attributes["http.route"] != "/items/{id}" { + t.Fatalf("observation=%s", b) + } + _, observation, err = Parse("requestlog_jsonl", line, time.Unix(1, 0), "client_ip", "query", "referrer", "user_agent", "session_id") + if err != nil || observation.Attributes["client.address"] != "192.0.2.2" || observation.Attributes["url.query"] != "view=full" || observation.Attributes["http.request.referrer"] != "https://example.test/start" || observation.Attributes["user_agent.original"] != "example-agent" || observation.Attributes["session.id"] != "anonymous-session" { + t.Fatalf("sensitive requestlog observation=%+v err=%v", observation, err) + } +} + +func TestTendCollectorIsStrict(t *testing.T) { + line := []byte(`{"version":1,"operation_id":"0123456789abcdef0123456789abcdef","service":"site","artifact_digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","release_version":"v0.2.0-preview.1","phase":"activation","slot":"green","duration_ms":25,"outcome":"succeeded","observed_at":"2026-08-17T01:02:03Z"}`) + signal, observation, err := Parse("tend_events_jsonl", line, time.Unix(1, 0)) + if err != nil { + t.Fatal(err) + } + if signal != "deployments" || observation.Attributes["deployment.outcome"] != "succeeded" || observation.CorrelationID != "0123456789abcdef0123456789abcdef" || len(observation.Attributes) != 9 { + t.Fatalf("signal=%s observation=%+v", signal, observation) + } + if _, _, err := Parse("tend_events_jsonl", append(line[:len(line)-1], []byte(`,"secret":"x"}`)...), time.Unix(1, 0)); err == nil { + t.Fatal("expected unknown field rejection") + } + rollback := []byte(`{"version":1,"operation_id":"fedcba9876543210fedcba9876543210","service":"site","artifact_digest":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","commit":"dddddddddddddddddddddddddddddddddddddddd","release_version":"v0.2.0-preview.1","phase":"rollback","slot":"blue","duration_ms":12,"outcome":"succeeded","observed_at":"2026-08-17T01:03:04.123Z"}`) + _, observation, err = Parse("tend_events_jsonl", rollback, time.Unix(1, 0)) + if err != nil || observation.Attributes["deployment.phase"] != "rollback" || observation.Attributes["deployment.artifact"] != strings.Repeat("c", 64) { + t.Fatalf("rollback observation=%+v err=%v", observation, err) + } + invalid := [][]byte{ + []byte(strings.Replace(string(line), `"version":1`, `"version":2`, 1)), + []byte(strings.Replace(string(line), "0123456789abcdef0123456789abcdef", "short", 1)), + []byte(strings.Replace(string(line), strings.Repeat("a", 64), strings.Repeat("A", 64), 1)), + []byte(strings.Replace(string(line), strings.Repeat("b", 40), strings.Repeat("b", 39), 1)), + []byte(strings.Replace(string(line), `"service":"site"`, `"service":"unsafe service"`, 1)), + []byte(strings.Replace(string(line), `"duration_ms":25`, `"duration_ms":-1`, 1)), + []byte(strings.Replace(string(line), "2026-08-17T01:02:03Z", "not-a-time", 1)), + append(append([]byte{}, line...), []byte(` {}`)...), + []byte(strings.Repeat(" ", 4097)), + } + for index, candidate := range invalid { + if _, _, err := Parse("tend_events_jsonl", candidate, time.Unix(1, 0)); err == nil { + t.Fatalf("invalid Tend event %d was accepted", index) + } + } +} + +func FuzzTendCollector(f *testing.F) { + f.Add([]byte(`{"version":1,"operation_id":"0123456789abcdef0123456789abcdef","service":"site","artifact_digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","release_version":"v0.2.0-preview.1","phase":"activation","slot":"green","duration_ms":25,"outcome":"succeeded","observed_at":"2026-08-17T01:02:03Z"}`)) + f.Add([]byte(`{"version":1}`)) + f.Add([]byte{0, 1, 2, 3}) + f.Fuzz(func(t *testing.T, line []byte) { + signal, observation, err := Parse("tend_events_jsonl", line, time.Unix(1, 0)) + if err != nil { + return + } + if signal != "deployments" || observation.Name != "tend.deployment" || observation.CorrelationID == "" || len(observation.Attributes) < 8 || len(observation.Attributes) > 9 { + t.Fatalf("accepted event violated invariants: signal=%q observation=%+v", signal, observation) + } + allowed := map[string]bool{"deployment.operation_id": true, "service.name": true, "deployment.artifact": true, "deployment.commit": true, "deployment.version": true, "deployment.phase": true, "deployment.slot": true, "deployment.duration_ms": true, "deployment.outcome": true} + for name := range observation.Attributes { + if !allowed[name] { + t.Fatalf("accepted unexpected attribute %q", name) + } + } + }) +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..8ecb6c8 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package config + +import ( + "bytes" + "crypto/ecdh" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "gamertan.com/observatory/internal/hostmetrics" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +const SchemaVersion = 1 + +type Server struct { + Schema int `json:"schema"` + Listen string `json:"listen"` + PublicURL string `json:"public_url"` + DataDir string `json:"data_dir"` + MaxBodyBytes int64 `json:"max_body_bytes"` + MaxConcurrentIngest int `json:"max_concurrent_ingest,omitempty"` + SessionLifetimeText string `json:"session_lifetime"` + SessionLifetime time.Duration `json:"-"` + Query QueryLimits `json:"query"` + Retention Retention `json:"retention"` + WebPush *WebPush `json:"web_push,omitempty"` +} + +type WebPush struct { + PrivateKeyFile string `json:"private_key_file"` + Subject string `json:"subject"` + QueueCapacity int `json:"queue_capacity"` + RequestTimeout string `json:"request_timeout"` + Timeout time.Duration `json:"-"` + PrivateKey []byte `json:"-"` +} + +type QueryLimits struct { + MaxDuration time.Duration `json:"-"` + MaxDurationText string `json:"max_duration"` + MaxRows int `json:"max_rows"` + MaxScannedBytes int64 `json:"max_scanned_bytes"` + MaxMemoryBytes int64 `json:"max_memory_bytes"` +} + +type Retention struct { + RawLogsDays int `json:"raw_logs_days"` + RawTracesDays int `json:"raw_traces_days"` + RawMetricsDays int `json:"raw_metrics_days"` + ColdRawDays int `json:"cold_raw_days"` + DeleteColdRaw bool `json:"delete_cold_raw"` + MetricRollupsDays int `json:"metric_rollups_days"` + EvidenceDays int `json:"evidence_days"` +} + +type FilePolicy struct { + RequireRoot bool + SystemdCredentialDirectory string +} + +func SystemdCredentialPolicy() (FilePolicy, error) { + directory := os.Getenv("CREDENTIALS_DIRECTORY") + if directory == "" { + return FilePolicy{}, errors.New("CREDENTIALS_DIRECTORY is not set") + } + if !filepath.IsAbs(directory) || filepath.Clean(directory) != directory { + return FilePolicy{}, errors.New("CREDENTIALS_DIRECTORY must be an absolute clean path") + } + info, err := os.Lstat(directory) + if err != nil { + return FilePolicy{}, fmt.Errorf("inspect CREDENTIALS_DIRECTORY: %w", err) + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return FilePolicy{}, errors.New("CREDENTIALS_DIRECTORY must be a non-symlink directory") + } + return FilePolicy{SystemdCredentialDirectory: directory}, nil +} + +type Agent struct { + Schema int `json:"schema"` + ServerURL string `json:"server_url"` + CredentialFile string `json:"credential_file"` + SpoolDir string `json:"spool_dir"` + StateFile string `json:"state_file"` + MaxSpoolBytes int64 `json:"max_spool_bytes"` + MaxSpoolAgeText string `json:"max_spool_age"` + MaxSpoolAge time.Duration `json:"-"` + BatchRecords int `json:"batch_records"` + FlushInterval string `json:"flush_interval"` + FlushEvery time.Duration `json:"-"` + Sources []AgentSource `json:"sources"` + AlertRules []AgentAlertRule `json:"alert_rules,omitempty"` +} + +type AgentSource struct { + Kind string `json:"kind"` + Path string `json:"path,omitempty"` + StreamID string `json:"stream_id"` + SensitiveFields []string `json:"sensitive_fields,omitempty"` + LinuxMetrics *hostmetrics.Config `json:"linux_metrics,omitempty"` +} + +type AgentAlertRule struct { + Version int `json:"version"` + ID string `json:"id"` + Revision int `json:"revision"` + StreamID string `json:"stream_id"` + Query string `json:"query"` + MinimumMatches int `json:"minimum_matches"` + AST query.AST `json:"-"` +} + +func LoadServer(path string, policy FilePolicy) (Server, error) { + var cfg Server + if err := loadStrict(path, policy, &cfg); err != nil { + return Server{}, err + } + if cfg.Schema != SchemaVersion { + return Server{}, fmt.Errorf("unsupported server configuration schema %d", cfg.Schema) + } + if cfg.Listen == "" { + return Server{}, errors.New("listen is required") + } + u, err := url.Parse(cfg.PublicURL) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" { + return Server{}, errors.New("public_url must be an absolute HTTPS origin") + } + if !filepath.IsAbs(cfg.DataDir) || filepath.Clean(cfg.DataDir) != cfg.DataDir { + return Server{}, errors.New("data_dir must be an absolute clean path") + } + if cfg.MaxBodyBytes < 1024 || cfg.MaxBodyBytes > 64<<20 { + return Server{}, errors.New("max_body_bytes must be between 1024 and 67108864") + } + if cfg.MaxConcurrentIngest == 0 { + cfg.MaxConcurrentIngest = 8 + } + if cfg.MaxConcurrentIngest < 1 || cfg.MaxConcurrentIngest > 64 { + return Server{}, errors.New("max_concurrent_ingest must be between 1 and 64") + } + sessionLifetime, err := time.ParseDuration(cfg.SessionLifetimeText) + if err != nil || sessionLifetime < 5*time.Minute || sessionLifetime > 30*24*time.Hour { + return Server{}, errors.New("session_lifetime must be between 5m and 720h") + } + cfg.SessionLifetime = sessionLifetime + if cfg.Query.MaxRows < 1 || cfg.Query.MaxRows > 100_000 { + return Server{}, errors.New("query.max_rows must be between 1 and 100000") + } + if cfg.Query.MaxScannedBytes < 1 || cfg.Query.MaxMemoryBytes < 1 { + return Server{}, errors.New("query byte limits must be positive") + } + d, err := time.ParseDuration(cfg.Query.MaxDurationText) + if err != nil || d < time.Millisecond || d > time.Minute { + return Server{}, errors.New("query.max_duration must be between 1ms and 1m") + } + cfg.Query.MaxDuration = d + if err := cfg.Retention.validate(); err != nil { + return Server{}, err + } + if cfg.WebPush != nil { + if !filepath.IsAbs(cfg.WebPush.PrivateKeyFile) || filepath.Clean(cfg.WebPush.PrivateKeyFile) != cfg.WebPush.PrivateKeyFile { + return Server{}, errors.New("web_push.private_key_file must be an absolute clean path") + } + if err = validateWebPushSubject(cfg.WebPush.Subject); err != nil { + return Server{}, err + } + if cfg.WebPush.QueueCapacity < 1 || cfg.WebPush.QueueCapacity > 1024 { + return Server{}, errors.New("web_push.queue_capacity must be between 1 and 1024") + } + cfg.WebPush.Timeout, err = time.ParseDuration(cfg.WebPush.RequestTimeout) + if err != nil || cfg.WebPush.Timeout < time.Second || cfg.WebPush.Timeout > 30*time.Second { + return Server{}, errors.New("web_push.request_timeout must be between 1s and 30s") + } + cfg.WebPush.PrivateKey, err = LoadWebPushPrivateKey(cfg.WebPush.PrivateKeyFile, policy) + if err != nil { + return Server{}, err + } + } + return cfg, nil +} + +func LoadAgent(path string, policy FilePolicy) (Agent, error) { + var cfg Agent + if err := loadStrict(path, policy, &cfg); err != nil { + return Agent{}, err + } + if cfg.Schema != SchemaVersion { + return Agent{}, fmt.Errorf("unsupported agent configuration schema %d", cfg.Schema) + } + u, err := url.Parse(cfg.ServerURL) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" { + return Agent{}, errors.New("server_url must be an absolute HTTPS origin") + } + for label, path := range map[string]string{"credential_file": cfg.CredentialFile, "spool_dir": cfg.SpoolDir, "state_file": cfg.StateFile} { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return Agent{}, fmt.Errorf("%s must be an absolute clean path", label) + } + } + if cfg.SpoolDir == cfg.StateFile || !strings.HasPrefix(cfg.StateFile, cfg.SpoolDir+string(os.PathSeparator)) { + return Agent{}, errors.New("state_file must be below spool_dir") + } + if cfg.MaxSpoolBytes < 1<<20 || cfg.MaxSpoolBytes > 5<<30 { + return Agent{}, errors.New("max_spool_bytes must be between 1 MiB and 5 GiB") + } + age, err := time.ParseDuration(cfg.MaxSpoolAgeText) + if err != nil || age < time.Hour || age > 72*time.Hour { + return Agent{}, errors.New("max_spool_age must be between 1h and 72h") + } + cfg.MaxSpoolAge = age + flush, err := time.ParseDuration(cfg.FlushInterval) + if err != nil || flush < 100*time.Millisecond || flush > time.Minute { + return Agent{}, errors.New("flush_interval must be between 100ms and 1m") + } + cfg.FlushEvery = flush + if cfg.BatchRecords < 1 || cfg.BatchRecords > model.MaxRecords { + return Agent{}, fmt.Errorf("batch_records must be between 1 and %d", model.MaxRecords) + } + if len(cfg.Sources) < 1 || len(cfg.Sources) > 64 { + return Agent{}, errors.New("sources must contain between 1 and 64 entries") + } + streams := map[string]bool{} + for index, source := range cfg.Sources { + switch source.Kind { + case "caddy_json", "requestlog_jsonl", "tend_events_jsonl": + if source.LinuxMetrics != nil { + return Agent{}, fmt.Errorf("sources[%d].linux_metrics is only valid for linux_metrics", index) + } + if !filepath.IsAbs(source.Path) || filepath.Clean(source.Path) != source.Path { + return Agent{}, fmt.Errorf("sources[%d].path must be absolute and clean", index) + } + case "linux_metrics": + if source.Path != "" || source.LinuxMetrics == nil { + return Agent{}, fmt.Errorf("sources[%d] requires linux_metrics and no path", index) + } + if err := source.LinuxMetrics.Validate(); err != nil { + return Agent{}, fmt.Errorf("sources[%d]: %w", index, err) + } + default: + return Agent{}, fmt.Errorf("sources[%d].kind is unsupported", index) + } + allowedSensitive := map[string]bool{} + switch source.Kind { + case "caddy_json": + for _, name := range []string{"client_ip", "query", "referrer", "user_agent"} { + allowedSensitive[name] = true + } + case "requestlog_jsonl": + for _, name := range []string{"client_ip", "query", "referrer", "session_id", "user_agent"} { + allowedSensitive[name] = true + } + } + selectedSensitive := map[string]bool{} + for _, name := range source.SensitiveFields { + if !allowedSensitive[name] { + return Agent{}, fmt.Errorf("sources[%d].sensitive_fields contains unsupported field %q", index, name) + } + if selectedSensitive[name] { + return Agent{}, fmt.Errorf("sources[%d].sensitive_fields duplicates %q", index, name) + } + selectedSensitive[name] = true + } + if err := model.ValidateStreamID(source.StreamID); err != nil { + return Agent{}, fmt.Errorf("sources[%d]: %w", index, err) + } + if streams[source.StreamID] { + return Agent{}, fmt.Errorf("sources[%d].stream_id is duplicated", index) + } + streams[source.StreamID] = true + } + rules := map[string]bool{} + for index := range cfg.AlertRules { + rule := &cfg.AlertRules[index] + if rule.Version != 1 || model.ValidateSourceID(rule.ID) != nil || rule.Revision < 1 || rule.Revision > 1_000_000 || !streams[rule.StreamID] || rule.MinimumMatches < 1 || rule.MinimumMatches > model.MaxRecords || rules[rule.ID] { + return Agent{}, fmt.Errorf("alert_rules[%d] identity is invalid", index) + } + var sourceKind string + for _, source := range cfg.Sources { + if source.StreamID == rule.StreamID { + sourceKind = source.Kind + break + } + } + if sourceKind != "caddy_json" && sourceKind != "requestlog_jsonl" { + return Agent{}, fmt.Errorf("alert_rules[%d] requires a log stream", index) + } + rule.AST, err = query.Parse(rule.Query, model.MaxRecords) + if err != nil || rule.AST.Signal != model.SignalLogs || len(rule.AST.Filters) == 0 || rule.AST.Summary != nil || rule.AST.Sort != nil || rule.AST.Window != 0 || rule.AST.Limit < rule.MinimumMatches { + return Agent{}, fmt.Errorf("alert_rules[%d] query must be a bounded logs filter without sort, summary, or window", index) + } + for _, filter := range rule.AST.Filters { + switch query.CanonicalField(filter.Field) { + case "project.id", "environment.id", "service.id", "source.id", "stream.id": + return Agent{}, fmt.Errorf("alert_rules[%d] cannot filter server-derived scope", index) + } + } + rules[rule.ID] = true + } + return cfg, nil +} + +func LoadCredential(path string, policy FilePolicy) (string, error) { + var holder struct { + Credential string `json:"credential"` + } + if err := loadStrict(path, policy, &holder); err != nil { + return "", err + } + if len(holder.Credential) < 48 || len(holder.Credential) > 512 || !strings.HasPrefix(holder.Credential, "obs1.") || strings.ContainsAny(holder.Credential, " \t\r\n") { + return "", errors.New("credential file contains an invalid source credential") + } + return holder.Credential, nil +} + +func LoadEnrollmentToken(path string, policy FilePolicy) (string, error) { + var holder struct { + EnrollmentToken string `json:"enrollment_token"` + } + if err := loadStrict(path, policy, &holder); err != nil { + return "", err + } + if len(holder.EnrollmentToken) != len("obse1.")+64 || !strings.HasPrefix(holder.EnrollmentToken, "obse1.") || strings.ContainsAny(holder.EnrollmentToken, " \t\r\n") { + return "", errors.New("enrollment file contains an invalid token") + } + return holder.EnrollmentToken, nil +} + +func WriteEnrollmentToken(path, token string) error { + if len(token) != len("obse1.")+64 || !strings.HasPrefix(token, "obse1.") || strings.ContainsAny(token, " \t\r\n") { + return errors.New("invalid enrollment token") + } + return writePrivateJSON(path, struct { + EnrollmentToken string `json:"enrollment_token"` + }{token}) +} + +func WriteCredential(path, credential string) error { + if len(credential) < 48 || len(credential) > 512 || !strings.HasPrefix(credential, "obs1.") || strings.ContainsAny(credential, " \t\r\n") { + return errors.New("invalid source credential") + } + return writePrivateJSON(path, struct { + Credential string `json:"credential"` + }{credential}) +} + +func LoadWebPushPrivateKey(path string, policy FilePolicy) ([]byte, error) { + var holder struct { + PrivateKey string `json:"private_key"` + } + if err := loadStrict(path, policy, &holder); err != nil { + return nil, err + } + key, err := base64.RawURLEncoding.DecodeString(holder.PrivateKey) + if err != nil || len(key) != 32 { + return nil, errors.New("web push private key file is invalid") + } + if _, err = ecdh.P256().NewPrivateKey(key); err != nil { + return nil, errors.New("web push private key file is invalid") + } + return key, nil +} + +func WriteWebPushPrivateKey(path string, key []byte) error { + if len(key) != 32 { + return errors.New("invalid web push private key") + } + if _, err := ecdh.P256().NewPrivateKey(key); err != nil { + return errors.New("invalid web push private key") + } + return writePrivateJSON(path, struct { + PrivateKey string `json:"private_key"` + }{base64.RawURLEncoding.EncodeToString(key)}) +} + +func writePrivateJSON(path string, value any) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return errors.New("secret output path must be absolute and clean") + } + parent := filepath.Dir(path) + info, err := os.Lstat(parent) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("secret output directory must be an existing non-symlink directory") + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return fmt.Errorf("create secret output: %w", err) + } + body, marshalErr := json.Marshal(value) + if marshalErr == nil { + body = append(body, '\n') + _, marshalErr = file.Write(body) + } + if marshalErr == nil { + marshalErr = file.Sync() + } + if closeErr := file.Close(); marshalErr == nil { + marshalErr = closeErr + } + if marshalErr != nil { + _ = os.Remove(path) + return fmt.Errorf("write secret output: %w", marshalErr) + } + directory, err := os.Open(parent) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} + +func (r Retention) validate() error { + values := []int{r.RawLogsDays, r.RawTracesDays, r.RawMetricsDays, r.ColdRawDays, r.MetricRollupsDays, r.EvidenceDays} + for _, days := range values { + if days < 1 || days > 3650 { + return errors.New("retention values must be between 1 and 3650 days") + } + } + if r.MetricRollupsDays < r.RawMetricsDays { + return errors.New("metric rollup retention cannot be shorter than raw metric retention") + } + if r.ColdRawDays < r.RawLogsDays || r.ColdRawDays < r.RawTracesDays || r.ColdRawDays < r.RawMetricsDays || r.ColdRawDays < r.EvidenceDays { + return errors.New("cold raw retention cannot be shorter than a hot raw or evidence retention window") + } + return nil +} + +func validateWebPushSubject(subject string) error { + if len(subject) < 8 || len(subject) > 512 || strings.ContainsAny(subject, " \t\r\n") { + return errors.New("web_push.subject is invalid") + } + parsed, err := url.Parse(subject) + if err != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return errors.New("web_push.subject is invalid") + } + if parsed.Scheme == "mailto" && parsed.Opaque != "" && strings.Contains(parsed.Opaque, "@") { + return nil + } + if parsed.Scheme == "https" && parsed.Host != "" && parsed.User == nil { + return nil + } + return errors.New("web_push.subject must be a mailto address or HTTPS URL") +} + +func loadStrict(path string, policy FilePolicy, out any) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return errors.New("configuration path must be absolute and clean") + } + info, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("inspect configuration: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return errors.New("configuration must be a regular non-symlink file") + } + if policy.SystemdCredentialDirectory != "" { + if filepath.Dir(path) != policy.SystemdCredentialDirectory { + return errors.New("runtime credential must be a direct child of CREDENTIALS_DIRECTORY") + } + mode := info.Mode().Perm() + if mode != 0o400 && mode != 0o440 && mode != 0o600 { + return fmt.Errorf("runtime credential mode must be 0400, 0440, or 0600, got %04o", mode) + } + if runtime.GOOS != "linux" { + return errors.New("systemd credential validation is supported only on Linux") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || (stat.Uid != 0 && stat.Uid != uint32(os.Geteuid())) { + return errors.New("runtime credential must be owned by root or the service user") + } + } else if info.Mode().Perm() != 0o600 { + return fmt.Errorf("configuration mode must be 0600, got %04o", info.Mode().Perm()) + } + if policy.RequireRoot { + if runtime.GOOS != "linux" { + return errors.New("root ownership validation is supported only on Linux") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Uid != 0 { + return errors.New("configuration must be owned by root") + } + } + b, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read configuration: %w", err) + } + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + if err := dec.Decode(out); err != nil { + return fmt.Errorf("decode configuration: %w", err) + } + if err := dec.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return errors.New("configuration must contain exactly one JSON value") + } + return nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..f30e821 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package config + +import ( + "crypto/ecdh" + "crypto/rand" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestLoadServerStrictAndValidated(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "server.json") + body := `{"schema":1,"listen":"127.0.0.1:9010","public_url":"https://observatory.example","data_dir":"` + filepath.Join(dir, "data") + `","max_body_bytes":1048576,"session_lifetime":"12h","query":{"max_duration":"2s","max_rows":1000,"max_scanned_bytes":10485760,"max_memory_bytes":8388608},"retention":{"raw_logs_days":30,"raw_traces_days":30,"raw_metrics_days":14,"cold_raw_days":400,"metric_rollups_days":400,"evidence_days":400}}` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadServer(path, FilePolicy{}) + if err != nil { + t.Fatal(err) + } + if cfg.Query.MaxDuration.String() != "2s" || cfg.MaxConcurrentIngest != 8 || cfg.Retention.EvidenceDays != 400 || cfg.Retention.DeleteColdRaw { + t.Fatalf("unexpected config: %#v", cfg) + } + deleting := strings.Replace(body, `"cold_raw_days":400`, `"cold_raw_days":400,"delete_cold_raw":true`, 1) + if err = os.WriteFile(path, []byte(deleting), 0o600); err != nil { + t.Fatal(err) + } + if cfg, err = LoadServer(path, FilePolicy{}); err != nil || !cfg.Retention.DeleteColdRaw { + t.Fatalf("explicit cold deletion config=%#v err=%v", cfg.Retention, err) + } + + bad := strings.Replace(body, `"schema":1`, `"schema":1,"surprise":true`, 1) + if err := os.WriteFile(path, []byte(bad), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadServer(path, FilePolicy{}); err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("expected unknown-field rejection, got %v", err) + } + bad = strings.Replace(body, `"max_body_bytes":1048576`, `"max_body_bytes":1048576,"max_concurrent_ingest":65`, 1) + if err = os.WriteFile(path, []byte(bad), 0o600); err != nil { + t.Fatal(err) + } + if _, err = LoadServer(path, FilePolicy{}); err == nil || !strings.Contains(err.Error(), "max_concurrent_ingest") { + t.Fatalf("expected ingestion concurrency rejection, got %v", err) + } +} + +func TestDogfoodServerConfigurationMatchesTheDeploymentBoundary(t *testing.T) { + body, err := os.ReadFile(filepath.Join("..", "..", "release", "server.json")) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + keyPath := filepath.Join(dir, "web-push.json") + key, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + if err = WriteWebPushPrivateKey(keyPath, key.Bytes()); err != nil { + t.Fatal(err) + } + const runtimeKey = "/run/credentials/gamertan-observatory.service/web-push.json" + rewritten := strings.Replace(string(body), runtimeKey, keyPath, 1) + if rewritten == string(body) || strings.Contains(rewritten, runtimeKey) { + t.Fatal("dogfood Web Push credential path was not replaced exactly once") + } + path := filepath.Join(dir, "server.json") + if err = os.WriteFile(path, []byte(rewritten), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadServer(path, FilePolicy{}) + if err != nil { + t.Fatal(err) + } + if cfg.Listen != "127.0.0.1:8093" || cfg.PublicURL != "https://observatory.gamertan.com" || cfg.DataDir != "/var/lib/gamertan-observatory" || cfg.MaxBodyBytes != 32<<20 || cfg.MaxConcurrentIngest != 8 || cfg.Retention.DeleteColdRaw || cfg.WebPush == nil || cfg.WebPush.Subject != "mailto:security@sandwichhime.com" || cfg.WebPush.QueueCapacity != 64 || cfg.WebPush.Timeout != 10*time.Second || string(cfg.WebPush.PrivateKey) != string(key.Bytes()) { + t.Fatalf("unexpected dogfood config: %#v", cfg) + } +} + +func TestLoadServerRejectsRollupsShorterThanRawMetrics(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "server.json") + body := `{"schema":1,"listen":"127.0.0.1:9010","public_url":"https://observatory.example","data_dir":"` + filepath.Join(dir, "data") + `","max_body_bytes":1048576,"session_lifetime":"12h","query":{"max_duration":"2s","max_rows":1000,"max_scanned_bytes":10485760,"max_memory_bytes":8388608},"retention":{"raw_logs_days":30,"raw_traces_days":30,"raw_metrics_days":14,"cold_raw_days":400,"metric_rollups_days":7,"evidence_days":400}}` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadServer(path, FilePolicy{}); err == nil || !strings.Contains(err.Error(), "cannot be shorter") { + t.Fatalf("short metric rollup retention err=%v", err) + } +} + +func TestLoadServerRejectsColdWindowShorterThanHotEvidence(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "server.json") + body := `{"schema":1,"listen":"127.0.0.1:9010","public_url":"https://observatory.example","data_dir":"` + filepath.Join(dir, "data") + `","max_body_bytes":1048576,"session_lifetime":"12h","query":{"max_duration":"2s","max_rows":1000,"max_scanned_bytes":10485760,"max_memory_bytes":8388608},"retention":{"raw_logs_days":30,"raw_traces_days":30,"raw_metrics_days":14,"cold_raw_days":399,"metric_rollups_days":400,"evidence_days":400}}` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadServer(path, FilePolicy{}); err == nil || !strings.Contains(err.Error(), "cold raw retention") { + t.Fatalf("short cold retention err=%v", err) + } +} + +func TestLoadServerWebPushUsesSeparatePrivateKeyFile(t *testing.T) { + dir := t.TempDir() + keyPath := filepath.Join(dir, "web-push.json") + key, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + if err = WriteWebPushPrivateKey(keyPath, key.Bytes()); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "server.json") + body := `{"schema":1,"listen":"127.0.0.1:9010","public_url":"https://observatory.example","data_dir":"` + filepath.Join(dir, "data") + `","max_body_bytes":1048576,"session_lifetime":"12h","query":{"max_duration":"2s","max_rows":1000,"max_scanned_bytes":10485760,"max_memory_bytes":8388608},"retention":{"raw_logs_days":30,"raw_traces_days":30,"raw_metrics_days":14,"cold_raw_days":400,"metric_rollups_days":400,"evidence_days":400},"web_push":{"private_key_file":"` + keyPath + `","subject":"mailto:security@sandwichhime.com","queue_capacity":16,"request_timeout":"5s"}}` + if err = os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadServer(path, FilePolicy{}) + if err != nil { + t.Fatal(err) + } + if cfg.WebPush == nil || cfg.WebPush.Timeout != 5*time.Second || cfg.WebPush.QueueCapacity != 16 || string(cfg.WebPush.PrivateKey) != string(key.Bytes()) { + t.Fatalf("web push=%+v", cfg.WebPush) + } + if err = os.WriteFile(keyPath, []byte(`{"private_key":"not-a-key"}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err = LoadServer(path, FilePolicy{}); err == nil { + t.Fatal("invalid Web Push private key accepted") + } +} + +func TestLoadServerRejectsModeAndSymlink(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "server.json") + if err := os.WriteFile(path, []byte(`{}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadServer(path, FilePolicy{}); err == nil || !strings.Contains(err.Error(), "0600") { + t.Fatalf("expected mode rejection, got %v", err) + } + link := filepath.Join(dir, "link.json") + if err := os.Symlink(path, link); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + if _, err := LoadServer(link, FilePolicy{}); err == nil || !strings.Contains(err.Error(), "non-symlink") { + t.Fatalf("expected symlink rejection, got %v", err) + } +} + +func TestLoadAgentKeepsSourcesLocalAndBounded(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "agent.json") + body := `{"schema":1,"server_url":"https://observatory.example","credential_file":"` + filepath.Join(dir, "credential.json") + `","spool_dir":"` + filepath.Join(dir, "spool") + `","state_file":"` + filepath.Join(dir, "spool", "state.json") + `","max_spool_bytes":5368709120,"max_spool_age":"72h","batch_records":500,"flush_interval":"1s","sources":[{"kind":"caddy_json","path":"/var/log/caddy/access.jsonl","stream_id":"caddy","sensitive_fields":["client_ip","query","referrer","user_agent"]}]}` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadAgent(path, FilePolicy{}) + if err != nil { + t.Fatal(err) + } + if cfg.MaxSpoolAge != 72*time.Hour || cfg.FlushEvery != time.Second || cfg.Sources[0].Path != "/var/log/caddy/access.jsonl" || len(cfg.Sources[0].SensitiveFields) != 4 { + t.Fatalf("agent=%+v", cfg) + } + bad := strings.Replace(body, `"kind":"caddy_json"`, `"kind":"shell"`, 1) + if err := os.WriteFile(path, []byte(bad), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadAgent(path, FilePolicy{}); err == nil { + t.Fatal("expected collector-kind rejection") + } + for _, change := range [][2]string{ + {`"sensitive_fields":["client_ip","query","referrer","user_agent"]`, `"sensitive_fields":["client_ip","client_ip"]`}, + {`"sensitive_fields":["client_ip","query","referrer","user_agent"]`, `"sensitive_fields":["cookie"]`}, + {`"kind":"caddy_json"`, `"kind":"tend_events_jsonl"`}, + } { + candidate := strings.Replace(body, change[0], change[1], 1) + if err := os.WriteFile(path, []byte(candidate), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadAgent(path, FilePolicy{}); err == nil { + t.Fatalf("invalid sensitive-field configuration accepted: %s", change[1]) + } + } +} + +func TestLoadAgentAcceptsExplicitLinuxMetricSelectors(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "agent.json") + body := `{"schema":1,"server_url":"https://observatory.example","credential_file":"` + filepath.Join(dir, "credential.json") + `","spool_dir":"` + filepath.Join(dir, "spool") + `","state_file":"` + filepath.Join(dir, "spool", "state.json") + `","max_spool_bytes":1048576,"max_spool_age":"1h","batch_records":500,"flush_interval":"1s","sources":[{"kind":"linux_metrics","stream_id":"host-metrics","linux_metrics":{"proc_root":"/proc","cgroup_root":"/sys/fs/cgroup","filesystems":[{"name":"root","path":"/"}],"processes":[{"name":"caddy","pid_file":"/run/caddy.pid"}],"cgroups":[{"name":"caddy-service","path":"system.slice/caddy.service"}]}}]}` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadAgent(path, FilePolicy{}) + if err != nil { + t.Fatal(err) + } + if cfg.Sources[0].LinuxMetrics == nil || cfg.Sources[0].LinuxMetrics.Filesystems[0].Name != "root" { + t.Fatalf("agent=%+v", cfg) + } + bad := strings.Replace(body, `"path":"system.slice/caddy.service"`, `"path":"../escape"`, 1) + if err = os.WriteFile(path, []byte(bad), 0o600); err != nil { + t.Fatal(err) + } + if _, err = LoadAgent(path, FilePolicy{}); err == nil { + t.Fatal("escaping cgroup selector accepted") + } +} + +func TestLoadAgentAcceptsOnlyLocalBoundedLogAlertRules(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "agent.json") + body := `{"schema":1,"server_url":"https://observatory.example","credential_file":"` + filepath.Join(dir, "credential.json") + `","spool_dir":"` + filepath.Join(dir, "spool") + `","state_file":"` + filepath.Join(dir, "spool", "state.json") + `","max_spool_bytes":1048576,"max_spool_age":"1h","batch_records":500,"flush_interval":"1s","sources":[{"kind":"requestlog_jsonl","path":"/var/log/example/request.jsonl","stream_id":"requests"}],"alert_rules":[{"version":1,"id":"http-failures","revision":2,"stream_id":"requests","query":"logs | where status >= 500 | limit 10","minimum_matches":1}]}` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadAgent(path, FilePolicy{}) + if err != nil || len(cfg.AlertRules) != 1 || cfg.AlertRules[0].AST.Signal != "logs" || len(cfg.AlertRules[0].AST.Filters) != 1 { + t.Fatalf("agent=%+v err=%v", cfg, err) + } + for _, replacement := range []string{ + `"stream_id":"missing"`, + `"query":"metrics | where value > 0 | limit 10"`, + `"query":"logs | summarize count() | limit 10"`, + `"query":"logs | where service == other | limit 10"`, + `"minimum_matches":11`, + } { + candidate := body + switch { + case strings.Contains(replacement, "missing"): + candidate = strings.Replace(candidate, `"stream_id":"requests","query"`, replacement+`,"query"`, 1) + case strings.HasPrefix(replacement, `"query"`): + candidate = strings.Replace(candidate, `"query":"logs | where status >= 500 | limit 10"`, replacement, 1) + default: + candidate = strings.Replace(candidate, `"minimum_matches":1`, replacement, 1) + } + if err = os.WriteFile(path, []byte(candidate), 0o600); err != nil { + t.Fatal(err) + } + if _, err = LoadAgent(path, FilePolicy{}); err == nil { + t.Fatalf("invalid alert rule accepted: %s", replacement) + } + } +} + +func TestLoadCredentialDoesNotAcceptWhitespaceOrExtraFields(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "credential.json") + valid := `{"credential":"obs1.source.` + strings.Repeat("a", 64) + `"}` + if err := os.WriteFile(path, []byte(valid), 0o600); err != nil { + t.Fatal(err) + } + if credential, err := LoadCredential(path, FilePolicy{}); err != nil || !strings.HasPrefix(credential, "obs1.source.") { + t.Fatalf("credential=%q err=%v", credential, err) + } + if err := os.WriteFile(path, []byte(`{"credential":"obs1.source.bad value"}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadCredential(path, FilePolicy{}); err == nil { + t.Fatal("expected whitespace rejection") + } +} + +func TestEnrollmentAndCredentialOutputsArePrivateAndExclusive(t *testing.T) { + dir := t.TempDir() + enrollmentPath := filepath.Join(dir, "enrollment.json") + enrollment := "obse1." + strings.Repeat("e", 64) + if err := WriteEnrollmentToken(enrollmentPath, enrollment); err != nil { + t.Fatal(err) + } + if got, err := LoadEnrollmentToken(enrollmentPath, FilePolicy{}); err != nil || got != enrollment { + t.Fatalf("token=%q err=%v", got, err) + } + if err := WriteEnrollmentToken(enrollmentPath, enrollment); err == nil { + t.Fatal("enrollment output overwritten") + } + credentialPath := filepath.Join(dir, "credential.json") + credential := "obs1.source." + strings.Repeat("a", 64) + if err := WriteCredential(credentialPath, credential); err != nil { + t.Fatal(err) + } + if got, err := LoadCredential(credentialPath, FilePolicy{}); err != nil || got != credential { + t.Fatalf("credential=%q err=%v", got, err) + } + for _, path := range []string{enrollmentPath, credentialPath} { + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("path=%s info=%v err=%v", path, info, err) + } + } +} + +func TestSystemdCredentialPolicyIsExplicitAndConfined(t *testing.T) { + dir := t.TempDir() + t.Setenv("CREDENTIALS_DIRECTORY", dir) + policy, err := SystemdCredentialPolicy() + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "credential.json") + credential := `{"credential":"obs1.source.` + strings.Repeat("a", 64) + `"}` + if err := os.WriteFile(path, []byte(credential), 0o400); err != nil { + t.Fatal(err) + } + if _, err := LoadCredential(path, policy); err != nil { + t.Fatalf("load systemd credential: %v", err) + } + if err := os.Chmod(path, 0o440); err != nil { + t.Fatal(err) + } + if _, err := LoadCredential(path, policy); err != nil { + t.Fatalf("load systemd mode-0440 credential: %v", err) + } + if err := os.Chmod(path, 0o444); err != nil { + t.Fatal(err) + } + if _, err := LoadCredential(path, policy); err == nil || !strings.Contains(err.Error(), "0400, 0440, or 0600") { + t.Fatalf("expected public credential mode rejection, got %v", err) + } + if err := os.Chmod(path, 0o400); err != nil { + t.Fatal(err) + } + + outside := filepath.Join(t.TempDir(), "credential.json") + if err := os.WriteFile(outside, []byte(credential), 0o400); err != nil { + t.Fatal(err) + } + if _, err := LoadCredential(outside, policy); err == nil || !strings.Contains(err.Error(), "direct child") { + t.Fatalf("expected confinement rejection, got %v", err) + } + + nested := filepath.Join(dir, "nested") + if err := os.Mkdir(nested, 0o700); err != nil { + t.Fatal(err) + } + nestedPath := filepath.Join(nested, "credential.json") + if err := os.WriteFile(nestedPath, []byte(credential), 0o400); err != nil { + t.Fatal(err) + } + if _, err := LoadCredential(nestedPath, policy); err == nil || !strings.Contains(err.Error(), "direct child") { + t.Fatalf("expected nested-path rejection, got %v", err) + } +} diff --git a/internal/edgealert/evaluator.go b/internal/edgealert/evaluator.go new file mode 100644 index 0000000..fa8d070 --- /dev/null +++ b/internal/edgealert/evaluator.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package edgealert + +import ( + "errors" + "time" + + "gamertan.com/observatory/internal/config" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +type Evaluation struct { + State string + Matches int + WindowStart time.Time + WindowEnd time.Time + ObservedAt time.Time +} + +// Evaluate applies one locally configured, filter-only log rule to one exact +// durable batch. It does not perform I/O, mutate incident state, or retain +// telemetry beyond the caller-owned batch. +func Evaluate(rule config.AgentAlertRule, batch model.Batch) (Evaluation, error) { + if rule.AST.Signal != model.SignalLogs || batch.Signal != model.SignalLogs || rule.StreamID != batch.StreamID || len(batch.Records) == 0 { + return Evaluation{}, errors.New("edge alert rule and batch are incompatible") + } + first, last := batch.Records[0].Timestamp.UTC(), batch.Records[0].Timestamp.UTC() + matches := 0 + state := "clear" + for _, observation := range batch.Records { + timestamp := observation.Timestamp.UTC() + if timestamp.Before(first) { + first = timestamp + } + if timestamp.After(last) { + last = timestamp + } + matched, err := query.MatchObservation(observation, rule.AST, nil) + if err != nil { + return Evaluation{State: "error", Matches: matches, WindowStart: first, WindowEnd: last, ObservedAt: maxTime(batch.ObservedAt.UTC(), last)}, nil + } + if matched { + matches++ + if matches >= rule.MinimumMatches { + state = "matched" + } + } + } + return Evaluation{State: state, Matches: matches, WindowStart: first, WindowEnd: last, ObservedAt: maxTime(batch.ObservedAt.UTC(), last)}, nil +} + +func maxTime(left, right time.Time) time.Time { + if left.After(right) { + return left + } + return right +} diff --git a/internal/edgealert/evaluator_test.go b/internal/edgealert/evaluator_test.go new file mode 100644 index 0000000..f509559 --- /dev/null +++ b/internal/edgealert/evaluator_test.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package edgealert + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "gamertan.com/observatory/internal/config" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/storage" +) + +func TestEvaluateMatchesBoundedBatchWithoutIO(t *testing.T) { + now := time.Date(2026, 8, 19, 0, 0, 0, 0, time.UTC) + ast, err := query.Parse("logs | where status >= 500 | limit 10", model.MaxRecords) + if err != nil { + t.Fatal(err) + } + rule := config.AgentAlertRule{Version: 1, ID: "rule-a", Revision: 1, StreamID: "requests", Query: "logs | where status >= 500 | limit 10", MinimumMatches: 2, AST: ast} + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 4, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + {Timestamp: now.Add(-time.Second), Name: "request", Attributes: map[string]string{"http.status_code": "503"}}, + {Timestamp: now, Name: "request", Attributes: map[string]string{"http.status_code": "502"}}, + }} + evaluation, err := Evaluate(rule, batch) + if err != nil || evaluation.State != "matched" || evaluation.Matches != 2 || !evaluation.WindowStart.Equal(now.Add(-time.Second)) || !evaluation.WindowEnd.Equal(now) || !evaluation.ObservedAt.Equal(now) { + t.Fatalf("evaluation=%+v err=%v", evaluation, err) + } + batch.Records[1].Attributes["http.status_code"] = "200" + evaluation, err = Evaluate(rule, batch) + if err != nil || evaluation.State != "clear" || evaluation.Matches != 1 { + t.Fatalf("evaluation=%+v err=%v", evaluation, err) + } +} + +func TestEdgeEvaluationMatchesCentralOracleForExactBatch(t *testing.T) { + ctx := context.Background() + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + now := time.Date(2026, 8, 19, 0, 5, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + queryText := "logs | where status >= 500 | limit 10" + ast, err := query.Parse(queryText, model.MaxRecords) + if err != nil { + t.Fatal(err) + } + saved, err := store.SaveQuery(ctx, storage.SavedQueryInput{OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", MaxRows: 100, Name: "Failures", Query: queryText, Scope: storage.ResourceScope{ProjectID: scope.ProjectID, EnvironmentID: scope.EnvironmentID, ServiceID: scope.ServiceID}}, now) + if err != nil { + t.Fatal(err) + } + _, err = store.SaveAlertRule(ctx, storage.AlertRuleInput{OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", SavedQueryID: saved.ID, Name: "Failures", Severity: "warning", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true}, now) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request", Attributes: map[string]string{"http.status_code": "503"}}, {Timestamp: now, Name: "request", Attributes: map[string]string{"http.status_code": "200"}}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + for { + report, projectErr := store.ProjectPending(ctx) + if projectErr != nil { + t.Fatal(projectErr) + } + if report.ProjectedSegments == 0 { + break + } + } + edge, err := Evaluate(config.AgentAlertRule{Version: 1, ID: "rule-a", Revision: 1, StreamID: "requests", Query: queryText, MinimumMatches: 1, AST: ast}, batch) + if err != nil { + t.Fatal(err) + } + central, err := store.EvaluateDueAlertRules(ctx, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 16 << 20, MaxMemoryBytes: 8 << 20}, now) + if err != nil || len(central) != 1 || (edge.State == "matched") != central[0].Matched || edge.Matches != central[0].Rows { + t.Fatalf("edge=%+v central=%+v err=%v", edge, central, err) + } +} + +func TestEvaluateReturnsBoundedErrorStateForTypedMismatch(t *testing.T) { + now := time.Now().UTC() + ast, err := query.Parse("logs | where status >= nope | limit 10", model.MaxRecords) + if err != nil { + t.Fatal(err) + } + rule := config.AgentAlertRule{Version: 1, ID: "rule-a", Revision: 1, StreamID: "requests", Query: "logs | where status >= nope | limit 10", MinimumMatches: 1, AST: ast} + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request", Attributes: map[string]string{"http.status_code": "503"}}}} + evaluation, err := Evaluate(rule, batch) + if err != nil || evaluation.State != "error" || evaluation.Matches != 0 { + t.Fatalf("evaluation=%+v err=%v", evaluation, err) + } +} diff --git a/internal/hostmetrics/hostmetrics.go b/internal/hostmetrics/hostmetrics.go new file mode 100644 index 0000000..fb48248 --- /dev/null +++ b/internal/hostmetrics/hostmetrics.go @@ -0,0 +1,487 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package hostmetrics + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "syscall" + "time" + "unicode/utf8" + + "gamertan.com/observatory/internal/model" +) + +const maxSourceBytes = 1 << 20 + +type Config struct { + ProcRoot string `json:"proc_root"` + CgroupRoot string `json:"cgroup_root,omitempty"` + Filesystems []Filesystem `json:"filesystems,omitempty"` + Processes []Process `json:"processes,omitempty"` + ControlGroups []Cgroup `json:"cgroups,omitempty"` +} + +type Filesystem struct { + Name string `json:"name"` + Path string `json:"path"` +} + +type Process struct { + Name string `json:"name"` + PIDFile string `json:"pid_file"` +} + +type Cgroup struct { + Name string `json:"name"` + Path string `json:"path"` +} + +func (configuration Config) Validate() error { + if runtime.GOOS != "linux" { + return errors.New("Linux metrics require Linux") + } + if err := absoluteClean("proc_root", configuration.ProcRoot); err != nil { + return err + } + if configuration.CgroupRoot != "" { + if err := absoluteClean("cgroup_root", configuration.CgroupRoot); err != nil { + return err + } + } + if len(configuration.Filesystems) > 32 || len(configuration.Processes) > 32 || len(configuration.ControlGroups) > 32 { + return errors.New("Linux metric selector limit exceeded") + } + names := map[string]bool{} + for _, filesystem := range configuration.Filesystems { + if err := selectorName(filesystem.Name, names); err != nil { + return fmt.Errorf("filesystem: %w", err) + } + if err := absoluteClean("filesystem path", filesystem.Path); err != nil { + return err + } + } + for _, process := range configuration.Processes { + if err := selectorName(process.Name, names); err != nil { + return fmt.Errorf("process: %w", err) + } + if err := absoluteClean("pid_file", process.PIDFile); err != nil { + return err + } + } + for _, group := range configuration.ControlGroups { + if err := selectorName(group.Name, names); err != nil { + return fmt.Errorf("cgroup: %w", err) + } + if configuration.CgroupRoot == "" { + return errors.New("cgroup_root is required when cgroups are selected") + } + if group.Path == "" || filepath.IsAbs(group.Path) || filepath.Clean(group.Path) != group.Path || group.Path == "." || strings.HasPrefix(group.Path, ".."+string(os.PathSeparator)) { + return errors.New("cgroup path must be a clean relative path below cgroup_root") + } + } + return nil +} + +func Collect(configuration Config, now time.Time) ([]model.Observation, error) { + if err := configuration.Validate(); err != nil { + return nil, err + } + if now.IsZero() { + return nil, errors.New("collection time is required") + } + if err := secureDirectory(configuration.ProcRoot); err != nil { + return nil, errors.New("proc_root is unavailable") + } + var observations []model.Observation + var problems []error + appendResult := func(result []model.Observation, err error) { + observations = append(observations, result...) + if err != nil { + problems = append(problems, err) + } + } + result, err := collectStat(configuration.ProcRoot, now) + appendResult(result, err) + result, err = collectMemory(configuration.ProcRoot, now) + appendResult(result, err) + result, err = collectLoad(configuration.ProcRoot, now) + appendResult(result, err) + result, err = collectNetwork(configuration.ProcRoot, now) + appendResult(result, err) + for _, filesystem := range configuration.Filesystems { + result, err = collectFilesystem(filesystem, now) + appendResult(result, err) + } + for _, process := range configuration.Processes { + result, err = collectProcess(configuration.ProcRoot, process, now) + appendResult(result, err) + } + for _, group := range configuration.ControlGroups { + result, err = collectCgroup(configuration.CgroupRoot, group, now) + appendResult(result, err) + } + if len(observations) > model.MaxRecords { + return nil, errors.New("Linux metric record limit exceeded") + } + return observations, errors.Join(problems...) +} + +func collectStat(root string, now time.Time) ([]model.Observation, error) { + body, err := readBounded(filepath.Join(root, "stat")) + if err != nil { + return nil, errors.New("read proc stat") + } + var observations []model.Observation + for _, line := range strings.Split(string(body), "\n") { + fields := strings.Fields(line) + if len(fields) < 5 || fields[0] != "cpu" { + continue + } + states := []string{"user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", "guest", "guest_nice"} + for index := 1; index < len(fields) && index <= len(states); index++ { + value, parseErr := strconv.ParseFloat(fields[index], 64) + if parseErr != nil { + return nil, errors.New("proc stat contains an invalid CPU counter") + } + observations = append(observations, metric(now, "system.cpu.time_ticks", value, "ticks", map[string]string{"state": states[index-1]})) + } + break + } + uptime, err := readBounded(filepath.Join(root, "uptime")) + if err == nil { + fields := strings.Fields(string(uptime)) + if len(fields) > 0 { + if value, parseErr := strconv.ParseFloat(fields[0], 64); parseErr == nil { + observations = append(observations, metric(now, "system.uptime", value, "seconds", nil)) + } + } + } + if len(observations) == 0 { + return nil, errors.New("proc stat contains no aggregate CPU record") + } + return observations, nil +} + +func collectMemory(root string, now time.Time) ([]model.Observation, error) { + body, err := readBounded(filepath.Join(root, "meminfo")) + if err != nil { + return nil, errors.New("read proc meminfo") + } + wanted := map[string]string{"MemTotal": "total", "MemAvailable": "available", "SwapTotal": "swap_total", "SwapFree": "swap_free"} + var observations []model.Observation + for _, line := range strings.Split(string(body), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + state, ok := wanted[strings.TrimSuffix(fields[0], ":")] + if !ok { + continue + } + value, parseErr := strconv.ParseFloat(fields[1], 64) + if parseErr != nil { + return nil, errors.New("proc meminfo contains an invalid value") + } + if len(fields) > 2 && fields[2] == "kB" { + value *= 1024 + } + observations = append(observations, metric(now, "system.memory", value, "bytes", map[string]string{"state": state})) + } + if len(observations) != len(wanted) { + return observations, errors.New("proc meminfo is missing required values") + } + return observations, nil +} + +func collectLoad(root string, now time.Time) ([]model.Observation, error) { + body, err := readBounded(filepath.Join(root, "loadavg")) + if err != nil { + return nil, errors.New("read proc loadavg") + } + fields := strings.Fields(string(body)) + if len(fields) < 3 { + return nil, errors.New("proc loadavg is incomplete") + } + periods := []string{"1m", "5m", "15m"} + observations := make([]model.Observation, 0, 3) + for index := range periods { + value, parseErr := strconv.ParseFloat(fields[index], 64) + if parseErr != nil { + return nil, errors.New("proc loadavg contains an invalid value") + } + observations = append(observations, metric(now, "system.load.average", value, "1", map[string]string{"period": periods[index]})) + } + return observations, nil +} + +func collectNetwork(root string, now time.Time) ([]model.Observation, error) { + body, err := readBounded(filepath.Join(root, "net", "dev")) + if err != nil { + return nil, errors.New("read proc network counters") + } + var observations []model.Observation + for _, line := range strings.Split(string(body), "\n") { + separator := strings.IndexByte(line, ':') + if separator < 0 { + continue + } + name := strings.TrimSpace(line[:separator]) + if !safeLabel(name) { + return nil, errors.New("proc network interface name is invalid") + } + fields := strings.Fields(line[separator+1:]) + if len(fields) != 16 { + return nil, errors.New("proc network counter record is invalid") + } + indexes := []struct { + field int + name string + dir string + }{{0, "system.network.bytes", "receive"}, {1, "system.network.packets", "receive"}, {3, "system.network.dropped_packets", "receive"}, {8, "system.network.bytes", "transmit"}, {9, "system.network.packets", "transmit"}, {11, "system.network.dropped_packets", "transmit"}} + for _, selected := range indexes { + value, parseErr := strconv.ParseFloat(fields[selected.field], 64) + if parseErr != nil { + return nil, errors.New("proc network counter is invalid") + } + unit := "1" + if selected.name == "system.network.bytes" { + unit = "bytes" + } + observations = append(observations, metric(now, selected.name, value, unit, map[string]string{"interface": name, "direction": selected.dir})) + } + if len(observations) > 128*6 { + return nil, errors.New("proc network interface limit exceeded") + } + } + return observations, nil +} + +func collectFilesystem(selected Filesystem, now time.Time) ([]model.Observation, error) { + info, err := os.Lstat(selected.Path) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("filesystem %s is unavailable", selected.Name) + } + var status syscall.Statfs_t + if err = syscall.Statfs(selected.Path, &status); err != nil { + return nil, fmt.Errorf("inspect filesystem %s", selected.Name) + } + blockSize := float64(status.Bsize) + attributes := map[string]string{"filesystem": selected.Name} + return []model.Observation{ + metric(now, "system.filesystem.bytes", float64(status.Blocks)*blockSize, "bytes", with(attributes, "state", "total")), + metric(now, "system.filesystem.bytes", float64(status.Bavail)*blockSize, "bytes", with(attributes, "state", "available")), + metric(now, "system.filesystem.inodes", float64(status.Files), "1", with(attributes, "state", "total")), + metric(now, "system.filesystem.inodes", float64(status.Ffree), "1", with(attributes, "state", "free")), + }, nil +} + +func collectProcess(procRoot string, selected Process, now time.Time) ([]model.Observation, error) { + pidBody, err := readBounded(selected.PIDFile) + if err != nil { + return []model.Observation{metric(now, "process.up", 0, "1", map[string]string{"process": selected.Name})}, fmt.Errorf("process %s PID is unavailable", selected.Name) + } + pidText := strings.TrimSpace(string(pidBody)) + pid, err := strconv.ParseUint(pidText, 10, 31) + if err != nil || pid == 0 { + return []model.Observation{metric(now, "process.up", 0, "1", map[string]string{"process": selected.Name})}, fmt.Errorf("process %s PID is invalid", selected.Name) + } + stat, err := readBounded(filepath.Join(procRoot, pidText, "stat")) + if err != nil { + return []model.Observation{metric(now, "process.up", 0, "1", map[string]string{"process": selected.Name})}, fmt.Errorf("process %s is unavailable", selected.Name) + } + closing := strings.LastIndexByte(string(stat), ')') + if closing < 2 { + return nil, fmt.Errorf("process %s stat is invalid", selected.Name) + } + fields := strings.Fields(string(stat)[closing+1:]) + // fields begin with state (field 3); utime, stime, starttime, vsize, rss + // are therefore indexes 11, 12, 19, 20, and 21 in this slice. + if len(fields) < 22 { + return nil, fmt.Errorf("process %s stat is incomplete", selected.Name) + } + values := make([]float64, 5) + for index, field := range []int{11, 12, 19, 20, 21} { + values[index], err = strconv.ParseFloat(fields[field], 64) + if err != nil { + return nil, fmt.Errorf("process %s stat contains an invalid counter", selected.Name) + } + } + pageSize := float64(os.Getpagesize()) + base := map[string]string{"process": selected.Name} + observations := []model.Observation{ + metric(now, "process.up", 1, "1", base), + metric(now, "process.start_time_ticks", values[2], "ticks", base), + metric(now, "process.cpu.time_ticks", values[0], "ticks", with(base, "state", "user")), + metric(now, "process.cpu.time_ticks", values[1], "ticks", with(base, "state", "system")), + metric(now, "process.memory.virtual", values[3], "bytes", base), + metric(now, "process.memory.resident", values[4]*pageSize, "bytes", base), + } + if ioBody, ioErr := readBounded(filepath.Join(procRoot, pidText, "io")); ioErr == nil { + for _, line := range strings.Split(string(ioBody), "\n") { + fields := strings.Fields(line) + if len(fields) != 2 || fields[0] != "read_bytes:" && fields[0] != "write_bytes:" { + continue + } + value, parseErr := strconv.ParseFloat(fields[1], 64) + if parseErr != nil { + continue + } + direction := strings.TrimSuffix(fields[0], "_bytes:") + observations = append(observations, metric(now, "process.io.bytes", value, "bytes", with(base, "direction", direction))) + } + } + return observations, nil +} + +func collectCgroup(root string, selected Cgroup, now time.Time) ([]model.Observation, error) { + directory, err := secureRelativeDirectory(root, selected.Path) + if err != nil { + return []model.Observation{metric(now, "cgroup.up", 0, "1", map[string]string{"cgroup": selected.Name})}, fmt.Errorf("cgroup %s is unavailable", selected.Name) + } + base := map[string]string{"cgroup": selected.Name} + observations := []model.Observation{metric(now, "cgroup.up", 1, "1", base)} + for _, scalar := range []struct { + file, name, unit, state string + }{{"memory.current", "cgroup.memory", "bytes", "current"}, {"memory.peak", "cgroup.memory", "bytes", "peak"}, {"memory.swap.current", "cgroup.memory", "bytes", "swap"}, {"pids.current", "cgroup.pids", "1", "current"}} { + body, readErr := readBounded(filepath.Join(directory, scalar.file)) + if readErr != nil { + continue + } + value, parseErr := strconv.ParseFloat(strings.TrimSpace(string(body)), 64) + if parseErr == nil { + observations = append(observations, metric(now, scalar.name, value, scalar.unit, with(base, "state", scalar.state))) + } + } + if body, readErr := readBounded(filepath.Join(directory, "cpu.stat")); readErr == nil { + for _, line := range strings.Split(string(body), "\n") { + fields := strings.Fields(line) + if len(fields) != 2 || fields[0] != "usage_usec" && fields[0] != "user_usec" && fields[0] != "system_usec" { + continue + } + value, parseErr := strconv.ParseFloat(fields[1], 64) + if parseErr == nil { + observations = append(observations, metric(now, "cgroup.cpu.time", value, "microseconds", with(base, "state", strings.TrimSuffix(fields[0], "_usec")))) + } + } + } + if body, readErr := readBounded(filepath.Join(directory, "io.stat")); readErr == nil { + var readBytes, writeBytes float64 + for _, line := range strings.Split(string(body), "\n") { + for _, field := range strings.Fields(line) { + key, text, found := strings.Cut(field, "=") + if !found || key != "rbytes" && key != "wbytes" { + continue + } + value, parseErr := strconv.ParseFloat(text, 64) + if parseErr != nil { + continue + } + if key == "rbytes" { + readBytes += value + } else { + writeBytes += value + } + } + } + observations = append(observations, + metric(now, "cgroup.io.bytes", readBytes, "bytes", with(base, "direction", "read")), + metric(now, "cgroup.io.bytes", writeBytes, "bytes", with(base, "direction", "write")), + ) + } + return observations, nil +} + +func metric(now time.Time, name string, value float64, unit string, attributes map[string]string) model.Observation { + copied := with(attributes, "unit", unit) + return model.Observation{Timestamp: now.UTC(), Name: name, Value: &value, Attributes: copied} +} + +func with(source map[string]string, key, value string) map[string]string { + result := make(map[string]string, len(source)+1) + for existingKey, existingValue := range source { + result[existingKey] = existingValue + } + result[key] = value + return result +} + +func readBounded(path string) ([]byte, error) { + before, err := os.Lstat(path) + if err != nil || !before.Mode().IsRegular() || before.Mode()&os.ModeSymlink != 0 || before.Size() > maxSourceBytes { + return nil, errors.New("metric source is not a bounded regular file") + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + after, err := file.Stat() + if err != nil || !after.Mode().IsRegular() || !os.SameFile(before, after) { + return nil, errors.New("metric source changed during open") + } + body, err := io.ReadAll(io.LimitReader(bufio.NewReader(file), maxSourceBytes+1)) + if err != nil || len(body) > maxSourceBytes || !utf8.Valid(body) || strings.IndexByte(string(body), 0) >= 0 { + return nil, errors.New("metric source exceeds accepted bounds") + } + return body, nil +} + +func secureDirectory(path string) error { + info, err := os.Lstat(path) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("path is not a non-symlink directory") + } + return nil +} + +func secureRelativeDirectory(root, relative string) (string, error) { + if err := secureDirectory(root); err != nil { + return "", err + } + current := root + for _, part := range strings.Split(filepath.ToSlash(relative), "/") { + current = filepath.Join(current, part) + if err := secureDirectory(current); err != nil { + return "", err + } + } + return current, nil +} + +func absoluteClean(label, value string) error { + if !filepath.IsAbs(value) || filepath.Clean(value) != value { + return fmt.Errorf("%s must be absolute and clean", label) + } + return nil +} + +func selectorName(value string, names map[string]bool) error { + if !safeLabel(value) { + return errors.New("selector name is invalid") + } + if names[value] { + return errors.New("selector name is duplicated") + } + names[value] = true + return nil +} + +func safeLabel(value string) bool { + if value == "" || len(value) > 128 { + return false + } + for _, character := range value { + if !(character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || strings.ContainsRune("._-", character)) { + return false + } + } + return true +} diff --git a/internal/hostmetrics/hostmetrics_test.go b/internal/hostmetrics/hostmetrics_test.go new file mode 100644 index 0000000..d4e76e2 --- /dev/null +++ b/internal/hostmetrics/hostmetrics_test.go @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package hostmetrics + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/model" +) + +func TestCollectReadsOnlyConfiguredLinuxEvidence(t *testing.T) { + root := t.TempDir() + proc := filepath.Join(root, "proc") + cgroup := filepath.Join(root, "cgroup") + filesystem := filepath.Join(root, "filesystem") + pidFile := filepath.Join(root, "service.pid") + for _, directory := range []string{filepath.Join(proc, "net"), filepath.Join(proc, "123"), filepath.Join(cgroup, "system.slice", "example.service"), filesystem} { + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + } + write := func(path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(proc, "stat"), "cpu 100 2 30 400 5 6 7 8 9 10\n") + write(filepath.Join(proc, "uptime"), "99.5 80.0\n") + write(filepath.Join(proc, "meminfo"), "MemTotal: 1000 kB\nMemAvailable: 750 kB\nSwapTotal: 200 kB\nSwapFree: 150 kB\n") + write(filepath.Join(proc, "loadavg"), "0.10 0.20 0.30 1/100 123\n") + write(filepath.Join(proc, "net", "dev"), "Inter-| Receive | Transmit\nlo: 100 2 0 1 0 0 0 0 200 3 0 2 0 0 0 0\n") + write(pidFile, "123\n") + write(filepath.Join(proc, "123", "stat"), "123 (example worker) S 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23\n") + write(filepath.Join(proc, "123", "io"), "read_bytes: 4096\nwrite_bytes: 8192\n") + group := filepath.Join(cgroup, "system.slice", "example.service") + write(filepath.Join(group, "memory.current"), "1024\n") + write(filepath.Join(group, "memory.peak"), "2048\n") + write(filepath.Join(group, "memory.swap.current"), "0\n") + write(filepath.Join(group, "pids.current"), "3\n") + write(filepath.Join(group, "cpu.stat"), "usage_usec 300\nuser_usec 200\nsystem_usec 100\n") + write(filepath.Join(group, "io.stat"), "8:0 rbytes=100 wbytes=200 rios=1 wios=2\n8:1 rbytes=300 wbytes=400 rios=3 wios=4\n") + + configuration := Config{ + ProcRoot: proc, CgroupRoot: cgroup, + Filesystems: []Filesystem{{Name: "data", Path: filesystem}}, + Processes: []Process{{Name: "web", PIDFile: pidFile}}, + ControlGroups: []Cgroup{{Name: "web-service", Path: filepath.Join("system.slice", "example.service")}}, + } + now := time.Date(2026, 8, 17, 6, 0, 0, 0, time.UTC) + observations, err := Collect(configuration, now) + if err != nil { + t.Fatal(err) + } + wanted := map[string]bool{ + "system.cpu.time_ticks": false, "system.memory": false, "system.load.average": false, + "system.network.bytes": false, "system.filesystem.bytes": false, "process.up": false, + "process.start_time_ticks": false, "process.io.bytes": false, "cgroup.up": false, "cgroup.io.bytes": false, + } + for _, observation := range observations { + if _, ok := wanted[observation.Name]; ok { + wanted[observation.Name] = true + } + for _, value := range observation.Attributes { + if strings.Contains(value, root) { + t.Fatalf("local path leaked in attributes: %+v", observation) + } + } + if _, found := observation.Attributes["start_ticks"]; found { + t.Fatalf("dynamic process identity leaked into metric attributes: %+v", observation) + } + } + for name, found := range wanted { + if !found { + t.Errorf("missing %s", name) + } + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source", StreamID: "host-metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: observations} + if err = batch.Validate(now); err != nil { + t.Fatalf("collected batch: %v", err) + } +} + +func TestCollectRejectsSymlinkMetricSource(t *testing.T) { + proc := createMinimalProc(t) + target := filepath.Join(t.TempDir(), "pid") + if err := os.WriteFile(target, []byte("123\n"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "selected.pid") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + observations, err := Collect(Config{ProcRoot: proc, Processes: []Process{{Name: "selected", PIDFile: link}}}, time.Date(2026, 8, 17, 6, 0, 0, 0, time.UTC)) + if err == nil { + t.Fatal("symlink PID source accepted without a collection warning") + } + for _, observation := range observations { + if observation.Name == "process.up" && observation.Value != nil && *observation.Value == 0 { + return + } + } + t.Fatal("rejected symlink source did not retain process.up=0 evidence") +} + +func TestCollectReportsMissingSelectedProcessWithoutDroppingHostMetrics(t *testing.T) { + proc := createMinimalProc(t) + now := time.Date(2026, 8, 17, 6, 0, 0, 0, time.UTC) + observations, err := Collect(Config{ProcRoot: proc, Processes: []Process{{Name: "missing", PIDFile: filepath.Join(t.TempDir(), "missing.pid")}}}, now) + if err == nil || len(observations) == 0 { + t.Fatalf("observations=%d err=%v", len(observations), err) + } + foundDown := false + for _, observation := range observations { + if observation.Name == "process.up" && observation.Value != nil && *observation.Value == 0 { + foundDown = true + } + } + if !foundDown { + t.Fatal("missing process did not emit process.up=0") + } +} + +func TestValidationRejectsEscapingAndDuplicateSelectors(t *testing.T) { + root := t.TempDir() + for _, configuration := range []Config{ + {ProcRoot: "relative"}, + {ProcRoot: root, CgroupRoot: root, ControlGroups: []Cgroup{{Name: "bad", Path: "../escape"}}}, + {ProcRoot: root, Filesystems: []Filesystem{{Name: "same", Path: root}}, Processes: []Process{{Name: "same", PIDFile: filepath.Join(root, "pid")}}}, + } { + if err := configuration.Validate(); err == nil { + t.Fatalf("invalid configuration accepted: %+v", configuration) + } + } +} + +func createMinimalProc(t *testing.T) string { + t.Helper() + proc := filepath.Join(t.TempDir(), "proc") + if err := os.MkdirAll(filepath.Join(proc, "net"), 0o700); err != nil { + t.Fatal(err) + } + files := map[string]string{ + "stat": "cpu 1 1 1 1\n", "uptime": "1 1\n", + "meminfo": "MemTotal: 1 kB\nMemAvailable: 1 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n", + "loadavg": "0 0 0 1/1 1\n", filepath.Join("net", "dev"): "Inter-| Receive | Transmit\n", + } + for name, body := range files { + if err := os.WriteFile(filepath.Join(proc, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + return proc +} diff --git a/internal/httpserver/explore.go b/internal/httpserver/explore.go new file mode 100644 index 0000000..a9cdb50 --- /dev/null +++ b/internal/httpserver/explore.go @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "time" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/site" + "gamertan.com/web/access" + "gamertan.com/web/auth" + "gamertan.com/web/authhttp" +) + +const defaultExploreQuery = "logs | window 1h | limit 50" + +func (s *Server) explorePage(w http.ResponseWriter, r *http.Request) { + view, _, ok := s.exploreView(w, r, defaultExploreQuery) + if !ok { + return + } + s.renderHTML(w, r, http.StatusOK, site.Explore(view)) +} + +func (s *Server) exploreForm(w http.ResponseWriter, r *http.Request) { + values, err := readForm(w, r, 20<<10, "csrf_token", "query") + queryText := defaultExploreQuery + if err == nil { + queryText = values.Get("query") + } + view, token, ok := s.exploreView(w, r, queryText) + if !ok { + return + } + csrfOK := err == nil && authhttp.VerifyCSRF(token, "query:execute", values.Get("csrf_token")) + if !tokenBoundFormRequest(r, s.options.PublicOrigin, csrfOK) { + view.ErrorMessage = "This query form expired or could not be verified. Please try again." + s.renderHTML(w, r, http.StatusForbidden, site.Explore(view)) + return + } + if err != nil { + view.ErrorMessage = "The query form was not accepted." + s.renderHTML(w, r, http.StatusBadRequest, site.Explore(view)) + return + } + ast, err := query.Parse(queryText, s.options.MaxQueryRows) + if err != nil { + view.ErrorMessage = "The query could not be parsed. Check its stages, values, window, and limit." + s.renderHTML(w, r, http.StatusUnprocessableEntity, site.Explore(view)) + return + } + principal, _ := auth.PrincipalFromContext(r.Context()) + scope := access.Scope{OrganizationID: view.Organization.ID} + sensitive, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionTelemetryReadSensitive) + if err != nil { + view.ErrorMessage = "Authorization is temporarily unavailable." + s.renderHTML(w, r, http.StatusServiceUnavailable, site.Explore(view)) + return + } + result, err := s.store.Query(r.Context(), ast, query.Scope{OrganizationID: view.Organization.ID, Sensitive: sensitive.Allowed}, s.options.QueryBudget, s.now()) + switch { + case errors.Is(err, query.ErrSensitivePermissionRequired): + view.ErrorMessage = "This query requires permission to read sensitive fields." + s.renderHTML(w, r, http.StatusForbidden, site.Explore(view)) + return + case errors.Is(err, query.ErrBudgetExceeded): + view.ErrorMessage = "This query exceeded its execution budget. Narrow the time window, fields, or result limit." + s.renderHTML(w, r, http.StatusUnprocessableEntity, site.Explore(view)) + return + case errors.Is(err, query.ErrTypeMismatch): + view.ErrorMessage = "A query value did not match the selected field type." + s.renderHTML(w, r, http.StatusUnprocessableEntity, site.Explore(view)) + return + case err != nil: + view.ErrorMessage = "The bounded query is temporarily unavailable. Your query text remains here to retry." + s.renderHTML(w, r, http.StatusServiceUnavailable, site.Explore(view)) + return + } + view.Executed = true + view.Table = resultTable("Authorized query results", result) + view.Table.Empty = "No observations matched this query." + view.Stats = site.QueryStatsView{ + ScannedRows: result.Stats.ScannedRows, MatchedRows: result.Stats.MatchedRows, + ScannedBytes: formatQueryBytes(result.Stats.ScannedBytes), + Duration: formatQueryDuration(result.Stats.DurationNS), + Truncated: result.Stats.Truncated, Approximate: result.Stats.Approximate, + } + s.renderHTML(w, r, http.StatusOK, site.Explore(view)) +} + +func (s *Server) exploreView(w http.ResponseWriter, r *http.Request, queryText string) (site.ExploreView, string, bool) { + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok { + http.Redirect(w, r, "/login/", http.StatusSeeOther) + return site.ExploreView{}, "", false + } + values := r.URL.Query() + organizationID := values.Get("organization") + if len(values) != 1 || len(values["organization"]) != 1 || organizationID == "" { + writeProblem(w, http.StatusBadRequest, "organization is required") + return site.ExploreView{}, "", false + } + organizations, err := s.identity.OrganizationsForUser(r.Context(), principal.User.ID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "organization list unavailable") + return site.ExploreView{}, "", false + } + organizationName := "" + for _, organization := range organizations { + if organization.ID == organizationID { + organizationName = organization.Name + break + } + } + if organizationName == "" { + writeProblem(w, http.StatusForbidden, "organization access denied") + return site.ExploreView{}, "", false + } + scope := access.Scope{OrganizationID: organizationID} + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionTelemetryQuery) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "authorization unavailable") + return site.ExploreView{}, "", false + } + if !decision.Allowed { + writeProblem(w, http.StatusForbidden, "telemetry query access denied") + return site.ExploreView{}, "", false + } + if err = s.identity.ValidateResourceScope(r.Context(), scope); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid organization") + return site.ExploreView{}, "", false + } + token, ok := authhttp.SessionToken(r, s.cookie) + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return site.ExploreView{}, "", false + } + csrf, err := authhttp.CSRFToken(token, "query:execute") + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return site.ExploreView{}, "", false + } + view := site.ExploreView{ + Head: s.head("Explore — Gamertan Observatory", "Run an authorized, bounded query against organization evidence.", "/app/explore/"), + DisplayName: principal.User.DisplayName, + Organization: site.OrganizationOption{ID: organizationID, Name: organizationName, Selected: true}, + Query: queryText, CSRFToken: csrf, + EventsURL: "/app/events?organization=" + url.QueryEscape(organizationID), + } + return view, token, true +} + +func formatQueryBytes(value int64) string { + if value < 1024 { + return fmt.Sprintf("%d B", value) + } + units := []string{"KiB", "MiB", "GiB", "TiB"} + amount := float64(value) + for _, unit := range units { + amount /= 1024 + if amount < 1024 || unit == units[len(units)-1] { + return fmt.Sprintf("%.1f %s", amount, unit) + } + } + return fmt.Sprintf("%d B", value) +} + +func formatQueryDuration(nanoseconds int64) string { + duration := time.Duration(nanoseconds) + if duration < time.Microsecond { + return duration.String() + } + if duration < time.Millisecond { + return duration.Round(time.Microsecond).String() + } + return duration.Round(time.Millisecond).String() +} diff --git a/internal/httpserver/incidents.go b/internal/httpserver/incidents.go new file mode 100644 index 0000000..46e3950 --- /dev/null +++ b/internal/httpserver/incidents.go @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "context" + "net/http" + "net/url" + "strconv" + "time" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/site" + "gamertan.com/observatory/internal/storage" + "gamertan.com/web/access" + "gamertan.com/web/auth" + "gamertan.com/web/authhttp" +) + +// EvaluateAlerts performs one bounded due-rule pass. It publishes only a +// generic organization invalidation signal when an incident changed; +// telemetry and incident details never enter the SSE stream. +func (s *Server) EvaluateAlerts(ctx context.Context) (int, error) { + evaluations, err := s.store.EvaluateDueAlertRules(ctx, s.options.QueryBudget, s.now()) + if err != nil { + return 0, err + } + organizations := map[string]struct{}{} + for _, evaluation := range evaluations { + if evaluation.IncidentChanged { + organizations[evaluation.OrganizationID] = struct{}{} + } + if evaluation.IncidentChanged && evaluation.IncidentState == "firing" && s.options.PushDispatcher != nil { + s.options.PushDispatcher.Enqueue(evaluation.OrganizationID) + } + } + for organizationID := range organizations { + s.refresh.publish(organizationID) + } + return len(evaluations), nil +} + +func (s *Server) incidentInbox(w http.ResponseWriter, r *http.Request) { + principal, organizationID, organizationName, ok := s.authorizeIncidentRead(w, r) + if !ok { + return + } + scope := access.Scope{OrganizationID: organizationID} + incidents, err := s.store.Incidents(r.Context(), organizationID, true, 100) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "incidents unavailable") + return + } + rules, err := s.store.AlertRules(r.Context(), organizationID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "alert rules unavailable") + return + } + saved, err := s.store.SavedQueries(r.Context(), organizationID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "saved queries unavailable") + return + } + view := site.IncidentInboxView{ + Head: s.head("Incident inbox — Gamertan Observatory", "Authorized incident response and bounded alert rules.", "/app/incidents/"), + DisplayName: principal.User.DisplayName, + Organization: site.OrganizationOption{ID: organizationID, Name: organizationName, Selected: true}, + EventsURL: "/app/events?organization=" + url.QueryEscape(organizationID), + OfflineURL: "/app/incidents/offline/?organization=" + url.QueryEscape(organizationID), + CacheKey: "/app/incidents/?organization=" + url.QueryEscape(organizationID), + } + if s.options.PushDispatcher != nil { + token, sessionOK := authhttp.SessionToken(r, s.cookie) + if !sessionOK { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return + } + view.PushPublicKey = s.options.PushPublicKey + view.PushCSRF, err = authhttp.CSRFToken(token, "push:manage") + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + } + for _, incident := range incidents { + item := site.IncidentSummary{ID: incident.ID, Title: incident.Title, State: incident.State, Severity: incident.Severity, StartedAt: incident.StartedAt.Format("2006-01-02 15:04:05 UTC"), UpdatedAt: incident.UpdatedAt.Format("2006-01-02 15:04:05 UTC")} + if incident.SilencedUntil != nil { + item.SilencedUntil = incident.SilencedUntil.Format("2006-01-02 15:04:05 UTC") + } + view.Incidents = append(view.Incidents, item) + if incident.State != "resolved" { + view.OpenCount++ + } + } + for _, rule := range rules { + item := site.AlertRuleSummary{Name: rule.Name, Description: rule.Description, Severity: rule.Severity, Enabled: rule.Enabled, Interval: rule.EvaluationInterval.String(), LastError: rule.LastError} + if rule.LastEvaluatedAt != nil { + item.LastEvaluatedAt = rule.LastEvaluatedAt.Format("2006-01-02 15:04:05 UTC") + } + view.Rules = append(view.Rules, item) + } + for _, savedQuery := range saved { + view.SavedQueries = append(view.SavedQueries, site.SavedQuerySummary{ID: savedQuery.ID, Name: savedQuery.Name, Description: savedQuery.Description, Query: savedQuery.Query}) + } + manage, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionIncidentsManage) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "authorization unavailable") + return + } + view.CanManage = manage.Allowed + if view.CanManage { + token, sessionOK := authhttp.SessionToken(r, s.cookie) + if !sessionOK { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return + } + view.ManageCSRF, err = authhttp.CSRFToken(token, "incidents:manage") + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + } + s.renderHTML(w, r, http.StatusOK, site.IncidentInbox(view)) +} + +func (s *Server) offlineIncidentInbox(w http.ResponseWriter, r *http.Request) { + _, organizationID, organizationName, ok := s.authorizeIncidentRead(w, r) + if !ok { + return + } + incidents, err := s.store.Incidents(r.Context(), organizationID, false, 100) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "incidents unavailable") + return + } + view := site.OfflineIncidentView{ + Head: s.head("Saved incident inbox — Gamertan Observatory", "A deliberately saved read-only incident snapshot.", "/app/incidents/"), + Organization: site.OrganizationOption{ID: organizationID, Name: organizationName, Selected: true}, + CapturedAt: s.now().Format("2006-01-02 15:04:05 UTC"), + } + for _, incident := range incidents { + item := site.IncidentSummary{Title: incident.Title, State: incident.State, Severity: incident.Severity, StartedAt: incident.StartedAt.Format("2006-01-02 15:04:05 UTC"), UpdatedAt: incident.UpdatedAt.Format("2006-01-02 15:04:05 UTC")} + if incident.SilencedUntil != nil { + item.SilencedUntil = incident.SilencedUntil.Format("2006-01-02 15:04:05 UTC") + } + view.Incidents = append(view.Incidents, item) + } + s.renderHTML(w, r, http.StatusOK, site.OfflineIncidentInbox(view)) +} + +func (s *Server) authorizeIncidentRead(w http.ResponseWriter, r *http.Request) (auth.Principal, string, string, bool) { + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok { + http.Redirect(w, r, "/login/", http.StatusSeeOther) + return auth.Principal{}, "", "", false + } + values := r.URL.Query() + organizationValues, exists := values["organization"] + if !exists || len(values) != 1 || len(organizationValues) != 1 || organizationValues[0] == "" { + writeProblem(w, http.StatusBadRequest, "incident organization is required") + return auth.Principal{}, "", "", false + } + organizationID := organizationValues[0] + scope := access.Scope{OrganizationID: organizationID} + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionIncidentsRead) + if err != nil || !decision.Allowed { + writeProblem(w, http.StatusForbidden, "incident access denied") + return auth.Principal{}, "", "", false + } + if err = s.identity.ValidateResourceScope(r.Context(), scope); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid organization") + return auth.Principal{}, "", "", false + } + organizations, err := s.identity.OrganizationsForUser(r.Context(), principal.User.ID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "organization unavailable") + return auth.Principal{}, "", "", false + } + for _, organization := range organizations { + if organization.ID == organizationID { + return principal, organizationID, organization.Name, true + } + } + writeProblem(w, http.StatusForbidden, "incident access denied") + return auth.Principal{}, "", "", false +} + +func (s *Server) createAlertRule(w http.ResponseWriter, r *http.Request) { + values, principal, ok := s.authorizeIncidentForm(w, r, []string{"organization_id", "csrf_token", "name", "description", "saved_query_id", "severity", "minimum_matches", "required_consecutive", "evaluation_interval"}, nil) + if !ok { + return + } + minimumMatches, err := strconv.Atoi(values.Get("minimum_matches")) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "alert rule rejected") + return + } + requiredConsecutive, err := strconv.Atoi(values.Get("required_consecutive")) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "alert rule rejected") + return + } + intervals := map[string]time.Duration{"15s": 15 * time.Second, "30s": 30 * time.Second, "1m": time.Minute, "5m": 5 * time.Minute, "15m": 15 * time.Minute} + interval, exists := intervals[values.Get("evaluation_interval")] + if !exists { + writeProblem(w, http.StatusUnprocessableEntity, "alert rule rejected") + return + } + _, err = s.store.SaveAlertRule(r.Context(), storage.AlertRuleInput{ + OrganizationID: values.Get("organization_id"), Name: values.Get("name"), Description: values.Get("description"), + SavedQueryID: values.Get("saved_query_id"), Severity: values.Get("severity"), MinimumMatches: minimumMatches, + RequiredConsecutive: requiredConsecutive, EvaluationInterval: interval, Enabled: true, ActorUserID: principal.User.ID, + }, s.now()) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "alert rule rejected") + return + } + http.Redirect(w, r, "/app/incidents/?organization="+url.QueryEscape(values.Get("organization_id")), http.StatusSeeOther) +} + +func (s *Server) transitionIncident(w http.ResponseWriter, r *http.Request) { + values, principal, ok := s.authorizeIncidentForm(w, r, []string{"organization_id", "csrf_token", "action"}, []string{"silence_duration"}) + if !ok { + return + } + var silenceUntil *time.Time + if values.Get("action") == "silence" { + durations := map[string]time.Duration{"15m": 15 * time.Minute, "1h": time.Hour, "6h": 6 * time.Hour, "24h": 24 * time.Hour, "168h": 7 * 24 * time.Hour} + duration, exists := durations[values.Get("silence_duration")] + if !exists { + writeProblem(w, http.StatusUnprocessableEntity, "incident transition rejected") + return + } + until := s.now().Add(duration) + silenceUntil = &until + } else if values.Get("silence_duration") != "" { + writeProblem(w, http.StatusUnprocessableEntity, "incident transition rejected") + return + } + _, err := s.store.TransitionIncident(r.Context(), values.Get("organization_id"), r.PathValue("id"), values.Get("action"), principal.User.ID, silenceUntil, s.now()) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "incident transition rejected") + return + } + s.refresh.publish(values.Get("organization_id")) + http.Redirect(w, r, "/app/incidents/?organization="+url.QueryEscape(values.Get("organization_id")), http.StatusSeeOther) +} + +func (s *Server) authorizeIncidentForm(w http.ResponseWriter, r *http.Request, required, optional []string) (url.Values, auth.Principal, bool) { + principal, ok := auth.PrincipalFromContext(r.Context()) + values, err := readFormFields(w, r, 24<<10, required, optional) + token, sessionOK := authhttp.SessionToken(r, s.cookie) + csrfOK := err == nil && sessionOK && authhttp.VerifyCSRF(token, "incidents:manage", values.Get("csrf_token")) + if !tokenBoundFormRequest(r, s.options.PublicOrigin, csrfOK) { + writeProblem(w, http.StatusForbidden, "valid incident form required") + return nil, auth.Principal{}, false + } + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return nil, auth.Principal{}, false + } + if err != nil { + writeProblem(w, http.StatusBadRequest, "invalid incident management request") + return nil, auth.Principal{}, false + } + scope := access.Scope{OrganizationID: values.Get("organization_id")} + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionIncidentsManage) + if err != nil || !decision.Allowed { + writeProblem(w, http.StatusForbidden, "incident management denied") + return nil, auth.Principal{}, false + } + if err = s.identity.ValidateResourceScope(r.Context(), scope); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid organization") + return nil, auth.Principal{}, false + } + if !csrfOK { + writeProblem(w, http.StatusForbidden, "valid incident CSRF token required") + return nil, auth.Principal{}, false + } + return values, principal, true +} diff --git a/internal/httpserver/manage.go b/internal/httpserver/manage.go new file mode 100644 index 0000000..10c6f2f --- /dev/null +++ b/internal/httpserver/manage.go @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "math" + "net/http" + "net/url" + "strconv" + "strings" + "unicode/utf8" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/site" + "gamertan.com/observatory/internal/storage" + "gamertan.com/web/access" + "gamertan.com/web/auth" + "gamertan.com/web/authhttp" +) + +func (s *Server) createSavedQuery(w http.ResponseWriter, r *http.Request) { + values, principal, ok := s.authorizeManagementForm(w, r, "organization_id", "csrf_token", "name", "description", "query") + if !ok { + return + } + s.saveQuery(w, r, values, principal, values.Get("query")) +} + +func (s *Server) createBuiltQuery(w http.ResponseWriter, r *http.Request) { + required := []string{"organization_id", "csrf_token", "name", "description", "signal", "filter_operator", "window", "aggregate", "limit"} + optional := []string{"filter_field", "filter_value", "aggregate_field", "group_by", "bucket"} + values, principal, ok := s.authorizeManagementFormFields(w, r, required, optional) + if !ok { + return + } + text, err := buildAssistedQuery(values, s.options.MaxQueryRows) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "assisted query rejected") + return + } + s.saveQuery(w, r, values, principal, text) +} + +func (s *Server) saveQuery(w http.ResponseWriter, r *http.Request, values url.Values, principal auth.Principal, text string) { + _, err := s.store.SaveQuery(r.Context(), storage.SavedQueryInput{ + OrganizationID: values.Get("organization_id"), Name: values.Get("name"), + Description: values.Get("description"), Query: text, + ActorUserID: principal.User.ID, MaxRows: s.options.MaxQueryRows, + }, s.now()) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "saved query rejected") + return + } + http.Redirect(w, r, "/app/?organization="+url.QueryEscape(values.Get("organization_id"))+"#saved-work", http.StatusSeeOther) +} + +func buildAssistedQuery(values url.Values, maxRows int) (string, error) { + allowed := func(value string, candidates ...string) bool { + for _, candidate := range candidates { + if value == candidate { + return true + } + } + return false + } + signal := values.Get("signal") + if !allowed(signal, "logs", "metrics", "traces", "deployments") { + return "", fmt.Errorf("unsupported signal") + } + window := values.Get("window") + if !allowed(window, "15m", "1h", "6h", "24h", "168h") { + return "", fmt.Errorf("unsupported window") + } + limit, err := strconv.Atoi(values.Get("limit")) + if err != nil || limit < 1 || limit > maxRows || !allowed(values.Get("limit"), "10", "20", "50", "100", "250") { + return "", fmt.Errorf("unsupported limit") + } + stages := []string{signal} + filterField := values.Get("filter_field") + filterValue := values.Get("filter_value") + filterOperator := values.Get("filter_operator") + if !allowed(filterOperator, "==", "!=", ">=", "<=", ">", "<") { + return "", fmt.Errorf("invalid filter comparison") + } + if filterField == "" { + if filterValue != "" { + return "", fmt.Errorf("filter value requires a field") + } + } else { + if !allowed(filterField, "service", "project", "environment", "route", "status", "duration", "name", "severity", "value", "trace_id", "correlation_id") || + filterValue == "" || len(filterValue) > 256 || !utf8.ValidString(filterValue) || strings.IndexByte(filterValue, 0) >= 0 { + return "", fmt.Errorf("invalid filter") + } + quoted := strconv.Quote(filterValue) + quoted = strings.ReplaceAll(quoted, "|", `\u007c`) + stages = append(stages, "where "+filterField+" "+filterOperator+" "+quoted) + } + stages = append(stages, "window "+window) + + aggregate := values.Get("aggregate") + aggregateField := values.Get("aggregate_field") + groupBy := values.Get("group_by") + bucket := values.Get("bucket") + if aggregate == "none" { + if aggregateField != "" || groupBy != "" || bucket != "" { + return "", fmt.Errorf("summary options require an aggregate") + } + } else { + if !allowed(aggregate, "count", "min", "max", "sum", "avg", "p50", "p95", "p99") || + !allowed(groupBy, "", "service", "project", "environment", "route", "status", "name", "severity") || + !allowed(bucket, "", "1m", "5m", "15m", "1h") { + return "", fmt.Errorf("invalid summary") + } + expression := "count()" + if aggregate == "count" { + if aggregateField != "" { + return "", fmt.Errorf("count accepts no field") + } + } else { + if !allowed(aggregateField, "value", "duration", "status") { + return "", fmt.Errorf("numeric aggregate field required") + } + expression = aggregate + "(" + aggregateField + ")" + } + groups := make([]string, 0, 2) + if groupBy != "" { + groups = append(groups, groupBy) + } + if bucket != "" { + groups = append(groups, "window("+bucket+")") + } + stage := "summarize " + expression + if len(groups) > 0 { + stage += " by " + strings.Join(groups, ", ") + } + stages = append(stages, stage) + } + stages = append(stages, "limit "+strconv.Itoa(limit)) + text := strings.Join(stages, " | ") + if _, err = query.Parse(text, maxRows); err != nil { + return "", err + } + return text, nil +} + +func (s *Server) createDashboard(w http.ResponseWriter, r *http.Request) { + values, principal, ok := s.authorizeManagementForm(w, r, "organization_id", "csrf_token", "slug", "name", "description", "panel_title", "saved_query_id", "visualization") + if !ok { + return + } + organizationID := values.Get("organization_id") + queryValue, err := s.store.SavedQuery(r.Context(), organizationID, values.Get("saved_query_id")) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "dashboard query rejected") + return + } + visualization := values.Get("visualization") + if !validDashboardPresentation(queryValue, visualization) { + writeProblem(w, http.StatusUnprocessableEntity, "dashboard presentation does not match query") + return + } + _, err = s.store.SaveDashboard(r.Context(), storage.DashboardInput{ + OrganizationID: organizationID, Slug: values.Get("slug"), Name: values.Get("name"), + Description: values.Get("description"), ActorUserID: principal.User.ID, + Panels: []storage.DashboardPanel{{Position: 0, Title: values.Get("panel_title"), Visualization: visualization, SavedQueryID: queryValue.ID}}, + }, s.now()) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "dashboard rejected") + return + } + http.Redirect(w, r, "/app/dashboards/"+url.PathEscape(values.Get("slug"))+"/?organization="+url.QueryEscape(organizationID), http.StatusSeeOther) +} + +func (s *Server) updateDashboard(w http.ResponseWriter, r *http.Request) { + values, principal, ok := s.authorizeManagementForm(w, r, "organization_id", "csrf_token", "dashboard_id", "expected_revision", "slug", "name", "description") + if !ok { + return + } + current, ok := s.dashboardForRevision(w, r, values) + if !ok { + return + } + updated, err := s.store.SaveDashboard(r.Context(), storage.DashboardInput{ + ID: current.ID, ExpectedRevision: current.Revision, OrganizationID: current.OrganizationID, + Slug: values.Get("slug"), Name: values.Get("name"), Description: values.Get("description"), + Panels: current.Panels, ActorUserID: principal.User.ID, + }, s.now()) + if !writeDashboardRevisionResult(w, err) { + return + } + redirectDashboard(w, r, updated.Slug, current.OrganizationID) +} + +func (s *Server) addDashboardPanel(w http.ResponseWriter, r *http.Request) { + values, principal, ok := s.authorizeManagementForm(w, r, "organization_id", "csrf_token", "dashboard_id", "expected_revision", "panel_title", "saved_query_id", "visualization") + if !ok { + return + } + current, ok := s.dashboardForRevision(w, r, values) + if !ok { + return + } + queryValue, err := s.store.SavedQuery(r.Context(), current.OrganizationID, values.Get("saved_query_id")) + if err != nil || !validDashboardPresentation(queryValue, values.Get("visualization")) { + writeProblem(w, http.StatusUnprocessableEntity, "dashboard panel rejected") + return + } + panels := append([]storage.DashboardPanel(nil), current.Panels...) + panels = append(panels, storage.DashboardPanel{Position: len(panels), Title: values.Get("panel_title"), Visualization: values.Get("visualization"), SavedQueryID: queryValue.ID}) + updated, err := s.store.SaveDashboard(r.Context(), storage.DashboardInput{ + ID: current.ID, ExpectedRevision: current.Revision, OrganizationID: current.OrganizationID, + Slug: current.Slug, Name: current.Name, Description: current.Description, + Panels: panels, ActorUserID: principal.User.ID, + }, s.now()) + if !writeDashboardRevisionResult(w, err) { + return + } + redirectDashboard(w, r, updated.Slug, current.OrganizationID) +} + +func (s *Server) updateDashboardPanel(w http.ResponseWriter, r *http.Request) { + values, principal, ok := s.authorizeManagementForm(w, r, "organization_id", "csrf_token", "dashboard_id", "expected_revision", "panel_title", "saved_query_id", "visualization") + if !ok { + return + } + current, ok := s.dashboardForRevision(w, r, values) + if !ok { + return + } + queryValue, err := s.store.SavedQuery(r.Context(), current.OrganizationID, values.Get("saved_query_id")) + if err != nil || !validDashboardPresentation(queryValue, values.Get("visualization")) { + writeProblem(w, http.StatusUnprocessableEntity, "dashboard panel rejected") + return + } + panelID := r.PathValue("panel") + panels := append([]storage.DashboardPanel(nil), current.Panels...) + found := false + for index := range panels { + if panels[index].ID != panelID { + continue + } + panels[index].Title = values.Get("panel_title") + panels[index].Visualization = values.Get("visualization") + panels[index].SavedQueryID = queryValue.ID + found = true + break + } + if !found { + writeProblem(w, http.StatusNotFound, "dashboard panel not found") + return + } + updated, err := s.store.SaveDashboard(r.Context(), storage.DashboardInput{ + ID: current.ID, ExpectedRevision: current.Revision, OrganizationID: current.OrganizationID, + Slug: current.Slug, Name: current.Name, Description: current.Description, + Panels: panels, ActorUserID: principal.User.ID, + }, s.now()) + if !writeDashboardRevisionResult(w, err) { + return + } + redirectDashboard(w, r, updated.Slug, current.OrganizationID) +} + +func (s *Server) removeDashboardPanel(w http.ResponseWriter, r *http.Request) { + values, principal, ok := s.authorizeManagementForm(w, r, "organization_id", "csrf_token", "dashboard_id", "expected_revision") + if !ok { + return + } + current, ok := s.dashboardForRevision(w, r, values) + if !ok { + return + } + panelID := r.PathValue("panel") + panels := make([]storage.DashboardPanel, 0, len(current.Panels)) + for _, panel := range current.Panels { + if panel.ID != panelID { + panel.Position = len(panels) + panels = append(panels, panel) + } + } + if len(panels) == len(current.Panels) { + writeProblem(w, http.StatusNotFound, "dashboard panel not found") + return + } + updated, err := s.store.SaveDashboard(r.Context(), storage.DashboardInput{ + ID: current.ID, ExpectedRevision: current.Revision, OrganizationID: current.OrganizationID, + Slug: current.Slug, Name: current.Name, Description: current.Description, + Panels: panels, ActorUserID: principal.User.ID, + }, s.now()) + if !writeDashboardRevisionResult(w, err) { + return + } + redirectDashboard(w, r, updated.Slug, current.OrganizationID) +} + +func (s *Server) dashboardForRevision(w http.ResponseWriter, r *http.Request, values url.Values) (storage.Dashboard, bool) { + current, err := s.store.Dashboard(r.Context(), values.Get("organization_id"), r.PathValue("slug")) + if err != nil { + writeProblem(w, http.StatusNotFound, "dashboard not found") + return storage.Dashboard{}, false + } + revision, err := strconv.Atoi(values.Get("expected_revision")) + if err != nil || revision < 1 || values.Get("dashboard_id") != current.ID { + writeProblem(w, http.StatusBadRequest, "dashboard revision is invalid") + return storage.Dashboard{}, false + } + if revision != current.Revision { + writeProblem(w, http.StatusConflict, "dashboard changed; reload before editing") + return storage.Dashboard{}, false + } + return current, true +} + +func validDashboardPresentation(saved storage.SavedQuery, visualization string) bool { + switch visualization { + case "table": + return true + case "stat": + return saved.AST.Summary != nil + case "timeseries": + return saved.AST.Summary != nil && saved.AST.Bucket > 0 + default: + return false + } +} + +func writeDashboardRevisionResult(w http.ResponseWriter, err error) bool { + if err == nil { + return true + } + if errors.Is(err, storage.ErrDashboardRevisionConflict) { + writeProblem(w, http.StatusConflict, "dashboard changed; reload before editing") + } else { + writeProblem(w, http.StatusUnprocessableEntity, "dashboard revision rejected") + } + return false +} + +func redirectDashboard(w http.ResponseWriter, r *http.Request, slug, organizationID string) { + http.Redirect(w, r, "/app/dashboards/"+url.PathEscape(slug)+"/?organization="+url.QueryEscape(organizationID), http.StatusSeeOther) +} + +func (s *Server) authorizeManagementForm(w http.ResponseWriter, r *http.Request, fields ...string) (url.Values, auth.Principal, bool) { + return s.authorizeManagementFormFields(w, r, fields, nil) +} + +func (s *Server) authorizeManagementFormFields(w http.ResponseWriter, r *http.Request, required, optional []string) (url.Values, auth.Principal, bool) { + principal, ok := auth.PrincipalFromContext(r.Context()) + values, err := readFormFields(w, r, 24<<10, required, optional) + token, sessionOK := authhttp.SessionToken(r, s.cookie) + csrfOK := err == nil && sessionOK && authhttp.VerifyCSRF(token, "dashboards:manage", values.Get("csrf_token")) + if !tokenBoundFormRequest(r, s.options.PublicOrigin, csrfOK) { + writeProblem(w, http.StatusForbidden, "valid dashboard form required") + return nil, auth.Principal{}, false + } + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return nil, auth.Principal{}, false + } + if err != nil { + writeProblem(w, http.StatusBadRequest, "invalid dashboard management request") + return nil, auth.Principal{}, false + } + organizationID := values.Get("organization_id") + scope := access.Scope{OrganizationID: organizationID} + if err = s.identity.ValidateResourceScope(r.Context(), scope); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid organization") + return nil, auth.Principal{}, false + } + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionDashboardsManage) + if err != nil || !decision.Allowed { + writeProblem(w, http.StatusForbidden, "dashboard management denied") + return nil, auth.Principal{}, false + } + if !csrfOK { + writeProblem(w, http.StatusForbidden, "valid dashboard CSRF token required") + return nil, auth.Principal{}, false + } + return values, principal, true +} + +func (s *Server) dashboard(w http.ResponseWriter, r *http.Request) { + organizationID, principal, ok := s.authorizeDashboardRead(w, r) + if !ok { + return + } + dashboard, err := s.store.Dashboard(r.Context(), organizationID, r.PathValue("slug")) + if err != nil { + writeProblem(w, http.StatusNotFound, "dashboard not found") + return + } + organizations, err := s.identity.OrganizationsForUser(r.Context(), principal.User.ID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "organization unavailable") + return + } + organizationName := "Organization" + for _, organization := range organizations { + if organization.ID == organizationID { + organizationName = organization.Name + break + } + } + view := site.DashboardView{ + Head: s.head(dashboard.Name+" — Gamertan Observatory", dashboard.Description, "/app/dashboards/"+url.PathEscape(dashboard.Slug)+"/"), + DisplayName: principal.User.DisplayName, Organization: site.OrganizationOption{ID: organizationID, Name: organizationName, Selected: true}, + ID: dashboard.ID, Slug: dashboard.Slug, Revision: dashboard.Revision, + Name: dashboard.Name, Description: dashboard.Description, + ExportURL: "/app/dashboards/" + url.PathEscape(dashboard.Slug) + "/export.json?organization=" + url.QueryEscape(organizationID), + } + manage, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, access.Scope{OrganizationID: organizationID}, identity.PermissionDashboardsManage) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "authorization unavailable") + return + } + view.CanManage = manage.Allowed + if view.CanManage { + token, sessionOK := authhttp.SessionToken(r, s.cookie) + if !sessionOK { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return + } + view.ManageCSRF, err = authhttp.CSRFToken(token, "dashboards:manage") + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + savedQueries, loadErr := s.store.SavedQueries(r.Context(), organizationID) + if loadErr != nil { + writeProblem(w, http.StatusServiceUnavailable, "saved queries unavailable") + return + } + for _, saved := range savedQueries { + view.SavedQueries = append(view.SavedQueries, site.SavedQuerySummary{ID: saved.ID, Name: saved.Name, Description: saved.Description, Query: saved.Query}) + } + } + for _, panel := range dashboard.Panels { + view.Panels = append(view.Panels, s.dashboardPanel(r, principal.User.ID, organizationID, panel)) + } + s.renderHTML(w, r, http.StatusOK, site.Dashboard(view)) +} + +func (s *Server) dashboardPanel(r *http.Request, userID, organizationID string, panel storage.DashboardPanel) site.PanelView { + view := site.PanelView{ID: panel.ID, SavedQueryID: panel.SavedQueryID, Title: panel.Title, Visualization: panel.Visualization, Table: site.TableView{Caption: panel.Title, Columns: []site.TableColumn{{Label: "Status"}}, Empty: "Panel data is unavailable."}} + saved, err := s.store.SavedQuery(r.Context(), organizationID, panel.SavedQueryID) + if err != nil { + return view + } + view.Query = saved.Query + scope := access.Scope{OrganizationID: organizationID, ProjectID: saved.Scope.ProjectID, EnvironmentID: saved.Scope.EnvironmentID, ServiceID: saved.Scope.ServiceID} + decision, err := s.identity.Access.Authorize(r.Context(), userID, scope, identity.PermissionTelemetryQuery) + if err != nil || !decision.Allowed || s.identity.ValidateResourceScope(r.Context(), scope) != nil { + return view + } + sensitive, err := s.identity.Access.Authorize(r.Context(), userID, scope, identity.PermissionTelemetryReadSensitive) + if err != nil { + return view + } + result, err := s.store.Query(r.Context(), saved.AST, query.Scope{OrganizationID: organizationID, ProjectID: saved.Scope.ProjectID, EnvironmentID: saved.Scope.EnvironmentID, ServiceID: saved.Scope.ServiceID, Sensitive: sensitive.Allowed}, s.options.QueryBudget, s.now()) + if err != nil { + return view + } + view.Table = resultTable(panel.Title, result) + if panel.Visualization == "timeseries" { + view.Chart = resultChart(panel.Title, result) + } + if panel.Visualization == "stat" { + for _, row := range result.Rows { + for _, value := range row.Values { + if value != nil { + view.Stat = boundedCell(*value) + return view + } + } + } + } + return view +} + +func resultChart(title string, result query.Result) site.ChartView { + if len(result.Columns) < 2 || len(result.Rows) == 0 { + return site.ChartView{} + } + valueIndex := len(result.Columns) - 1 + valueType := result.Columns[valueIndex].Type + if valueType != "integer" && valueType != "float" && valueType != "duration" { + return site.ChartView{} + } + const maxPoints = 48 + points := make([]struct { + label, display string + value float64 + }, 0, min(len(result.Rows), maxPoints)) + maximum := float64(0) + for index, row := range result.Rows { + if index >= maxPoints || valueIndex >= len(row.Values) || row.Values[valueIndex] == nil { + continue + } + value, err := strconv.ParseFloat(*row.Values[valueIndex], 64) + if err != nil || math.IsNaN(value) || math.IsInf(value, 0) || value < 0 { + return site.ChartView{} + } + labels := make([]string, 0, valueIndex) + for column := 0; column < valueIndex && column < len(row.Values); column++ { + if row.Values[column] != nil { + labels = append(labels, boundedCell(*row.Values[column])) + } + } + label := boundedCell(strings.Join(labels, " · ")) + if label == "" { + label = fmt.Sprintf("Point %d", index+1) + } + display := boundedCell(*row.Values[valueIndex]) + if unit := result.Columns[valueIndex].Unit; unit != "" { + display += " " + boundedCell(unit) + } + points = append(points, struct { + label, display string + value float64 + }{label: label, display: display, value: value}) + maximum = math.Max(maximum, value) + } + if len(points) == 0 { + return site.ChartView{} + } + if maximum == 0 { + maximum = 1 + } + view := site.ChartView{Label: title + " visual summary"} + for _, point := range points { + view.Points = append(view.Points, site.ChartPoint{ + Label: point.label, Value: strconv.FormatFloat(point.value, 'g', -1, 64), + Maximum: strconv.FormatFloat(maximum, 'g', -1, 64), Display: point.display, + }) + } + return view +} + +func (s *Server) exportDashboard(w http.ResponseWriter, r *http.Request) { + organizationID, _, ok := s.authorizeDashboardRead(w, r) + if !ok { + return + } + exported, err := s.store.ExportDashboard(r.Context(), organizationID, r.PathValue("slug")) + if err != nil { + writeProblem(w, http.StatusNotFound, "dashboard not found") + return + } + var body bytes.Buffer + encoder := json.NewEncoder(&body) + encoder.SetIndent("", " ") + if err = encoder.Encode(exported); err != nil { + writeProblem(w, http.StatusInternalServerError, "dashboard export unavailable") + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", "observatory-dashboard-"+r.PathValue("slug")+".json")) + w.Header().Set("Content-Length", fmt.Sprintf("%d", body.Len())) + if r.Method != http.MethodHead { + _, _ = w.Write(body.Bytes()) + } +} + +func (s *Server) authorizeDashboardRead(w http.ResponseWriter, r *http.Request) (string, auth.Principal, bool) { + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return "", auth.Principal{}, false + } + values := r.URL.Query() + organizationID := values.Get("organization") + if len(values) != 1 || len(values["organization"]) != 1 || organizationID == "" { + writeProblem(w, http.StatusBadRequest, "organization is required") + return "", auth.Principal{}, false + } + scope := access.Scope{OrganizationID: organizationID} + if err := s.identity.ValidateResourceScope(r.Context(), scope); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid organization") + return "", auth.Principal{}, false + } + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionDashboardsRead) + if err != nil || !decision.Allowed { + writeProblem(w, http.StatusForbidden, "dashboard access denied") + return "", auth.Principal{}, false + } + return organizationID, principal, true +} diff --git a/internal/httpserver/native_replay_benchmark_test.go b/internal/httpserver/native_replay_benchmark_test.go new file mode 100644 index 0000000..d438851 --- /dev/null +++ b/internal/httpserver/native_replay_benchmark_test.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/nativeprotocol" + "gamertan.com/observatory/internal/storage" +) + +func BenchmarkNativeExactReplay(b *testing.B) { + root := filepath.Join(b.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + b.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + b.Fatal(err) + } + defer store.Close() + identities, err := identity.Open(root) + if err != nil { + b.Fatal(err) + } + defer identities.Close() + token, err := store.CreateSource(context.Background(), "source", model.Scope{OrganizationID: "org", ProjectID: "project", EnvironmentID: "production", ServiceID: "service"}) + if err != nil { + b.Fatal(err) + } + now := time.Date(2026, 8, 18, 20, 0, 0, 0, time.UTC) + server, err := New(store, identities, testOptions()) + if err != nil { + b.Fatal(err) + } + server.now = func() time.Time { return now } + handler := server.Handler() + + legacy := model.Batch{Version: model.BatchVersion, SourceID: "source", StreamID: "legacy", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: benchmarkRecords(now)} + legacyBody, _ := json.Marshal(legacy) + framed := legacy + framed.StreamID = "framed" + framedBody, _ := json.Marshal(framed) + framedEnvelope, _ := framed.Envelope(framedBody) + seed := func(path string, body []byte, envelope *model.BatchEnvelope) { + request := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(body)) + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Content-Type", "application/json") + if envelope != nil { + nativeprotocol.SetHeaders(request.Header, *envelope) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted { + b.Fatalf("seed %s: %d %s", path, response.Code, response.Body.String()) + } + } + seed("/api/v1/ingest/native", legacyBody, nil) + seed("/api/v2/ingest/native", framedBody, &framedEnvelope) + + for _, benchmark := range []struct { + name string + path string + body []byte + envelope *model.BatchEnvelope + }{{"legacy-v1", "/api/v1/ingest/native", legacyBody, nil}, {"framed-v2", "/api/v2/ingest/native", framedBody, &framedEnvelope}} { + b.Run(benchmark.name, func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(benchmark.body))) + for range b.N { + request := httptest.NewRequest(http.MethodPost, benchmark.path, bytes.NewReader(benchmark.body)) + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Content-Type", "application/json") + if benchmark.envelope != nil { + nativeprotocol.SetHeaders(request.Header, *benchmark.envelope) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted { + b.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + } + }) + } +} + +func benchmarkRecords(now time.Time) []model.Observation { + records := make([]model.Observation, 500) + for index := range records { + records[index] = model.Observation{Timestamp: now, Name: "http.request", Attributes: map[string]string{"route": "/items", "status": "200", "method": "GET"}} + } + return records +} diff --git a/internal/httpserver/push.go b/internal/httpserver/push.go new file mode 100644 index 0000000..9308c42 --- /dev/null +++ b/internal/httpserver/push.go @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "crypto/ecdh" + "encoding/base64" + "net/http" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/storage" + "gamertan.com/observatory/internal/webpush" + "gamertan.com/web/access" + "gamertan.com/web/auth" + "gamertan.com/web/authhttp" + "gamertan.com/web/websec" +) + +type pushSubscriptionRequest struct { + OrganizationID string `json:"organization_id"` + Endpoint string `json:"endpoint"` + Keys struct { + P256DH string `json:"p256dh"` + Auth string `json:"auth"` + } `json:"keys"` +} + +func (s *Server) savePushSubscription(w http.ResponseWriter, r *http.Request) { + request, principal, ok := s.authorizePushRequest(w, r) + if !ok { + return + } + p256dh, p256Err := base64.RawURLEncoding.DecodeString(request.Keys.P256DH) + authSecret, authErr := base64.RawURLEncoding.DecodeString(request.Keys.Auth) + if p256Err != nil || authErr != nil || len(p256dh) != 65 || len(authSecret) != 16 { + writeProblem(w, http.StatusUnprocessableEntity, "push subscription rejected") + return + } + if _, err := ecdh.P256().NewPublicKey(p256dh); err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "push subscription rejected") + return + } + if _, err := webpush.ValidateEndpoint(request.Endpoint); err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "push subscription rejected") + return + } + subscription, err := s.store.SavePushSubscription(r.Context(), storage.PushSubscriptionInput{OrganizationID: request.OrganizationID, UserID: principal.User.ID, Endpoint: request.Endpoint, P256DH: p256dh, Auth: authSecret}, s.now()) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "push subscription rejected") + return + } + writeJSON(w, http.StatusCreated, struct { + ID string `json:"id"` + }{subscription.ID}) +} + +func (s *Server) deletePushSubscription(w http.ResponseWriter, r *http.Request) { + request, principal, ok := s.authorizePushRequest(w, r) + if !ok { + return + } + if _, err := webpush.ValidateEndpoint(request.Endpoint); err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "push subscription rejected") + return + } + remaining, err := s.store.DeletePushSubscription(r.Context(), request.OrganizationID, principal.User.ID, request.Endpoint) + if err != nil { + writeProblem(w, http.StatusNotFound, "push subscription not found") + return + } + writeJSON(w, http.StatusOK, struct { + Remaining bool `json:"remaining"` + }{remaining}) +} + +func (s *Server) pushSubscriptionStatus(w http.ResponseWriter, r *http.Request) { + request, principal, ok := s.authorizePushRequest(w, r) + if !ok { + return + } + if _, err := webpush.ValidateEndpoint(request.Endpoint); err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "push subscription rejected") + return + } + subscribed, err := s.store.HasPushSubscription(r.Context(), request.OrganizationID, principal.User.ID, request.Endpoint) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "push subscription status unavailable") + return + } + writeJSON(w, http.StatusOK, struct { + Subscribed bool `json:"subscribed"` + }{subscribed}) +} + +func (s *Server) authorizePushRequest(w http.ResponseWriter, r *http.Request) (pushSubscriptionRequest, auth.Principal, bool) { + if s.options.PushDispatcher == nil { + writeProblem(w, http.StatusNotFound, "Web Push is not configured") + return pushSubscriptionRequest{}, auth.Principal{}, false + } + if !websec.SameOrigin(r, s.options.PublicOrigin) { + writeProblem(w, http.StatusForbidden, "same-origin request required") + return pushSubscriptionRequest{}, auth.Principal{}, false + } + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return pushSubscriptionRequest{}, auth.Principal{}, false + } + body := http.MaxBytesReader(w, r.Body, 8<<10) + defer body.Close() + var request pushSubscriptionRequest + if err := decodeOne(body, &request); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid push subscription request") + return pushSubscriptionRequest{}, auth.Principal{}, false + } + scope := access.Scope{OrganizationID: request.OrganizationID} + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionIncidentsRead) + if err != nil || !decision.Allowed { + writeProblem(w, http.StatusForbidden, "incident access denied") + return pushSubscriptionRequest{}, auth.Principal{}, false + } + if err = s.identity.ValidateResourceScope(r.Context(), scope); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid organization") + return pushSubscriptionRequest{}, auth.Principal{}, false + } + token, sessionOK := authhttp.SessionToken(r, s.cookie) + if !sessionOK || !authhttp.VerifyCSRF(token, "push:manage", r.Header.Get("X-CSRF-Token")) { + writeProblem(w, http.StatusForbidden, "valid push CSRF token required") + return pushSubscriptionRequest{}, auth.Principal{}, false + } + return request, principal, true +} diff --git a/internal/httpserver/pwa.go b/internal/httpserver/pwa.go new file mode 100644 index 0000000..867d72f --- /dev/null +++ b/internal/httpserver/pwa.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "fmt" + "net/http" + + "gamertan.com/observatory/internal/site" +) + +func (s *Server) webManifest(w http.ResponseWriter, r *http.Request) { + serveFixedBody(w, r, site.WebManifest(), "application/manifest+json") +} + +func (s *Server) serviceWorker(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Service-Worker-Allowed", "/") + serveFixedBody(w, r, site.ServiceWorker(), "text/javascript; charset=utf-8") +} + +func (s *Server) offlineShell(w http.ResponseWriter, r *http.Request) { + s.renderHTML(w, r, http.StatusOK, site.Offline(site.OfflineView{Head: s.head("Offline — Gamertan Observatory", "Observatory is temporarily unreachable.", "/offline/")})) +} + +func serveFixedBody(w http.ResponseWriter, r *http.Request, body []byte, contentType string) { + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body))) + if r.Method != http.MethodHead { + _, _ = w.Write(body) + } +} diff --git a/internal/httpserver/refresh.go b/internal/httpserver/refresh.go new file mode 100644 index 0000000..ee8ad9d --- /dev/null +++ b/internal/httpserver/refresh.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "errors" + "sync" +) + +var errRefreshCapacity = errors.New("live refresh capacity reached") + +// refreshHub carries only a coalescible invalidation signal. It never carries +// telemetry, resource names, incident details, or credentials. +type refreshHub struct { + mu sync.Mutex + next uint64 + total int + maxTotal int + maxPerOrganization int + subscribers map[string]map[uint64]chan struct{} +} + +func newRefreshHub(maxTotal, maxPerOrganization int) *refreshHub { + return &refreshHub{maxTotal: maxTotal, maxPerOrganization: maxPerOrganization, subscribers: make(map[string]map[uint64]chan struct{})} +} + +func (hub *refreshHub) subscribe(organizationID string) (<-chan struct{}, func(), error) { + hub.mu.Lock() + defer hub.mu.Unlock() + group := hub.subscribers[organizationID] + if hub.total >= hub.maxTotal || len(group) >= hub.maxPerOrganization { + return nil, nil, errRefreshCapacity + } + if group == nil { + group = make(map[uint64]chan struct{}) + hub.subscribers[organizationID] = group + } + hub.next++ + id := hub.next + updates := make(chan struct{}, 1) + group[id] = updates + hub.total++ + var once sync.Once + remove := func() { + once.Do(func() { + hub.mu.Lock() + defer hub.mu.Unlock() + if current := hub.subscribers[organizationID]; current != nil { + if _, exists := current[id]; exists { + delete(current, id) + hub.total-- + } + if len(current) == 0 { + delete(hub.subscribers, organizationID) + } + } + }) + } + return updates, remove, nil +} + +func (hub *refreshHub) publish(organizationID string) { + hub.mu.Lock() + defer hub.mu.Unlock() + for _, updates := range hub.subscribers[organizationID] { + select { + case updates <- struct{}{}: + default: + } + } +} diff --git a/internal/httpserver/refresh_test.go b/internal/httpserver/refresh_test.go new file mode 100644 index 0000000..8d71717 --- /dev/null +++ b/internal/httpserver/refresh_test.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "errors" + "testing" +) + +func TestRefreshHubIsBoundedCoalescedAndOrganizationScoped(t *testing.T) { + hub := newRefreshHub(2, 1) + first, removeFirst, err := hub.subscribe("organization-a") + if err != nil { + t.Fatal(err) + } + defer removeFirst() + if _, _, err = hub.subscribe("organization-a"); !errors.Is(err, errRefreshCapacity) { + t.Fatalf("same-organization capacity err=%v", err) + } + second, removeSecond, err := hub.subscribe("organization-b") + if err != nil { + t.Fatal(err) + } + defer removeSecond() + if _, _, err = hub.subscribe("organization-c"); !errors.Is(err, errRefreshCapacity) { + t.Fatalf("total capacity err=%v", err) + } + + hub.publish("organization-a") + hub.publish("organization-a") + select { + case <-first: + default: + t.Fatal("organization A did not receive refresh") + } + select { + case <-first: + t.Fatal("duplicate refresh was not coalesced") + default: + } + select { + case <-second: + t.Fatal("organization B received organization A refresh") + default: + } + + removeFirst() + if _, removeReplacement, err := hub.subscribe("organization-a"); err != nil { + t.Fatalf("released capacity was not reusable: %v", err) + } else { + removeReplacement() + } +} diff --git a/internal/httpserver/server.go b/internal/httpserver/server.go new file mode 100644 index 0000000..e285c9c --- /dev/null +++ b/internal/httpserver/server.go @@ -0,0 +1,810 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "bytes" + "compress/gzip" + "crypto/ecdh" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "strings" + "time" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/nativeprotocol" + "gamertan.com/observatory/internal/otlp" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/storage" + "gamertan.com/web/access" + "gamertan.com/web/auth" + "gamertan.com/web/authhttp" + "gamertan.com/web/requestmeta" + "gamertan.com/web/websec" +) + +type Options struct { + PublicOrigin string + MaxBodyBytes int64 + MaxConcurrentIngest int + MaxQueryRows int + QueryBudget query.Budget + SessionLifetime time.Duration + PushPublicKey string + PushDispatcher PushDispatcher +} + +type PushDispatcher interface { + Enqueue(organizationID string) bool +} + +type Server struct { + store *storage.Store + identity *identity.Services + options Options + cookie authhttp.CookieConfig + now func() time.Time + refresh *refreshHub + requests *requestmeta.Resolver + ingestSlots chan struct{} +} + +func New(store *storage.Store, identities *identity.Services, options Options) (*Server, error) { + if store == nil || identities == nil || identities.Auth == nil || identities.Access == nil { + return nil, errors.New("server storage and identity services are required") + } + origin, err := url.Parse(options.PublicOrigin) + if err != nil || origin.Scheme != "https" || origin.Host == "" || origin.User != nil || origin.Path != "" && origin.Path != "/" || origin.RawQuery != "" || origin.Fragment != "" { + return nil, errors.New("server public origin must be an absolute HTTPS origin") + } + options.PublicOrigin = strings.TrimSuffix(options.PublicOrigin, "/") + if options.MaxConcurrentIngest == 0 { + options.MaxConcurrentIngest = 8 + } + if options.MaxBodyBytes < 1024 || options.MaxConcurrentIngest < 1 || options.MaxConcurrentIngest > 64 || options.MaxQueryRows < 1 || options.SessionLifetime < 5*time.Minute || options.SessionLifetime > 30*24*time.Hour { + return nil, errors.New("server limits are invalid") + } + if options.QueryBudget.MaxRows != options.MaxQueryRows || options.QueryBudget.MaxDuration < time.Millisecond || options.QueryBudget.MaxScannedBytes < 1 || options.QueryBudget.MaxMemoryBytes < 1 { + return nil, errors.New("server query budget is invalid") + } + if (options.PushPublicKey == "") != (options.PushDispatcher == nil) { + return nil, errors.New("server Web Push key and dispatcher must be configured together") + } + if options.PushPublicKey != "" { + publicKey, decodeErr := base64.RawURLEncoding.DecodeString(options.PushPublicKey) + if decodeErr != nil || len(publicKey) != 65 { + return nil, errors.New("server Web Push public key is invalid") + } + if _, decodeErr = ecdh.P256().NewPublicKey(publicKey); decodeErr != nil { + return nil, errors.New("server Web Push public key is invalid") + } + } + cookie := authhttp.CookieConfig{Name: "__Host-observatory_session", Lifetime: options.SessionLifetime, SameSite: http.SameSiteStrictMode} + if err = cookie.Validate(); err != nil { + return nil, err + } + requests, err := requestmeta.New(requestmeta.Config{}) + if err != nil { + return nil, fmt.Errorf("server request metadata: %w", err) + } + return &Server{store: store, identity: identities, options: options, cookie: cookie, now: func() time.Time { return time.Now().UTC() }, refresh: newRefreshHub(256, 8), requests: requests, ingestSlots: make(chan struct{}, options.MaxConcurrentIngest)}, nil +} + +func (s *Server) enterIngest(w http.ResponseWriter) (func(), bool) { + select { + case s.ingestSlots <- struct{}{}: + return func() { <-s.ingestSlots }, true + default: + w.Header().Set("Retry-After", "1") + writeProblem(w, http.StatusServiceUnavailable, "ingestion capacity temporarily unavailable") + return nil, false + } +} + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", textOK("ok\n")) + mux.HandleFunc("HEAD /healthz", textOK("ok\n")) + mux.HandleFunc("GET /readyz", textOK("ready\n")) + mux.HandleFunc("HEAD /readyz", textOK("ready\n")) + mux.HandleFunc("GET /{$}", s.landing) + mux.HandleFunc("HEAD /{$}", s.landing) + mux.HandleFunc("GET /manifest.webmanifest", s.webManifest) + mux.HandleFunc("HEAD /manifest.webmanifest", s.webManifest) + mux.HandleFunc("GET /service-worker.js", s.serviceWorker) + mux.HandleFunc("HEAD /service-worker.js", s.serviceWorker) + mux.HandleFunc("GET /offline/{$}", s.offlineShell) + mux.HandleFunc("HEAD /offline/{$}", s.offlineShell) + mux.HandleFunc("GET /login/{$}", s.loginPage) + mux.HandleFunc("HEAD /login/{$}", s.loginPage) + mux.HandleFunc("POST /login/{$}", s.loginForm) + mux.HandleFunc("POST /logout/{$}", s.logoutForm) + mux.HandleFunc("GET /account/password/{$}", s.passwordPage) + mux.HandleFunc("HEAD /account/password/{$}", s.passwordPage) + mux.HandleFunc("POST /account/password/{$}", s.passwordForm) + mux.HandleFunc("GET /app/{$}", s.app) + mux.HandleFunc("HEAD /app/{$}", s.app) + mux.HandleFunc("GET /app/explore/{$}", s.explorePage) + mux.HandleFunc("HEAD /app/explore/{$}", s.explorePage) + mux.HandleFunc("POST /app/explore/{$}", s.exploreForm) + mux.HandleFunc("GET /app/events", s.events) + mux.HandleFunc("POST /app/queries/{$}", s.createSavedQuery) + mux.HandleFunc("POST /app/queries/builder/{$}", s.createBuiltQuery) + mux.HandleFunc("POST /app/dashboards/{$}", s.createDashboard) + mux.HandleFunc("GET /app/dashboards/{slug}/{$}", s.dashboard) + mux.HandleFunc("HEAD /app/dashboards/{slug}/{$}", s.dashboard) + mux.HandleFunc("POST /app/dashboards/{slug}/{$}", s.updateDashboard) + mux.HandleFunc("POST /app/dashboards/{slug}/panels/{$}", s.addDashboardPanel) + mux.HandleFunc("POST /app/dashboards/{slug}/panels/{panel}/{$}", s.updateDashboardPanel) + mux.HandleFunc("POST /app/dashboards/{slug}/panels/{panel}/remove/{$}", s.removeDashboardPanel) + mux.HandleFunc("GET /app/dashboards/{slug}/export.json", s.exportDashboard) + mux.HandleFunc("HEAD /app/dashboards/{slug}/export.json", s.exportDashboard) + mux.HandleFunc("GET /app/incidents/{$}", s.incidentInbox) + mux.HandleFunc("HEAD /app/incidents/{$}", s.incidentInbox) + mux.HandleFunc("GET /app/incidents/offline/{$}", s.offlineIncidentInbox) + mux.HandleFunc("HEAD /app/incidents/offline/{$}", s.offlineIncidentInbox) + mux.HandleFunc("POST /app/alert-rules/{$}", s.createAlertRule) + mux.HandleFunc("POST /app/incidents/{id}/{$}", s.transitionIncident) + mux.HandleFunc("GET /assets/", s.serveAsset) + mux.HandleFunc("HEAD /assets/", s.serveAsset) + mux.HandleFunc("POST /api/v1/ingest/native", s.ingest) + mux.HandleFunc("POST /api/v2/ingest/native", s.ingestFramed) + mux.HandleFunc("POST /v1/logs", s.ingestOTLP(otlp.Logs)) + mux.HandleFunc("POST /v1/metrics", s.ingestOTLP(otlp.Metrics)) + mux.HandleFunc("POST /v1/traces", s.ingestOTLP(otlp.Traces)) + mux.HandleFunc("POST /api/v1/agent/enroll", s.enrollAgent) + mux.HandleFunc("POST /api/v1/agent/alert-transition", s.recordAgentAlertTransition) + mux.HandleFunc("DELETE /api/v1/agent/source", s.revokeAgentSource) + mux.HandleFunc("POST /api/v1/session", s.login) + mux.HandleFunc("DELETE /api/v1/session", s.logout) + mux.HandleFunc("POST /api/v1/account/password", s.changePassword) + mux.HandleFunc("POST /api/v1/query/parse", s.parseQuery) + mux.HandleFunc("POST /api/v1/query/explain", s.explainQuery) + mux.HandleFunc("POST /api/v1/query", s.executeQuery) + mux.HandleFunc("POST /api/v1/push/subscription", s.savePushSubscription) + mux.HandleFunc("POST /api/v1/push/subscription/status", s.pushSubscriptionStatus) + mux.HandleFunc("DELETE /api/v1/push/subscription", s.deletePushSubscription) + return securityHeaders(s.requests.Middleware(authhttp.Optional(s.identity.Auth, s.cookie)(s.requirePasswordChange(mux)))) +} + +func (s *Server) recordAgentAlertTransition(w http.ResponseWriter, r *http.Request) { + token, ok := bearer(r.Header.Get("Authorization")) + if !ok || !strings.HasPrefix(token, "obs1.") { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + // Authenticate before decoding. RecordSourceAlertTransition authenticates + // again while binding scope and raw evidence to the credential. + if _, err := s.store.Authenticate(r.Context(), token); err != nil { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + leave, ok := s.enterIngest(w) + if !ok { + return + } + defer leave() + body := http.MaxBytesReader(w, r.Body, 64<<10) + defer body.Close() + var transition model.AlertTransition + if err := decodeOne(body, &transition); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid source alert transition") + return + } + ack, err := s.store.RecordSourceAlertTransition(r.Context(), token, transition, s.now()) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "source alert transition rejected") + return + } + writeJSON(w, http.StatusAccepted, ack) +} + +func (s *Server) requirePasswordChange(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok || !principal.User.PasswordChangeRequired || passwordChangeAllowed(r) { + next.ServeHTTP(w, r) + return + } + if r.Method == http.MethodGet || r.Method == http.MethodHead { + http.Redirect(w, r, "/account/password/", http.StatusSeeOther) + return + } + writeProblem(w, http.StatusForbidden, "password change required") + }) +} + +func passwordChangeAllowed(r *http.Request) bool { + switch r.URL.Path { + case "/healthz", "/readyz", "/manifest.webmanifest", "/service-worker.js", "/offline/", "/account/password/", "/logout/", "/api/v1/account/password", "/api/v1/session": + return true + } + return strings.HasPrefix(r.URL.Path, "/assets/") +} + +func (s *Server) revokeAgentSource(w http.ResponseWriter, r *http.Request) { + token, ok := bearer(r.Header.Get("Authorization")) + if !ok || !strings.HasPrefix(token, "obs1.") { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + source, err := s.store.Authenticate(r.Context(), token) + if err != nil { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + if err = s.store.RevokeSource(r.Context(), source.ID); err != nil { + writeProblem(w, http.StatusServiceUnavailable, "source revocation unavailable") + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) enrollAgent(w http.ResponseWriter, r *http.Request) { + token, ok := bearer(r.Header.Get("Authorization")) + if !ok || !strings.HasPrefix(token, "obse1.") { + writeProblem(w, http.StatusUnauthorized, "valid enrollment required") + return + } + body := http.MaxBytesReader(w, r.Body, 1) + defer body.Close() + payload, bodyErr := io.ReadAll(body) + if bodyErr != nil || len(payload) != 0 { + writeProblem(w, http.StatusBadRequest, "enrollment request body must be empty") + return + } + enrollment, credential, err := s.store.RedeemEnrollment(r.Context(), token, s.now()) + if err != nil { + writeProblem(w, http.StatusUnauthorized, "valid enrollment required") + return + } + writeJSON(w, http.StatusCreated, struct { + SourceID string `json:"source_id"` + Credential string `json:"credential"` + }{enrollment.SourceID, credential}) +} + +func (s *Server) ingest(w http.ResponseWriter, r *http.Request) { + token, ok := bearer(r.Header.Get("Authorization")) + if !ok { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + source, err := s.store.Authenticate(r.Context(), token) + if err != nil { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + leave, ok := s.enterIngest(w) + if !ok { + return + } + defer leave() + body := http.MaxBytesReader(w, r.Body, s.options.MaxBodyBytes) + defer body.Close() + var batch model.Batch + if err := decodeOne(body, &batch); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid native batch") + return + } + ack, err := s.store.Ingest(r.Context(), token, batch, s.now()) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "batch rejected") + return + } + s.refresh.publish(source.Scope.OrganizationID) + writeJSON(w, http.StatusAccepted, ack) +} + +func (s *Server) ingestFramed(w http.ResponseWriter, r *http.Request) { + token, ok := bearer(r.Header.Get("Authorization")) + if !ok { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + source, err := s.store.Authenticate(r.Context(), token) + if err != nil { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + leave, ok := s.enterIngest(w) + if !ok { + return + } + defer leave() + mediaType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/json" || len(parameters) != 0 { + writeProblem(w, http.StatusUnsupportedMediaType, "native JSON content type required") + return + } + envelope, err := nativeprotocol.ParseHeaders(r.Header, s.options.MaxBodyBytes) + if err != nil { + writeProblem(w, http.StatusBadRequest, "invalid native batch envelope") + return + } + body := http.MaxBytesReader(w, r.Body, s.options.MaxBodyBytes) + defer body.Close() + if _, exact, checkErr := s.store.CheckNativeReplay(r.Context(), token, envelope); checkErr != nil { + writeProblem(w, http.StatusUnprocessableEntity, "batch rejected") + return + } else if exact { + if err = verifyNativeReplayBody(body, envelope); err != nil { + writeProblem(w, http.StatusBadRequest, "native batch body does not match envelope") + return + } + ack, confirmErr := s.store.ConfirmNativeReplay(r.Context(), token, envelope) + if confirmErr != nil { + writeProblem(w, http.StatusUnprocessableEntity, "batch rejected") + return + } + writeJSON(w, http.StatusAccepted, ack) + return + } + encoded, err := io.ReadAll(body) + if err != nil { + writeProblem(w, http.StatusBadRequest, "invalid native batch") + return + } + var batch model.Batch + if err = decodeOne(bytes.NewReader(encoded), &batch); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid native batch") + return + } + ack, err := s.store.IngestNative(r.Context(), token, batch, envelope, encoded, s.now()) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "batch rejected") + return + } + if !ack.Duplicate { + s.refresh.publish(source.Scope.OrganizationID) + } + writeJSON(w, http.StatusAccepted, ack) +} + +func verifyNativeReplayBody(body io.Reader, envelope model.BatchEnvelope) error { + digest := sha256.New() + written, err := io.Copy(digest, body) + if err != nil || written != envelope.EncodedBytes || hex.EncodeToString(digest.Sum(nil)) != envelope.WireDigest { + return errors.New("native batch body does not match envelope") + } + return nil +} + +func (s *Server) ingestOTLP(signal otlp.Signal) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + token, ok := bearer(r.Header.Get("Authorization")) + if !ok || !strings.HasPrefix(token, "obs1.") { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + // Authenticate before parsing attacker-controlled protobuf. IngestAuto + // authenticates again while assigning the next sequence under its source + // lock so revocation cannot race into an acknowledged write. + source, err := s.store.Authenticate(r.Context(), token) + if err != nil { + writeProblem(w, http.StatusUnauthorized, "source authentication required") + return + } + leave, ok := s.enterIngest(w) + if !ok { + return + } + defer leave() + mediaType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/x-protobuf" || len(parameters) != 0 { + writeProblem(w, http.StatusUnsupportedMediaType, "OTLP protobuf content type required") + return + } + body, status, err := readOTLPBody(w, r, s.options.MaxBodyBytes) + if err != nil { + title := "invalid OTLP request body" + if status == http.StatusRequestEntityTooLarge { + title = "OTLP request body exceeds limit" + } else if status == http.StatusUnsupportedMediaType { + title = "unsupported OTLP content encoding" + } + writeProblem(w, status, title) + return + } + now := s.now() + records, err := otlp.Decode(signal, body, now) + if err != nil { + writeProblem(w, http.StatusBadRequest, "invalid OTLP protobuf payload") + return + } + if _, err = s.store.IngestAuto(r.Context(), token, signal.StreamID(), signal.ModelSignal(), records, now); err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "OTLP batch rejected") + return + } + s.refresh.publish(source.Scope.OrganizationID) + response, err := otlp.SuccessResponse(signal) + if err != nil { + writeProblem(w, http.StatusInternalServerError, "OTLP response unavailable") + return + } + w.Header().Set("Content-Type", "application/x-protobuf") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(response))) + w.WriteHeader(http.StatusOK) + if len(response) > 0 { + _, _ = w.Write(response) + } + } +} + +func readOTLPBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, int, error) { + compressed := http.MaxBytesReader(w, r.Body, limit) + defer compressed.Close() + encoding := strings.TrimSpace(strings.ToLower(r.Header.Get("Content-Encoding"))) + var reader io.Reader = compressed + var zipped *gzip.Reader + switch encoding { + case "", "identity": + case "gzip": + var err error + zipped, err = gzip.NewReader(compressed) + if err != nil { + return nil, http.StatusBadRequest, err + } + defer zipped.Close() + reader = io.LimitReader(zipped, limit+1) + default: + return nil, http.StatusUnsupportedMediaType, errors.New("unsupported content encoding") + } + body, err := io.ReadAll(reader) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return nil, http.StatusRequestEntityTooLarge, err + } + return nil, http.StatusBadRequest, err + } + if int64(len(body)) > limit { + return nil, http.StatusRequestEntityTooLarge, errors.New("decoded body exceeds limit") + } + return body, http.StatusOK, nil +} + +func (s *Server) login(w http.ResponseWriter, r *http.Request) { + if !websec.SameOrigin(r, s.options.PublicOrigin) { + writeProblem(w, http.StatusForbidden, "same-origin request required") + return + } + body := http.MaxBytesReader(w, r.Body, 16<<10) + defer body.Close() + var input struct { + Identifier string `json:"identifier"` + Password string `json:"password"` + } + if err := decodeOne(body, &input); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid session request") + return + } + token, principal, err := s.identity.Auth.Authenticate(r.Context(), input.Identifier, input.Password, s.options.SessionLifetime) + if err != nil { + writeProblem(w, http.StatusUnauthorized, "invalid credentials") + return + } + if err = authhttp.SetSession(w, s.cookie, token, s.now()); err != nil { + _ = s.identity.Auth.RevokeSession(r.Context(), token) + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + csrf, err := authhttp.CSRFToken(token, "session:delete") + if err != nil { + _ = s.identity.Auth.RevokeSession(r.Context(), token) + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + passwordCSRF := "" + if principal.User.PasswordChangeRequired { + passwordCSRF, err = authhttp.CSRFToken(token, "account:password:change") + if err != nil { + _ = s.identity.Auth.RevokeSession(r.Context(), token) + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + } + writeJSON(w, http.StatusOK, struct { + UserID string `json:"user_id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CSRFToken string `json:"csrf_token"` + PasswordChangeCSRF string `json:"password_change_csrf,omitempty"` + PasswordChangeRequired bool `json:"password_change_required"` + }{principal.User.ID, principal.User.Username, principal.User.DisplayName, csrf, passwordCSRF, principal.User.PasswordChangeRequired}) +} + +func (s *Server) changePassword(w http.ResponseWriter, r *http.Request) { + if !websec.SameOrigin(r, s.options.PublicOrigin) { + writeProblem(w, http.StatusForbidden, "same-origin request required") + return + } + principal, ok := auth.PrincipalFromContext(r.Context()) + token, tokenOK := authhttp.SessionToken(r, s.cookie) + if !ok || !tokenOK || !principal.User.PasswordChangeRequired || !authhttp.VerifyCSRF(token, "account:password:change", r.Header.Get("X-CSRF-Token")) { + writeProblem(w, http.StatusForbidden, "password change authorization required") + return + } + body := http.MaxBytesReader(w, r.Body, 8<<10) + defer body.Close() + var input struct { + CurrentPassword string `json:"current_password"` + NewPassword string `json:"new_password"` + } + if err := decodeOne(body, &input); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid password change request") + return + } + if err := s.identity.Auth.ChangePassword(r.Context(), principal.User.ID, input.CurrentPassword, input.NewPassword); err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "password change rejected") + return + } + if err := authhttp.ClearSession(w, s.cookie); err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) logout(w http.ResponseWriter, r *http.Request) { + if !websec.SameOrigin(r, s.options.PublicOrigin) { + writeProblem(w, http.StatusForbidden, "same-origin request required") + return + } + token, ok := authhttp.SessionToken(r, s.cookie) + if !ok || !authhttp.VerifyCSRF(token, "session:delete", r.Header.Get("X-CSRF-Token")) { + writeProblem(w, http.StatusForbidden, "valid session CSRF token required") + return + } + if err := s.identity.Auth.RevokeSession(r.Context(), token); err != nil && !errors.Is(err, auth.ErrSessionNotFound) { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + if err := authhttp.ClearSession(w, s.cookie); err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + w.WriteHeader(http.StatusNoContent) +} + +type queryRequest struct { + Query string `json:"query,omitempty"` + AST *query.AST `json:"ast,omitempty"` + OrganizationID string `json:"organization_id,omitempty"` + ProjectID string `json:"project_id,omitempty"` + EnvironmentID string `json:"environment_id,omitempty"` + ServiceID string `json:"service_id,omitempty"` +} + +func (s *Server) parseQuery(w http.ResponseWriter, r *http.Request) { + body := http.MaxBytesReader(w, r.Body, 16<<10) + defer body.Close() + var input queryRequest + if err := decodeOne(body, &input); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid query request") + return + } + ast, err := parseAST(input, s.options.MaxQueryRows) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "query rejected") + return + } + writeJSON(w, http.StatusOK, ast) +} + +func (s *Server) explainQuery(w http.ResponseWriter, r *http.Request) { + if !websec.SameOrigin(r, s.options.PublicOrigin) { + writeProblem(w, http.StatusForbidden, "same-origin request required") + return + } + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return + } + body := http.MaxBytesReader(w, r.Body, 16<<10) + defer body.Close() + var input queryRequest + if err := decodeOne(body, &input); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid query request") + return + } + ast, err := parseAST(input, s.options.MaxQueryRows) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "query rejected") + return + } + requested := access.Scope{OrganizationID: input.OrganizationID, ProjectID: input.ProjectID, EnvironmentID: input.EnvironmentID, ServiceID: input.ServiceID} + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, requested, identity.PermissionTelemetryQuery) + if err != nil { + writeProblem(w, http.StatusBadRequest, "invalid resource scope") + return + } + if !decision.Allowed { + writeProblem(w, http.StatusForbidden, "resource access denied") + return + } + if err := s.identity.ValidateResourceScope(r.Context(), requested); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid resource scope") + return + } + sensitive, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, requested, identity.PermissionTelemetryReadSensitive) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "authorization unavailable") + return + } + estimated, err := s.store.EstimateOrganizationBytes(input.OrganizationID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "query planning unavailable") + return + } + registry, _, err := s.store.ActiveDescriptors(r.Context(), input.OrganizationID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "query planning unavailable") + return + } + explain, err := query.Plan(ast, query.Scope{ + OrganizationID: input.OrganizationID, ProjectID: input.ProjectID, + EnvironmentID: input.EnvironmentID, ServiceID: input.ServiceID, + Sensitive: sensitive.Allowed, + }, registry, estimated, s.options.QueryBudget) + if errors.Is(err, query.ErrSensitivePermissionRequired) { + writeProblem(w, http.StatusForbidden, "sensitive-field permission required") + return + } + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "query plan rejected") + return + } + writeJSON(w, http.StatusOK, explain) +} + +func (s *Server) executeQuery(w http.ResponseWriter, r *http.Request) { + if !websec.SameOrigin(r, s.options.PublicOrigin) { + writeProblem(w, http.StatusForbidden, "same-origin request required") + return + } + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return + } + body := http.MaxBytesReader(w, r.Body, 16<<10) + defer body.Close() + var input queryRequest + if err := decodeOne(body, &input); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid query request") + return + } + ast, err := parseAST(input, s.options.MaxQueryRows) + if err != nil { + writeProblem(w, http.StatusUnprocessableEntity, "query rejected") + return + } + requested := access.Scope{OrganizationID: input.OrganizationID, ProjectID: input.ProjectID, EnvironmentID: input.EnvironmentID, ServiceID: input.ServiceID} + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, requested, identity.PermissionTelemetryQuery) + if err != nil { + writeProblem(w, http.StatusBadRequest, "invalid resource scope") + return + } + if !decision.Allowed { + writeProblem(w, http.StatusForbidden, "resource access denied") + return + } + if err = s.identity.ValidateResourceScope(r.Context(), requested); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid resource scope") + return + } + sensitive, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, requested, identity.PermissionTelemetryReadSensitive) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "authorization unavailable") + return + } + result, err := s.store.Query(r.Context(), ast, query.Scope{ + OrganizationID: input.OrganizationID, ProjectID: input.ProjectID, + EnvironmentID: input.EnvironmentID, ServiceID: input.ServiceID, + Sensitive: sensitive.Allowed, + }, s.options.QueryBudget, s.now()) + switch { + case errors.Is(err, query.ErrSensitivePermissionRequired): + writeProblem(w, http.StatusForbidden, "sensitive-field permission required") + return + case errors.Is(err, query.ErrBudgetExceeded): + writeProblem(w, http.StatusUnprocessableEntity, "query execution budget exceeded") + return + case errors.Is(err, query.ErrTypeMismatch): + writeProblem(w, http.StatusUnprocessableEntity, "query field type mismatch") + return + case err != nil: + writeProblem(w, http.StatusServiceUnavailable, "query execution unavailable") + return + } + writeJSON(w, http.StatusOK, result) +} + +func parseAST(input queryRequest, maxRows int) (query.AST, error) { + if (input.Query == "") == (input.AST == nil) { + return query.AST{}, errors.New("provide exactly one query or AST") + } + if input.AST == nil { + return query.Parse(input.Query, maxRows) + } + ast := *input.AST + var err error + if ast.WindowText != "" { + ast.Window, err = time.ParseDuration(ast.WindowText) + } + if err == nil && ast.BucketText != "" { + ast.Bucket, err = time.ParseDuration(ast.BucketText) + } + if err == nil { + err = query.Validate(ast, maxRows) + } + return ast, err +} + +func decodeOne(reader io.Reader, value any) error { + decoder := json.NewDecoder(reader) + decoder.DisallowUnknownFields() + if err := decoder.Decode(value); err != nil { + return err + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return errors.New("request must contain exactly one JSON value") + } + return nil +} + +func bearer(value string) (string, bool) { + const prefix = "Bearer " + if !strings.HasPrefix(value, prefix) || strings.ContainsAny(value[len(prefix):], " \t\r\n") { + return "", false + } + return value[len(prefix):], value[len(prefix):] != "" +} + +func textOK(body string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body))) + if r.Method != http.MethodHead { + _, _ = io.WriteString(w, body) + } + } +} + +func securityHeaders(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Content-Security-Policy", "default-src 'none'; base-uri 'none'; connect-src 'self'; form-action 'self'; frame-ancestors 'none'; img-src 'self'; manifest-src 'self'; script-src 'self'; style-src 'self'; worker-src 'self'") + w.Header().Set("Cross-Origin-Opener-Policy", "same-origin") + w.Header().Set("Permissions-Policy", "camera=(), geolocation=(), microphone=()") + w.Header().Set("Referrer-Policy", "no-referrer") + w.Header().Set("X-Content-Type-Options", "nosniff") + next.ServeHTTP(w, r) + }) +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} + +func writeProblem(w http.ResponseWriter, status int, title string) { + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(struct { + Title string `json:"title"` + Status int `json:"status"` + }{Title: title, Status: status}) +} diff --git a/internal/httpserver/server_test.go b/internal/httpserver/server_test.go new file mode 100644 index 0000000..2444085 --- /dev/null +++ b/internal/httpserver/server_test.go @@ -0,0 +1,496 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/nativeprotocol" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" + "gamertan.com/observatory/internal/storage" + commonpb "go.opentelemetry.io/proto/otlp/common/v1" + logspb "go.opentelemetry.io/proto/otlp/logs/v1" + "google.golang.org/protobuf/proto" +) + +func TestIngestionConcurrencyIsBoundedAndReusable(t *testing.T) { + server, store, identities, _ := newUITestServer(t) + defer store.Close() + defer identities.Close() + server.ingestSlots = make(chan struct{}, 1) + + first := httptest.NewRecorder() + leave, ok := server.enterIngest(first) + if !ok || leave == nil { + t.Fatal("first ingestion slot was unavailable") + } + second := httptest.NewRecorder() + if secondLeave, admitted := server.enterIngest(second); admitted || secondLeave != nil || second.Code != http.StatusServiceUnavailable || second.Header().Get("Retry-After") != "1" || !strings.Contains(second.Body.String(), "ingestion capacity temporarily unavailable") { + t.Fatalf("second admission admitted=%t status=%d retry=%q body=%s", admitted, second.Code, second.Header().Get("Retry-After"), second.Body.String()) + } + leave() + third := httptest.NewRecorder() + thirdLeave, admitted := server.enterIngest(third) + if !admitted || thirdLeave == nil { + t.Fatal("released ingestion capacity was not reusable") + } + thirdLeave() +} + +func TestServerWebPushConfigurationIsAllOrNothing(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + identities, err := identity.Open(root) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + dispatcher := &recordingPushDispatcher{} + options := testOptions() + options.PushDispatcher = dispatcher + if _, err = New(store, identities, options); err == nil { + t.Fatal("dispatcher without public key accepted") + } + options.PushPublicKey = "invalid" + if _, err = New(store, identities, options); err == nil { + t.Fatal("invalid public key accepted") + } + private, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + options.PushPublicKey = base64.RawURLEncoding.EncodeToString(private.PublicKey().Bytes()) + if _, err = New(store, identities, options); err != nil { + t.Fatalf("valid Web Push configuration: %v", err) + } +} + +func TestIngestDoesNotExposeValuesInErrors(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + token, err := store.CreateSource(context.Background(), "source", model.Scope{OrganizationID: "org", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"}) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + identities, err := identity.Open(root) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + server, err := New(store, identities, testOptions()) + if err != nil { + t.Fatal(err) + } + server.now = func() time.Time { return now } + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request", Body: "credential=do-not-echo"}}} + body, _ := json.Marshal(batch) + req := httptest.NewRequest(http.MethodPost, "/api/v1/ingest/native", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + rec := httptest.NewRecorder() + server.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusAccepted { + t.Fatalf("expected 202, got %d: %s", rec.Code, rec.Body.String()) + } + + req = httptest.NewRequest(http.MethodPost, "/api/v1/ingest/native", bytes.NewBufferString(`{"secret":"do-not-echo"}`)) + req.Header.Set("Authorization", "Bearer "+token) + rec = httptest.NewRecorder() + server.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest || bytes.Contains(rec.Body.Bytes(), []byte("do-not-echo")) { + t.Fatalf("unsafe error response %d: %s", rec.Code, rec.Body.String()) + } +} + +func TestFramedNativeIngestAcknowledgesExactReplayAndOverlappingTime(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + token, err := store.CreateSource(context.Background(), "source", model.Scope{OrganizationID: "org", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"}) + if err != nil { + t.Fatal(err) + } + identities, err := identity.Open(root) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + now := time.Date(2026, 8, 18, 20, 0, 0, 0, time.UTC) + server, err := New(store, identities, testOptions()) + if err != nil { + t.Fatal(err) + } + server.now = func() time.Time { return now } + handler := server.Handler() + send := func(batch model.Batch, body []byte, envelope model.BatchEnvelope) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v2/ingest/native", bytes.NewReader(body)) + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Content-Type", "application/json") + nativeprotocol.SetHeaders(request.Header, envelope) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "first"}}} + body, _ := json.Marshal(batch) + envelope, _ := batch.Envelope(body) + first := send(batch, body, envelope) + if first.Code != http.StatusAccepted || strings.Contains(first.Body.String(), `"duplicate":true`) { + t.Fatalf("first status=%d body=%s", first.Code, first.Body.String()) + } + replay := send(batch, body, envelope) + if replay.Code != http.StatusAccepted || !strings.Contains(replay.Body.String(), `"duplicate":true`) { + t.Fatalf("replay status=%d body=%s", replay.Code, replay.Body.String()) + } + tampered := append(append([]byte(nil), body...), ' ') + bad := send(batch, tampered, envelope) + if bad.Code != http.StatusBadRequest { + t.Fatalf("tampered status=%d body=%s", bad.Code, bad.Body.String()) + } + batch.Sequence = 2 + batch.ObservedAt = now.Add(time.Second) + // The second batch intentionally overlaps the first batch's observed time. + body, _ = json.Marshal(batch) + envelope, _ = batch.Envelope(body) + overlap := send(batch, body, envelope) + if overlap.Code != http.StatusAccepted { + t.Fatalf("overlap status=%d body=%s", overlap.Code, overlap.Body.String()) + } +} + +func TestOTLPHTTPIngestionIsAuthenticatedBoundedAndCompressed(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(context.Background(), "otlp-source", scope) + if err != nil { + t.Fatal(err) + } + identities, err := identity.Open(root) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + now := time.Date(2026, 8, 17, 3, 0, 0, 0, time.UTC) + server, err := New(store, identities, testOptions()) + if err != nil { + t.Fatal(err) + } + server.now = func() time.Time { return now } + handler := server.Handler() + payload, err := proto.Marshal(&logspb.LogsData{ResourceLogs: []*logspb.ResourceLogs{{ScopeLogs: []*logspb.ScopeLogs{{LogRecords: []*logspb.LogRecord{{ + TimeUnixNano: uint64(now.UnixNano()), EventName: "http.request", Body: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: "accepted"}}, + }}}}}}}) + if err != nil { + t.Fatal(err) + } + + send := func(body []byte, encoding string) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(http.MethodPost, "https://observatory.example/v1/logs", bytes.NewReader(body)) + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Content-Type", "application/x-protobuf") + if encoding != "" { + request.Header.Set("Content-Encoding", encoding) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response + } + + plain := send(payload, "") + if plain.Code != http.StatusOK || plain.Header().Get("Content-Type") != "application/x-protobuf" || plain.Header().Get("Content-Length") != "0" || plain.Body.Len() != 0 { + t.Fatalf("plain status=%d headers=%v body=%q", plain.Code, plain.Header(), plain.Body.String()) + } + var compressed bytes.Buffer + zipper := gzip.NewWriter(&compressed) + if _, err = zipper.Write(payload); err != nil { + t.Fatal(err) + } + if err = zipper.Close(); err != nil { + t.Fatal(err) + } + if response := send(compressed.Bytes(), "gzip"); response.Code != http.StatusOK { + t.Fatalf("gzip status=%d body=%s", response.Code, response.Body.String()) + } + if err = store.Recover(context.Background()); err != nil { + t.Fatal(err) + } + if bytes, err := store.EstimateOrganizationBytes(scope.OrganizationID); err != nil || bytes == 0 { + t.Fatalf("projection bytes=%d err=%v", bytes, err) + } + + unauthorized := httptest.NewRequest(http.MethodPost, "https://observatory.example/v1/logs", bytes.NewReader(payload)) + unauthorized.Header.Set("Content-Type", "application/x-protobuf") + unauthorizedResult := httptest.NewRecorder() + handler.ServeHTTP(unauthorizedResult, unauthorized) + if unauthorizedResult.Code != http.StatusUnauthorized { + t.Fatalf("unauthorized status=%d", unauthorizedResult.Code) + } + wrongMedia := httptest.NewRequest(http.MethodPost, "https://observatory.example/v1/logs", bytes.NewReader(payload)) + wrongMedia.Header.Set("Authorization", "Bearer "+token) + wrongMedia.Header.Set("Content-Type", "application/json") + wrongMediaResult := httptest.NewRecorder() + handler.ServeHTTP(wrongMediaResult, wrongMedia) + if wrongMediaResult.Code != http.StatusUnsupportedMediaType { + t.Fatalf("media status=%d", wrongMediaResult.Code) + } + + compressed.Reset() + zipper = gzip.NewWriter(&compressed) + if _, err = zipper.Write(bytes.Repeat([]byte{'x'}, int(testOptions().MaxBodyBytes)+1)); err != nil { + t.Fatal(err) + } + if err = zipper.Close(); err != nil { + t.Fatal(err) + } + tooLarge := send(compressed.Bytes(), "gzip") + if tooLarge.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversized status=%d body=%s", tooLarge.Code, tooLarge.Body.String()) + } +} + +func TestSessionAndScopedExplain(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + identities, err := identity.Open(root) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + bootstrap, err := identities.Bootstrap(context.Background(), identity.BootstrapInput{Username: "operator", Email: "operator@example.test", DisplayName: "Operator", Password: "correct horse battery staple"}) + if err != nil { + t.Fatal(err) + } + server, err := New(store, identities, testOptions()) + if err != nil { + t.Fatal(err) + } + handler := server.Handler() + + login := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v1/session", strings.NewReader(`{"identifier":"operator","password":"correct horse battery staple"}`)) + login.Header.Set("Origin", "https://observatory.example") + loginResult := httptest.NewRecorder() + handler.ServeHTTP(loginResult, login) + if loginResult.Code != http.StatusOK || strings.Contains(loginResult.Body.String(), "correct horse") { + t.Fatalf("login status=%d body=%s", loginResult.Code, loginResult.Body.String()) + } + var session struct { + CSRFToken string `json:"csrf_token"` + } + if err = json.Unmarshal(loginResult.Body.Bytes(), &session); err != nil || session.CSRFToken == "" { + t.Fatalf("session=%+v err=%v", session, err) + } + cookies := loginResult.Result().Cookies() + if len(cookies) != 1 || cookies[0].Name != "__Host-observatory_session" || !cookies[0].Secure || !cookies[0].HttpOnly { + t.Fatalf("cookies=%+v", cookies) + } + + explainBody := fmt.Sprintf(`{"organization_id":%q,"query":"logs | where status >= 500 | limit 10"}`, bootstrap.Organization.ID) + explain := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v1/query/explain", strings.NewReader(explainBody)) + explain.Header.Set("Origin", "https://observatory.example") + explain.AddCookie(cookies[0]) + explainResult := httptest.NewRecorder() + handler.ServeHTTP(explainResult, explain) + if explainResult.Code != http.StatusOK || !strings.Contains(explainResult.Body.String(), `"projected_sources"`) { + t.Fatalf("explain status=%d body=%s", explainResult.Code, explainResult.Body.String()) + } + now := time.Now().UTC() + sourceToken, err := store.CreateSource(context.Background(), "query-source", model.Scope{OrganizationID: bootstrap.Organization.ID, ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "query-source", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.http.request", Attributes: map[string]string{"http.status_code": "503", "http.route": "/failed", "workshop.queue_depth": "12"}}}} + if _, err = store.Ingest(context.Background(), sourceToken, batch, now); err != nil { + t.Fatal(err) + } + if err = store.Recover(context.Background()); err != nil { + t.Fatal(err) + } + execute := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v1/query", strings.NewReader(explainBody)) + execute.Header.Set("Origin", "https://observatory.example") + execute.AddCookie(cookies[0]) + executeResult := httptest.NewRecorder() + handler.ServeHTTP(executeResult, execute) + if executeResult.Code != http.StatusOK || !strings.Contains(executeResult.Body.String(), `"http.status_code"`) || !strings.Contains(executeResult.Body.String(), `"503"`) { + t.Fatalf("execute status=%d body=%s", executeResult.Code, executeResult.Body.String()) + } + reviewed := schema.Descriptor{Version: schema.DescriptorVersion, Signal: model.SignalLogs, Field: "workshop.queue_depth", Type: schema.TypeInteger, Meaning: "Reviewed queue depth for one application service.", Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, Index: schema.IndexRange, Retention: schema.RetentionRaw, ProjectionVersion: 1} + if _, err = store.ActivateDescriptor(context.Background(), bootstrap.Organization.ID, reviewed, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + customBody := fmt.Sprintf(`{"organization_id":%q,"query":"logs | where workshop.queue_depth >= 10 | limit 10"}`, bootstrap.Organization.ID) + for _, path := range []string{"/api/v1/query/explain", "/api/v1/query"} { + request := httptest.NewRequest(http.MethodPost, "https://observatory.example"+path, strings.NewReader(customBody)) + request.Header.Set("Origin", "https://observatory.example") + request.AddCookie(cookies[0]) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"workshop.queue_depth"`) || !strings.Contains(response.Body.String(), `"indexed":true`) { + t.Fatalf("path=%s status=%d body=%s", path, response.Code, response.Body.String()) + } + } + + for _, path := range []string{"/api/v1/query/explain", "/api/v1/query"} { + denied := httptest.NewRequest(http.MethodPost, "https://observatory.example"+path, strings.NewReader(`{"organization_id":"unowned1","query":"logs | limit 10"}`)) + denied.Header.Set("Origin", "https://observatory.example") + denied.AddCookie(cookies[0]) + deniedResult := httptest.NewRecorder() + handler.ServeHTTP(deniedResult, denied) + if deniedResult.Code != http.StatusForbidden { + t.Fatalf("path=%s cross-organization status=%d body=%s", path, deniedResult.Code, deniedResult.Body.String()) + } + } + + logout := httptest.NewRequest(http.MethodDelete, "https://observatory.example/api/v1/session", nil) + logout.Header.Set("Origin", "https://observatory.example") + logout.Header.Set("X-CSRF-Token", session.CSRFToken) + logout.AddCookie(cookies[0]) + logoutResult := httptest.NewRecorder() + handler.ServeHTTP(logoutResult, logout) + if logoutResult.Code != http.StatusNoContent { + t.Fatalf("logout status=%d body=%s", logoutResult.Code, logoutResult.Body.String()) + } +} + +func TestCrossOriginLoginIsRejected(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + identities, err := identity.Open(root) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + server, err := New(store, identities, testOptions()) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v1/session", strings.NewReader(`{"identifier":"operator","password":"secret"}`)) + request.Header.Set("Origin", "https://attacker.example") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusForbidden { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} + +func TestAgentEnrollmentIsSingleUseAndCredentialCanSelfRevoke(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + identities, err := identity.Open(root) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + now := time.Date(2026, 8, 17, 2, 0, 0, 0, time.UTC) + enrollmentToken, _, err := store.CreateEnrollment(context.Background(), "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}, "operator-a", 15*time.Minute, now) + if err != nil { + t.Fatal(err) + } + server, err := New(store, identities, testOptions()) + if err != nil { + t.Fatal(err) + } + server.now = func() time.Time { return now.Add(time.Minute) } + enroll := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v1/agent/enroll", nil) + enroll.Header.Set("Authorization", "Bearer "+enrollmentToken) + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, enroll) + if response.Code != http.StatusCreated || strings.Contains(response.Body.String(), enrollmentToken) { + t.Fatalf("enroll status=%d body=%s", response.Code, response.Body.String()) + } + var enrolled struct { + SourceID string `json:"source_id"` + Credential string `json:"credential"` + } + if err = json.Unmarshal(response.Body.Bytes(), &enrolled); err != nil || enrolled.SourceID != "source-a" || enrolled.Credential == "" { + t.Fatalf("enrolled=%+v err=%v", enrolled, err) + } + replay := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v1/agent/enroll", nil) + replay.Header.Set("Authorization", "Bearer "+enrollmentToken) + replayResponse := httptest.NewRecorder() + server.Handler().ServeHTTP(replayResponse, replay) + if replayResponse.Code != http.StatusUnauthorized { + t.Fatalf("replay status=%d", replayResponse.Code) + } + revoke := httptest.NewRequest(http.MethodDelete, "https://observatory.example/api/v1/agent/source", nil) + revoke.Header.Set("Authorization", "Bearer "+enrolled.Credential) + revokeResponse := httptest.NewRecorder() + server.Handler().ServeHTTP(revokeResponse, revoke) + if revokeResponse.Code != http.StatusNoContent { + t.Fatalf("revoke status=%d body=%s", revokeResponse.Code, revokeResponse.Body.String()) + } + if _, err = store.Authenticate(context.Background(), enrolled.Credential); err == nil { + t.Fatal("revoked source credential remained active") + } +} + +func testOptions() Options { + return Options{PublicOrigin: "https://observatory.example", MaxBodyBytes: 1 << 20, MaxQueryRows: 1000, SessionLifetime: time.Hour, QueryBudget: query.Budget{MaxDuration: 2 * time.Second, MaxRows: 1000, MaxScannedBytes: 10 << 20, MaxMemoryBytes: 8 << 20}} +} diff --git a/internal/httpserver/source_alerts_test.go b/internal/httpserver/source_alerts_test.go new file mode 100644 index 0000000..c166fb9 --- /dev/null +++ b/internal/httpserver/source_alerts_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/storage" +) + +func TestAgentAlertTransitionIsAuthenticatedBoundedAndEvidenceBacked(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + identities, err := identity.Open(root) + if err != nil { + t.Fatal(err) + } + defer identities.Close() + now := time.Date(2026, 8, 18, 23, 45, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(context.Background(), "source-a", scope) + if err != nil { + t.Fatal(err) + } + saved, err := store.SaveQuery(context.Background(), storage.SavedQueryInput{OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", MaxRows: 100, Name: "Logs", Query: "logs | limit 10", Scope: storage.ResourceScope{ProjectID: scope.ProjectID, EnvironmentID: scope.EnvironmentID, ServiceID: scope.ServiceID}}, now) + if err != nil { + t.Fatal(err) + } + rule, err := store.SaveAlertRule(context.Background(), storage.AlertRuleInput{OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", SavedQueryID: saved.ID, Name: "Logs", Severity: "warning", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true}, now) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}} + ingested, err := store.Ingest(context.Background(), token, batch, now) + if err != nil { + t.Fatal(err) + } + server, err := New(store, identities, testOptions()) + if err != nil { + t.Fatal(err) + } + server.now = func() time.Time { return now } + handler := server.Handler() + transition := model.AlertTransition{Version: model.AlertTransitionVersion, RuleID: rule.ID, RuleRevision: rule.Revision, AgentEpoch: strings.Repeat("a", 32), Sequence: 1, StreamID: batch.StreamID, BatchSequence: batch.Sequence, SegmentDigest: ingested.Digest, WindowStart: now, WindowEnd: now, State: "matched", ObservedAt: now} + payload, err := json.Marshal(transition) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v1/agent/alert-transition", bytes.NewReader(payload)) + request.Header.Set("Authorization", "Bearer "+token) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusAccepted { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + var ack storage.SourceAlertTransitionAck + if err = json.Unmarshal(response.Body.Bytes(), &ack); err != nil || ack.SourceID != "source-a" || ack.RuleID != rule.ID || ack.Duplicate { + t.Fatalf("ack=%+v err=%v", ack, err) + } + + unauthorized := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v1/agent/alert-transition", bytes.NewReader([]byte(`{"state":"do-not-echo"}`))) + unauthorizedResult := httptest.NewRecorder() + handler.ServeHTTP(unauthorizedResult, unauthorized) + if unauthorizedResult.Code != http.StatusUnauthorized || strings.Contains(unauthorizedResult.Body.String(), "do-not-echo") { + t.Fatalf("unauthorized status=%d body=%q", unauthorizedResult.Code, unauthorizedResult.Body.String()) + } + oversized := httptest.NewRequest(http.MethodPost, "https://observatory.example/api/v1/agent/alert-transition", strings.NewReader(strings.Repeat("x", (64<<10)+1))) + oversized.Header.Set("Authorization", "Bearer "+token) + oversizedResult := httptest.NewRecorder() + handler.ServeHTTP(oversizedResult, oversized) + if oversizedResult.Code != http.StatusBadRequest { + t.Fatalf("oversized status=%d body=%q", oversizedResult.Code, oversizedResult.Body.String()) + } +} diff --git a/internal/httpserver/ui.go b/internal/httpserver/ui.go new file mode 100644 index 0000000..b217a5d --- /dev/null +++ b/internal/httpserver/ui.go @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "bytes" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "strings" + "time" + "unicode/utf8" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/site" + "gamertan.com/sandwich-hime/sando" + "gamertan.com/web/access" + "gamertan.com/web/auth" + "gamertan.com/web/authhttp" + "gamertan.com/web/websec" +) + +const loginCSRFCookieName = "__Host-observatory_login_csrf" + +const ( + loginFormFailure = "This sign-in form expired or could not be verified. Please try again." + loginCredentialFailure = "The username or password was not accepted." + passwordFormFailure = "This password form expired or could not be verified. Please try again." + passwordMatchFailure = "The new passwords did not match. Please enter them again." + passwordChangeFailure = "The password could not be changed. Check the temporary password and choose a different password of at least 12 characters." +) + +func (s *Server) landing(w http.ResponseWriter, r *http.Request) { + if _, ok := auth.PrincipalFromContext(r.Context()); ok { + http.Redirect(w, r, "/app/", http.StatusSeeOther) + return + } + view := site.LandingView{Head: s.head("Gamertan Observatory", "A self-hosted observability platform in development for carefully operated Linux systems.", "/")} + s.renderHTML(w, r, http.StatusOK, site.Landing(view)) +} + +func (s *Server) loginPage(w http.ResponseWriter, r *http.Request) { + if principal, ok := auth.PrincipalFromContext(r.Context()); ok { + if principal.User.PasswordChangeRequired { + http.Redirect(w, r, "/account/password/", http.StatusSeeOther) + return + } + http.Redirect(w, r, "/app/", http.StatusSeeOther) + return + } + s.renderLogin(w, r, http.StatusOK, "") +} + +func (s *Server) loginForm(w http.ResponseWriter, r *http.Request) { + values, err := readFormFields(w, r, 16<<10, []string{"identifier", "password"}, []string{"csrf_token"}) + csrfOK := err == nil && validLoginCSRF(r, values.Get("csrf_token")) + if !tokenBoundFormRequest(r, s.options.PublicOrigin, csrfOK) { + s.renderLogin(w, r, http.StatusForbidden, loginFormFailure) + return + } + if err != nil { + s.renderLogin(w, r, http.StatusBadRequest, loginFormFailure) + return + } + token, principal, err := s.identity.Auth.Authenticate(r.Context(), values.Get("identifier"), values.Get("password"), s.options.SessionLifetime) + if err != nil { + s.renderLogin(w, r, http.StatusUnauthorized, loginCredentialFailure) + return + } + if err = authhttp.SetSession(w, s.cookie, token, s.now()); err != nil { + _ = s.identity.Auth.RevokeSession(r.Context(), token) + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + if _, cookieErr := r.Cookie(loginCSRFCookieName); cookieErr == nil { + clearLoginCSRF(w) + } + location := "/app/" + if principal.User.PasswordChangeRequired { + location = "/account/password/" + } + http.Redirect(w, r, location, http.StatusSeeOther) +} + +// tokenBoundFormRequest makes the purpose-bound form token the primary CSRF +// proof. Browser origin metadata is defense in depth: an explicit cross-site +// or contradictory origin still fails closed, while absent or opaque metadata +// does not break an otherwise valid ordinary HTML form submission. +func tokenBoundFormRequest(r *http.Request, publicOrigin string, validToken bool) bool { + if !validToken { + return false + } + fetchSite := strings.ToLower(strings.TrimSpace(r.Header.Get("Sec-Fetch-Site"))) + if fetchSite != "" && fetchSite != "same-origin" && fetchSite != "none" { + return false + } + origin := strings.TrimSpace(r.Header.Get("Origin")) + if origin == "" || origin == "null" { + return true + } + return websec.SameOrigin(r, publicOrigin) +} + +func (s *Server) renderLogin(w http.ResponseWriter, r *http.Request, status int, message string) { + csrf, err := s.issueLoginCSRF(w) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "login unavailable") + return + } + view := site.LoginView{Head: s.head("Sign in — Gamertan Observatory", "Sign in to the local Gamertan Observatory workshop.", "/login/"), CSRFToken: csrf, ErrorMessage: message} + s.renderHTML(w, r, status, site.Login(view)) +} + +func (s *Server) issueLoginCSRF(w http.ResponseWriter) (string, error) { + secret := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, secret); err != nil { + return "", errors.New("generate login CSRF token") + } + token := base64.RawURLEncoding.EncodeToString(secret) + http.SetCookie(w, &http.Cookie{Name: loginCSRFCookieName, Value: token, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, Expires: s.now().Add(10 * time.Minute), MaxAge: 600}) + return token, nil +} + +func validLoginCSRF(r *http.Request, candidate string) bool { + cookie, err := r.Cookie(loginCSRFCookieName) + if err != nil { + return false + } + want, wantErr := base64.RawURLEncoding.DecodeString(cookie.Value) + got, gotErr := base64.RawURLEncoding.DecodeString(candidate) + return wantErr == nil && gotErr == nil && len(want) == 32 && len(got) == len(want) && subtle.ConstantTimeCompare(want, got) == 1 +} + +func clearLoginCSRF(w http.ResponseWriter) { + http.SetCookie(w, &http.Cookie{Name: loginCSRFCookieName, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, MaxAge: -1}) +} + +func (s *Server) passwordPage(w http.ResponseWriter, r *http.Request) { + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok { + http.Redirect(w, r, "/login/", http.StatusSeeOther) + return + } + if !principal.User.PasswordChangeRequired { + http.Redirect(w, r, "/app/", http.StatusSeeOther) + return + } + s.renderPassword(w, r, http.StatusOK, "") +} + +func (s *Server) passwordForm(w http.ResponseWriter, r *http.Request) { + principal, ok := auth.PrincipalFromContext(r.Context()) + token, tokenOK := authhttp.SessionToken(r, s.cookie) + values, err := readForm(w, r, 8<<10, "csrf_token", "current_password", "new_password", "confirm_password") + csrfOK := err == nil && tokenOK && authhttp.VerifyCSRF(token, "account:password:change", values.Get("csrf_token")) + if !tokenBoundFormRequest(r, s.options.PublicOrigin, csrfOK) { + if ok && tokenOK && principal.User.PasswordChangeRequired { + s.renderPassword(w, r, http.StatusForbidden, passwordFormFailure) + } else { + http.Redirect(w, r, "/login/", http.StatusSeeOther) + } + return + } + if !ok || !tokenOK || !principal.User.PasswordChangeRequired { + writeProblem(w, http.StatusForbidden, "password change authorization required") + return + } + if err != nil || !csrfOK { + s.renderPassword(w, r, http.StatusBadRequest, passwordFormFailure) + return + } + if values.Get("new_password") != values.Get("confirm_password") { + s.renderPassword(w, r, http.StatusUnprocessableEntity, passwordMatchFailure) + return + } + if err = s.identity.Auth.ChangePassword(r.Context(), principal.User.ID, values.Get("current_password"), values.Get("new_password")); err != nil { + s.renderPassword(w, r, http.StatusUnprocessableEntity, passwordChangeFailure) + return + } + if err = authhttp.ClearSession(w, s.cookie); err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + http.Redirect(w, r, "/login/?password=changed", http.StatusSeeOther) +} + +func (s *Server) renderPassword(w http.ResponseWriter, r *http.Request, status int, message string) { + token, ok := authhttp.SessionToken(r, s.cookie) + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return + } + csrf, err := authhttp.CSRFToken(token, "account:password:change") + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + view := site.PasswordView{Head: s.head("Choose your password — Gamertan Observatory", "Replace the one-time Observatory credential before continuing.", "/account/password/"), CSRFToken: csrf, ErrorMessage: message} + s.renderHTML(w, r, status, site.Password(view)) +} + +func (s *Server) logoutForm(w http.ResponseWriter, r *http.Request) { + values, err := readForm(w, r, 4<<10, "csrf_token") + token, ok := authhttp.SessionToken(r, s.cookie) + csrfOK := err == nil && ok && authhttp.VerifyCSRF(token, "session:delete", values.Get("csrf_token")) + if !tokenBoundFormRequest(r, s.options.PublicOrigin, csrfOK) { + writeProblem(w, http.StatusForbidden, "valid sign-out form required") + return + } + if err != nil { + writeProblem(w, http.StatusBadRequest, "invalid sign-out request") + return + } + if !csrfOK { + writeProblem(w, http.StatusForbidden, "valid session CSRF token required") + return + } + if err = s.identity.Auth.RevokeSession(r.Context(), token); err != nil && !errors.Is(err, auth.ErrSessionNotFound) { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + if err = authhttp.ClearSession(w, s.cookie); err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + http.Redirect(w, r, "/login/", http.StatusSeeOther) +} + +func (s *Server) app(w http.ResponseWriter, r *http.Request) { + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok { + http.Redirect(w, r, "/login/", http.StatusSeeOther) + return + } + organizations, err := s.identity.OrganizationsForUser(r.Context(), principal.User.ID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "organization list unavailable") + return + } + if len(organizations) == 0 { + writeProblem(w, http.StatusForbidden, "organization access required") + return + } + queryValues := r.URL.Query() + requested := queryValues.Get("organization") + if len(queryValues) > 0 { + selectedValues, exists := queryValues["organization"] + if !exists || len(queryValues) != 1 || len(selectedValues) != 1 || selectedValues[0] == "" { + writeProblem(w, http.StatusBadRequest, "invalid organization selection") + return + } + } + selected := organizations[0] + if requested != "" { + found := false + for _, organization := range organizations { + if organization.ID == requested { + selected, found = organization, true + break + } + } + if !found { + writeProblem(w, http.StatusForbidden, "organization access denied") + return + } + } + scope := access.Scope{OrganizationID: selected.ID} + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionDashboardsRead) + if err != nil || !decision.Allowed { + writeProblem(w, http.StatusForbidden, "dashboard access denied") + return + } + token, ok := authhttp.SessionToken(r, s.cookie) + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return + } + csrf, err := authhttp.CSRFToken(token, "session:delete") + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + view := site.AppView{ + Head: s.head("Overview — Gamertan Observatory", "Recent authorized telemetry and saved Observatory work.", "/app/"), + DisplayName: principal.User.DisplayName, CSRFToken: csrf, + EventsURL: "/app/events?organization=" + url.QueryEscape(selected.ID), + RefreshedAt: s.now().Format("2006-01-02 15:04:05 UTC"), + Organization: site.OrganizationOption{ID: selected.ID, Name: selected.Name, Selected: true}, + IncidentsURL: "/app/incidents/?organization=" + url.QueryEscape(selected.ID), + } + projectionStatus, projectionErr := s.store.OrganizationProjectionStatus(r.Context(), selected.ID, s.now()) + if projectionErr == nil && projectionStatus.PendingSegments > 0 { + view.PendingBatches = projectionStatus.PendingSegments + view.ProjectionLag = formatProjectionLag(projectionStatus.OldestPendingLag) + } + incidentDecision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionIncidentsRead) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "authorization unavailable") + return + } + if incidentDecision.Allowed { + incidents, incidentErr := s.store.Incidents(r.Context(), selected.ID, false, 100) + if incidentErr != nil { + writeProblem(w, http.StatusServiceUnavailable, "incidents unavailable") + return + } + view.OpenIncidents = len(incidents) + } + manage, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionDashboardsManage) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "authorization unavailable") + return + } + view.CanManage = manage.Allowed + if view.CanManage { + view.ManageCSRF, err = authhttp.CSRFToken(token, "dashboards:manage") + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "session unavailable") + return + } + } + for _, organization := range organizations { + view.Organizations = append(view.Organizations, site.OrganizationOption{ID: organization.ID, Name: organization.Name, Selected: organization.ID == selected.ID}) + } + view.Signals = s.overviewSignals(r, selected.ID) + saved, err := s.store.SavedQueries(r.Context(), selected.ID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "saved queries unavailable") + return + } + for _, item := range saved { + view.SavedQueries = append(view.SavedQueries, site.SavedQuerySummary{ID: item.ID, Name: item.Name, Description: item.Description, Query: item.Query}) + } + dashboards, err := s.store.Dashboards(r.Context(), selected.ID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "dashboards unavailable") + return + } + for _, item := range dashboards { + view.Dashboards = append(view.Dashboards, site.DashboardSummary{Slug: item.Slug, Name: item.Name, Description: item.Description, PanelCount: len(item.Panels)}) + } + s.renderHTML(w, r, http.StatusOK, site.App(view)) +} + +func formatProjectionLag(lag time.Duration) string { + if lag < time.Second { + return "less than one second" + } + return lag.Round(time.Second).String() +} + +func (s *Server) overviewSignals(r *http.Request, organizationID string) []site.SignalView { + definitions := []struct { + signal model.Signal + id, name string + description string + }{ + {model.SignalLogs, "logs", "logs", "Recent structured events accepted for this organization."}, + {model.SignalMetrics, "metrics", "metrics", "Recent numeric observations with a table alternative."}, + {model.SignalTraces, "traces", "traces", "Recent spans and their correlation identities."}, + {model.SignalDeployments, "deployments", "deployments", "Recent bounded deployment evidence."}, + } + views := make([]site.SignalView, 0, len(definitions)) + for _, definition := range definitions { + text := string(definition.signal) + " | window 1h | limit 20" + ast, err := query.Parse(text, s.options.MaxQueryRows) + var result query.Result + if err == nil { + result, err = s.store.Query(r.Context(), ast, query.Scope{OrganizationID: organizationID}, s.options.QueryBudget, s.now()) + } + table := site.TableView{Caption: "Recent " + definition.name, Columns: []site.TableColumn{{Label: "Status"}}, Empty: "No observations are available in the last hour."} + if err != nil { + table.Empty = "This bounded query is temporarily unavailable." + } else { + table = resultTable("Recent "+definition.name, result) + } + views = append(views, site.SignalView{ID: definition.id, Name: definition.name, Description: definition.description, Query: text, Table: table}) + } + return views +} + +func resultTable(caption string, result query.Result) site.TableView { + table := site.TableView{Caption: caption, Empty: "No observations are available in the last hour."} + for _, column := range result.Columns { + table.Columns = append(table.Columns, site.TableColumn{Label: column.Field, Unit: column.Unit}) + } + for _, row := range result.Rows { + view := site.TableRow{Values: make([]string, len(result.Columns))} + for index := range view.Values { + view.Values[index] = "—" + if index < len(row.Values) && row.Values[index] != nil { + view.Values[index] = boundedCell(*row.Values[index]) + } + } + table.Rows = append(table.Rows, view) + } + return table +} + +func boundedCell(value string) string { + const maxRunes = 256 + if utf8.RuneCountInString(value) <= maxRunes { + return value + } + runes := []rune(value) + return string(runes[:maxRunes]) + "…" +} + +func (s *Server) events(w http.ResponseWriter, r *http.Request) { + principal, ok := auth.PrincipalFromContext(r.Context()) + if !ok { + writeProblem(w, http.StatusUnauthorized, "authentication required") + return + } + organizationID := r.URL.Query().Get("organization") + if len(r.URL.Query()) != 1 || len(r.URL.Query()["organization"]) != 1 { + writeProblem(w, http.StatusBadRequest, "organization is required") + return + } + scope := access.Scope{OrganizationID: organizationID} + if err := s.identity.ValidateResourceScope(r.Context(), scope); err != nil { + writeProblem(w, http.StatusBadRequest, "invalid organization") + return + } + decision, err := s.identity.Access.Authorize(r.Context(), principal.User.ID, scope, identity.PermissionDashboardsRead) + if err != nil || !decision.Allowed { + writeProblem(w, http.StatusForbidden, "dashboard access denied") + return + } + updates, remove, err := s.refresh.subscribe(organizationID) + if err != nil { + writeProblem(w, http.StatusServiceUnavailable, "live refresh capacity reached") + return + } + defer remove() + flusher, ok := w.(http.Flusher) + if !ok { + writeProblem(w, http.StatusNotImplemented, "streaming unavailable") + return + } + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Accel-Buffering", "no") + _, _ = io.WriteString(w, "event: ready\ndata: {}\n\n") + flusher.Flush() + heartbeat := time.NewTicker(20 * time.Second) + defer heartbeat.Stop() + for { + select { + case <-r.Context().Done(): + return + case <-updates: + _, _ = io.WriteString(w, "event: refresh\ndata: {}\n\n") + flusher.Flush() + case <-heartbeat.C: + _, _ = io.WriteString(w, ": keepalive\n\n") + flusher.Flush() + } + } +} + +func (s *Server) serveAsset(w http.ResponseWriter, r *http.Request) { + body, contentType, ok := site.Asset(r.URL.Path) + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body))) + if r.Method != http.MethodHead { + _, _ = w.Write(body) + } +} + +func (s *Server) head(title, description, path string) site.HeadView { + return site.HeadView{Title: title, Description: description, CanonicalURL: s.options.PublicOrigin + path, Assets: site.AssetPaths()} +} + +func (s *Server) renderHTML(w http.ResponseWriter, r *http.Request, status int, component sando.Component) { + var body bytes.Buffer + if err := sando.Render(r.Context(), &body, component); err != nil { + writeProblem(w, http.StatusInternalServerError, "interface render unavailable") + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Content-Length", fmt.Sprintf("%d", body.Len())) + w.WriteHeader(status) + if r.Method != http.MethodHead { + _, _ = w.Write(body.Bytes()) + } +} + +func readForm(w http.ResponseWriter, r *http.Request, limit int64, fields ...string) (url.Values, error) { + return readFormFields(w, r, limit, fields, nil) +} + +func readFormFields(w http.ResponseWriter, r *http.Request, limit int64, required, optional []string) (url.Values, error) { + mediaType, parameters, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/x-www-form-urlencoded" || len(parameters) > 1 || len(parameters) == 1 && !strings.EqualFold(parameters["charset"], "utf-8") { + return nil, errors.New("URL-encoded form required") + } + body := http.MaxBytesReader(w, r.Body, limit) + defer body.Close() + encoded, err := io.ReadAll(body) + if err != nil || !utf8.Valid(encoded) { + return nil, errors.New("invalid form body") + } + values, err := url.ParseQuery(string(encoded)) + if err != nil || len(values) != len(required)+len(optional) { + return nil, errors.New("invalid form fields") + } + for _, field := range required { + if len(values[field]) != 1 || values.Get(field) == "" { + return nil, errors.New("invalid form field") + } + } + for _, field := range optional { + if len(values[field]) != 1 { + return nil, errors.New("invalid form field") + } + } + return values, nil +} diff --git a/internal/httpserver/ui_test.go b/internal/httpserver/ui_test.go new file mode 100644 index 0000000..3e53d89 --- /dev/null +++ b/internal/httpserver/ui_test.go @@ -0,0 +1,1122 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpserver + +import ( + "bytes" + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "html" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" + "gamertan.com/observatory/internal/site" + "gamertan.com/observatory/internal/storage" + "gamertan.com/web/authhttp" +) + +func TestHandlerAssignsFreshBoundedRequestIDs(t *testing.T) { + server, store, identities, _ := newUITestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + + var previous string + for _, test := range []struct { + method string + target string + }{ + {http.MethodGet, "https://observatory.example/healthz"}, + {http.MethodHead, "https://observatory.example/readyz"}, + {http.MethodGet, "https://observatory.example/"}, + {http.MethodGet, "https://observatory.example/not-found"}, + } { + header := http.Header{"X-Request-ID": []string{"attacker-selected"}} + response := perform(handler, test.method, test.target, nil, nil, header) + requestID := response.Header().Get("X-Request-ID") + decoded, err := hex.DecodeString(requestID) + if err != nil || len(decoded) != 16 || requestID == "attacker-selected" || requestID == previous { + t.Fatalf("%s %s request_id=%q decoded=%d err=%v previous=%q", test.method, test.target, requestID, len(decoded), err, previous) + } + previous = requestID + } +} + +func TestHTMLInterfaceAuthenticationAssetsAndOverview(t *testing.T) { + server, store, identities, bootstrap := newUITestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + + landing := perform(handler, http.MethodGet, "https://observatory.example/", nil, nil) + if landing.Code != http.StatusOK || landing.Header().Get("Content-Type") != "text/html; charset=utf-8" || !strings.Contains(landing.Body.String(), "Keep the evidence close.") { + t.Fatalf("landing status=%d body=%s", landing.Code, landing.Body.String()) + } + csp := landing.Header().Get("Content-Security-Policy") + if strings.Contains(csp, "unsafe-inline") || !strings.Contains(csp, "script-src 'self'") || !strings.Contains(csp, "style-src 'self'") || !strings.Contains(csp, "manifest-src 'self'") || !strings.Contains(csp, "worker-src 'self'") || strings.Contains(landing.Body.String(), "`) { + t.Fatalf("landing omitted manifest discovery: %s", landing.Body.String()) + } + head := perform(handler, http.MethodHead, "https://observatory.example/", nil, nil) + if head.Code != http.StatusOK || head.Body.Len() != 0 || head.Header().Get("Content-Length") != landing.Header().Get("Content-Length") { + t.Fatalf("HEAD status=%d length=%q body=%d", head.Code, head.Header().Get("Content-Length"), head.Body.Len()) + } + for _, path := range []string{site.AssetPaths().StylePath, site.AssetPaths().ScriptPath, site.AssetPaths().IconPath} { + asset := perform(handler, http.MethodGet, "https://observatory.example"+path, nil, nil) + if asset.Code != http.StatusOK || asset.Body.Len() == 0 || asset.Header().Get("Cache-Control") != "public, max-age=31536000, immutable" { + t.Fatalf("asset %s status=%d cache=%q", path, asset.Code, asset.Header().Get("Cache-Control")) + } + assetHead := perform(handler, http.MethodHead, "https://observatory.example"+path, nil, nil) + if assetHead.Code != http.StatusOK || assetHead.Body.Len() != 0 || assetHead.Header().Get("Content-Length") != asset.Header().Get("Content-Length") { + t.Fatalf("asset HEAD %s status=%d", path, assetHead.Code) + } + } + manifest := perform(handler, http.MethodGet, "https://observatory.example/manifest.webmanifest", nil, nil) + if manifest.Code != http.StatusOK || manifest.Header().Get("Content-Type") != "application/manifest+json" || !strings.Contains(manifest.Body.String(), site.AssetPaths().IconPath) { + t.Fatalf("manifest status=%d headers=%v body=%s", manifest.Code, manifest.Header(), manifest.Body.String()) + } + manifestHead := perform(handler, http.MethodHead, "https://observatory.example/manifest.webmanifest", nil, nil) + if manifestHead.Code != http.StatusOK || manifestHead.Body.Len() != 0 || manifestHead.Header().Get("Content-Length") != manifest.Header().Get("Content-Length") { + t.Fatalf("manifest HEAD status=%d body=%d", manifestHead.Code, manifestHead.Body.Len()) + } + worker := perform(handler, http.MethodGet, "https://observatory.example/service-worker.js", nil, nil) + if worker.Code != http.StatusOK || worker.Header().Get("Content-Type") != "text/javascript; charset=utf-8" || worker.Header().Get("Cache-Control") != "no-cache" || worker.Header().Get("Service-Worker-Allowed") != "/" || !strings.Contains(worker.Body.String(), "cache-inbox") { + t.Fatalf("worker status=%d headers=%v", worker.Code, worker.Header()) + } + offline := perform(handler, http.MethodGet, "https://observatory.example/offline/", nil, nil) + if offline.Code != http.StatusOK || !strings.Contains(offline.Body.String(), "The evidence is still safe.") { + t.Fatalf("offline status=%d body=%s", offline.Code, offline.Body.String()) + } + disabledPush := perform(handler, http.MethodPost, "https://observatory.example/api/v1/push/subscription", strings.NewReader(`{}`), nil) + if disabledPush.Code != http.StatusNotFound { + t.Fatalf("disabled push status=%d body=%s", disabledPush.Code, disabledPush.Body.String()) + } + + unauthenticated := perform(handler, http.MethodGet, "https://observatory.example/app/", nil, nil) + if unauthenticated.Code != http.StatusSeeOther || unauthenticated.Header().Get("Location") != "/login/" { + t.Fatalf("unauthenticated status=%d location=%q", unauthenticated.Code, unauthenticated.Header().Get("Location")) + } + extraQuery := perform(handler, http.MethodGet, "https://observatory.example/app/?unexpected=true", nil, []*http.Cookie{loginHTML(t, handler)}) + if extraQuery.Code != http.StatusBadRequest { + t.Fatalf("unexpected query status=%d", extraQuery.Code) + } + + loginCookie := loginHTML(t, handler) + now := server.now() + token, err := store.CreateSource(context.Background(), "ui-source", model.Scope{OrganizationID: bootstrap.Organization.ID, ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "ui-source", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.http.request", Attributes: map[string]string{"http.status_code": "200", "http.route": "/"}}}} + body, _ := json.Marshal(batch) + updates, remove, err := server.refresh.subscribe(bootstrap.Organization.ID) + if err != nil { + t.Fatal(err) + } + defer remove() + ingestHeaders := http.Header{"Authorization": []string{"Bearer " + token}, "Content-Type": []string{"application/json"}} + ingest := perform(handler, http.MethodPost, "https://observatory.example/api/v1/ingest/native", bytes.NewReader(body), nil, ingestHeaders) + if ingest.Code != http.StatusAccepted { + t.Fatalf("ingest status=%d body=%s", ingest.Code, ingest.Body.String()) + } + pending := perform(handler, http.MethodGet, "https://observatory.example/app/", nil, []*http.Cookie{loginCookie}) + if pending.Code != http.StatusOK || !strings.Contains(pending.Body.String(), "Durable evidence is still being indexed.") || !strings.Contains(pending.Body.String(), "Accepted batches safely stored: 1.") || !strings.Contains(pending.Body.String(), `role="status"`) { + t.Fatalf("pending app status=%d body=%s", pending.Code, pending.Body.String()) + } + if err = store.Recover(context.Background()); err != nil { + t.Fatal(err) + } + select { + case <-updates: + default: + t.Fatal("successful ingest did not publish organization refresh") + } + + app := perform(handler, http.MethodGet, "https://observatory.example/app/", nil, []*http.Cookie{loginCookie}) + if app.Code != http.StatusOK || !strings.Contains(app.Body.String(), bootstrap.Organization.Name) || !strings.Contains(app.Body.String(), "application.http.request") || !strings.Contains(app.Body.String(), "Recent metrics") || !strings.Contains(app.Body.String(), "Recent traces") || !strings.Contains(app.Body.String(), "Recent deployments") || !strings.Contains(app.Body.String(), `action="/app/queries/builder/"`) || !strings.Contains(app.Body.String(), "Build a query") { + t.Fatalf("app status=%d body=%s", app.Code, app.Body.String()) + } + if strings.Contains(app.Body.String(), "Durable evidence is still being indexed.") { + t.Fatalf("app retained indexing status after projection completed: %s", app.Body.String()) + } + for _, forbidden := range []string{"Render #", "request number", "position:sticky", "unsafe-inline"} { + if strings.Contains(app.Body.String(), forbidden) { + t.Fatalf("app exposed forbidden marker %q", forbidden) + } + } + appHead := perform(handler, http.MethodHead, "https://observatory.example/app/", nil, []*http.Cookie{loginCookie}) + if appHead.Code != http.StatusOK || appHead.Body.Len() != 0 || appHead.Header().Get("Content-Length") != app.Header().Get("Content-Length") { + t.Fatalf("app HEAD status=%d body=%d", appHead.Code, appHead.Body.Len()) + } + + sessionToken := loginCookie.Value + csrf, err := authhttp.CSRFToken(sessionToken, "session:delete") + if err != nil { + t.Fatal(err) + } + logout := url.Values{"csrf_token": []string{csrf}}.Encode() + logoutHeaders := http.Header{"Origin": []string{"null"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + loggedOut := perform(handler, http.MethodPost, "https://observatory.example/logout/", strings.NewReader(logout), []*http.Cookie{loginCookie}, logoutHeaders) + if loggedOut.Code != http.StatusSeeOther || loggedOut.Header().Get("Location") != "/login/" { + t.Fatalf("logout status=%d location=%q", loggedOut.Code, loggedOut.Header().Get("Location")) + } +} + +func TestExploreWorkbenchUsesBoundedServerRenderedQueries(t *testing.T) { + server, store, identities, bootstrap := newUITestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + + unauthenticated := perform(handler, http.MethodGet, "https://observatory.example/app/explore/?organization="+url.QueryEscape(bootstrap.Organization.ID), nil, nil) + if unauthenticated.Code != http.StatusSeeOther || unauthenticated.Header().Get("Location") != "/login/" { + t.Fatalf("unauthenticated status=%d location=%q", unauthenticated.Code, unauthenticated.Header().Get("Location")) + } + cookie := loginHTML(t, handler) + missingOrganization := perform(handler, http.MethodGet, "https://observatory.example/app/explore/", nil, []*http.Cookie{cookie}) + if missingOrganization.Code != http.StatusBadRequest { + t.Fatalf("missing organization status=%d body=%s", missingOrganization.Code, missingOrganization.Body.String()) + } + target := "https://observatory.example/app/explore/?organization=" + url.QueryEscape(bootstrap.Organization.ID) + page := perform(handler, http.MethodGet, target, nil, []*http.Cookie{cookie}) + pageBody := page.Body.String() + currentLink := `Explore` + if page.Code != http.StatusOK || !strings.Contains(pageBody, "Follow the evidence.") || !strings.Contains(pageBody, currentLink) || !strings.Contains(pageBody, defaultExploreQuery) || strings.Contains(pageBody, "Authorized query results") { + t.Fatalf("explore status=%d body=%s", page.Code, pageBody) + } + if !strings.Contains(pageBody, `method="post" action="/app/explore/?organization=`+bootstrap.Organization.ID+`"`) || strings.Contains(pageBody, "?query=") { + t.Fatalf("explore form did not keep query in POST body: %s", pageBody) + } + head := perform(handler, http.MethodHead, target, nil, []*http.Cookie{cookie}) + if head.Code != http.StatusOK || head.Body.Len() != 0 || head.Header().Get("Content-Length") != page.Header().Get("Content-Length") { + t.Fatalf("explore HEAD status=%d length=%q body=%d", head.Code, head.Header().Get("Content-Length"), head.Body.Len()) + } + + now := server.now() + sourceToken, err := store.CreateSource(context.Background(), "explore-source", model.Scope{OrganizationID: bootstrap.Organization.ID, ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "explore-source", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.http.request", Attributes: map[string]string{"http.status_code": "200", "http.route": "/explore-proof"}}}} + if _, err = store.Ingest(context.Background(), sourceToken, batch, now); err != nil { + t.Fatal(err) + } + if err = store.Recover(context.Background()); err != nil { + t.Fatal(err) + } + csrf, err := authhttp.CSRFToken(cookie.Value, "query:execute") + if err != nil { + t.Fatal(err) + } + queryText := `logs | where route == "/explore-proof" | window 1h | limit 10` + form := url.Values{"csrf_token": []string{csrf}, "query": []string{queryText}}.Encode() + headers := http.Header{"Content-Type": []string{"application/x-www-form-urlencoded"}} + result := perform(handler, http.MethodPost, target, strings.NewReader(form), []*http.Cookie{cookie}, headers) + resultBody := result.Body.String() + for _, required := range []string{"Authorized query results", "/explore-proof", "Scanned rows", "Matched rows", "Scanned bytes", "Execution", html.EscapeString(queryText)} { + if !strings.Contains(resultBody, required) { + t.Fatalf("result omitted %q: status=%d body=%s", required, result.Code, resultBody) + } + } + if result.Code != http.StatusOK || strings.Contains(result.Header().Get("Content-Type"), "application/json") { + t.Fatalf("query status=%d type=%q body=%s", result.Code, result.Header().Get("Content-Type"), resultBody) + } + + invalidForm := url.Values{"csrf_token": []string{"invalid"}, "query": []string{"logs | limit 10"}}.Encode() + invalid := perform(handler, http.MethodPost, target, strings.NewReader(invalidForm), []*http.Cookie{cookie}, headers) + if invalid.Code != http.StatusForbidden || !strings.Contains(invalid.Body.String(), "query form expired") || strings.Contains(invalid.Header().Get("Content-Type"), "application/json") { + t.Fatalf("invalid CSRF status=%d body=%s", invalid.Code, invalid.Body.String()) + } + + badQuery := url.Values{"csrf_token": []string{csrf}, "query": []string{"logs | become unbounded"}}.Encode() + rejected := perform(handler, http.MethodPost, target, strings.NewReader(badQuery), []*http.Cookie{cookie}, headers) + if rejected.Code != http.StatusUnprocessableEntity || !strings.Contains(rejected.Body.String(), "could not be parsed") || !strings.Contains(rejected.Body.String(), "logs | become unbounded") { + t.Fatalf("rejected query status=%d body=%s", rejected.Code, rejected.Body.String()) + } +} + +func TestHTMLLoginFailsClosed(t *testing.T) { + server, store, identities, _ := newUITestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + + wrongOrigin := http.Header{"Origin": []string{"https://attacker.example"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + result := perform(handler, http.MethodPost, "https://observatory.example/login/", strings.NewReader("identifier=operator&password=do-not-echo"), nil, wrongOrigin) + if result.Code != http.StatusForbidden || !strings.Contains(result.Body.String(), loginFormFailure) || strings.Contains(result.Body.String(), "do-not-echo") || strings.Contains(result.Header().Get("Content-Type"), "application/json") { + t.Fatalf("wrong origin status=%d body=%s", result.Code, result.Body.String()) + } + csrfCookie, csrfToken := loginFormCSRF(t, handler) + extraField := http.Header{"Origin": []string{"https://observatory.example"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + extraForm := url.Values{"csrf_token": []string{csrfToken}, "identifier": []string{"operator"}, "password": []string{"wrong"}, "next": []string{"https://attacker.example"}}.Encode() + result = perform(handler, http.MethodPost, "https://observatory.example/login/", strings.NewReader(extraForm), []*http.Cookie{csrfCookie}, extraField) + if result.Code != http.StatusForbidden || !strings.Contains(result.Body.String(), loginFormFailure) || strings.Contains(result.Body.String(), "attacker.example") || result.Header().Get("Location") != "" { + t.Fatalf("extra field status=%d location=%q body=%s", result.Code, result.Header().Get("Location"), result.Body.String()) + } +} + +func TestHTMLLoginUsesTokenWhenBrowserOmitsOrObscuresOriginMetadata(t *testing.T) { + server, store, identities, _ := newUITestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + + page := perform(handler, http.MethodGet, "https://observatory.example/login/", nil, nil) + if page.Code != http.StatusOK { + t.Fatalf("login page status=%d body=%s", page.Code, page.Body.String()) + } + var csrfCookie *http.Cookie + for _, cookie := range page.Result().Cookies() { + if cookie.Name == loginCSRFCookieName { + csrfCookie = cookie + } + } + if csrfCookie == nil || !csrfCookie.Secure || !csrfCookie.HttpOnly || csrfCookie.SameSite != http.SameSiteStrictMode || csrfCookie.MaxAge != 600 { + t.Fatalf("login CSRF cookie=%+v", csrfCookie) + } + const marker = `name="csrf_token" value="` + start := strings.Index(page.Body.String(), marker) + if start < 0 { + t.Fatalf("login page omitted CSRF token: %s", page.Body.String()) + } + start += len(marker) + end := strings.IndexByte(page.Body.String()[start:], '"') + if end < 0 { + t.Fatal("login page CSRF token is unterminated") + } + token := page.Body.String()[start : start+end] + if token == "" || token != csrfCookie.Value { + t.Fatal("login form and cookie CSRF tokens differ") + } + + form := url.Values{"csrf_token": []string{token}, "identifier": []string{"not-a-user"}, "password": []string{"not-a-password"}}.Encode() + contentType := http.Header{"Content-Type": []string{"application/x-www-form-urlencoded"}} + omittedMetadata := perform(handler, http.MethodPost, "https://observatory.example/login/", strings.NewReader(form), []*http.Cookie{csrfCookie}, contentType) + if omittedMetadata.Code != http.StatusUnauthorized || strings.Contains(omittedMetadata.Body.String(), "not-a-password") { + t.Fatalf("origin-metadata fallback status=%d body=%s", omittedMetadata.Code, omittedMetadata.Body.String()) + } + opaqueSameOriginHeaders := contentType.Clone() + opaqueSameOriginHeaders.Set("Origin", "null") + opaqueSameOriginHeaders.Set("Sec-Fetch-Site", "same-origin") + opaqueSameOrigin := perform(handler, http.MethodPost, "https://observatory.example/login/", strings.NewReader(form), []*http.Cookie{csrfCookie}, opaqueSameOriginHeaders) + if opaqueSameOrigin.Code != http.StatusUnauthorized || strings.Contains(opaqueSameOrigin.Body.String(), "not-a-password") { + t.Fatalf("opaque same-origin fallback status=%d body=%s", opaqueSameOrigin.Code, opaqueSameOrigin.Body.String()) + } + + withoutToken := perform(handler, http.MethodPost, "https://observatory.example/login/", strings.NewReader("identifier=not-a-user&password=not-a-password"), nil, contentType) + if withoutToken.Code != http.StatusForbidden { + t.Fatalf("originless tokenless status=%d body=%s", withoutToken.Code, withoutToken.Body.String()) + } + opaqueWithoutToken := perform(handler, http.MethodPost, "https://observatory.example/login/", strings.NewReader("identifier=not-a-user&password=not-a-password"), nil, opaqueSameOriginHeaders) + if opaqueWithoutToken.Code != http.StatusForbidden { + t.Fatalf("opaque tokenless status=%d body=%s", opaqueWithoutToken.Code, opaqueWithoutToken.Body.String()) + } + opaqueWithoutFetchMetadata := contentType.Clone() + opaqueWithoutFetchMetadata.Set("Origin", "null") + opaqueTokenOnly := perform(handler, http.MethodPost, "https://observatory.example/login/", strings.NewReader(form), []*http.Cookie{csrfCookie}, opaqueWithoutFetchMetadata) + if opaqueTokenOnly.Code != http.StatusUnauthorized || !strings.Contains(opaqueTokenOnly.Body.String(), loginCredentialFailure) { + t.Fatalf("opaque origin without same-origin fetch metadata status=%d body=%s", opaqueTokenOnly.Code, opaqueTokenOnly.Body.String()) + } + wrongOrigin := contentType.Clone() + wrongOrigin.Set("Origin", "https://attacker.example") + crossSite := perform(handler, http.MethodPost, "https://observatory.example/login/", strings.NewReader(form), []*http.Cookie{csrfCookie}, wrongOrigin) + if crossSite.Code != http.StatusForbidden { + t.Fatalf("cross-site token replay status=%d body=%s", crossSite.Code, crossSite.Body.String()) + } +} + +func TestTemporaryOperatorMustRotatePasswordBeforeUsingApplication(t *testing.T) { + server, store, identities, _ := newRotationTestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + const temporary = "temporary correct horse battery staple" + const replacement = "permanent correct horse battery staple" + + headers := http.Header{"Origin": []string{"https://observatory.example"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + login := loginHTMLResponse(t, handler, "operator", temporary) + if login.Code != http.StatusSeeOther || login.Header().Get("Location") != "/account/password/" || strings.Contains(login.Body.String(), temporary) { + t.Fatalf("login status=%d location=%q body=%s", login.Code, login.Header().Get("Location"), login.Body.String()) + } + var cookie *http.Cookie + for _, candidate := range login.Result().Cookies() { + if candidate.Name == "__Host-observatory_session" { + cookie = candidate + break + } + } + if cookie == nil { + t.Fatalf("login cookies=%+v", login.Result().Cookies()) + } + blocked := perform(handler, http.MethodGet, "https://observatory.example/app/", nil, []*http.Cookie{cookie}) + if blocked.Code != http.StatusSeeOther || blocked.Header().Get("Location") != "/account/password/" { + t.Fatalf("blocked app status=%d location=%q", blocked.Code, blocked.Header().Get("Location")) + } + blockedWrite := perform(handler, http.MethodPost, "https://observatory.example/api/v1/query", strings.NewReader(`{}`), []*http.Cookie{cookie}, headers) + if blockedWrite.Code != http.StatusForbidden { + t.Fatalf("blocked write status=%d body=%s", blockedWrite.Code, blockedWrite.Body.String()) + } + page := perform(handler, http.MethodGet, "https://observatory.example/account/password/", nil, []*http.Cookie{cookie}) + if page.Code != http.StatusOK || !strings.Contains(page.Body.String(), "Choose your password") || strings.Contains(page.Body.String(), temporary) { + t.Fatalf("password page status=%d body=%s", page.Code, page.Body.String()) + } + head := perform(handler, http.MethodHead, "https://observatory.example/account/password/", nil, []*http.Cookie{cookie}) + if head.Code != http.StatusOK || head.Body.Len() != 0 || head.Header().Get("Content-Length") != page.Header().Get("Content-Length") { + t.Fatalf("password HEAD status=%d length=%q body=%d", head.Code, head.Header().Get("Content-Length"), head.Body.Len()) + } + csrf, err := authhttp.CSRFToken(cookie.Value, "account:password:change") + if err != nil { + t.Fatal(err) + } + crossSiteHeaders := http.Header{"Origin": []string{"https://attacker.example"}, "Sec-Fetch-Site": []string{"cross-site"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + crossSiteForm := url.Values{"csrf_token": []string{csrf}, "current_password": []string{temporary}, "new_password": []string{replacement}, "confirm_password": []string{replacement}}.Encode() + crossSite := perform(handler, http.MethodPost, "https://observatory.example/account/password/", strings.NewReader(crossSiteForm), []*http.Cookie{cookie}, crossSiteHeaders) + if crossSite.Code != http.StatusForbidden || !strings.Contains(crossSite.Body.String(), passwordFormFailure) || strings.Contains(crossSite.Body.String(), temporary) || strings.Contains(crossSite.Body.String(), replacement) || strings.Contains(crossSite.Header().Get("Content-Type"), "application/json") { + t.Fatalf("cross-site password status=%d body=%s", crossSite.Code, crossSite.Body.String()) + } + invalidTokenForm := url.Values{"csrf_token": []string{"invalid"}, "current_password": []string{temporary}, "new_password": []string{replacement}, "confirm_password": []string{replacement}}.Encode() + invalidToken := perform(handler, http.MethodPost, "https://observatory.example/account/password/", strings.NewReader(invalidTokenForm), []*http.Cookie{cookie}, headers) + if invalidToken.Code != http.StatusForbidden || !strings.Contains(invalidToken.Body.String(), passwordFormFailure) || strings.Contains(invalidToken.Body.String(), temporary) || strings.Contains(invalidToken.Body.String(), replacement) || strings.Contains(invalidToken.Header().Get("Content-Type"), "application/json") { + t.Fatalf("invalid-token password status=%d body=%s", invalidToken.Code, invalidToken.Body.String()) + } + mismatch := url.Values{"csrf_token": []string{csrf}, "current_password": []string{temporary}, "new_password": []string{replacement}, "confirm_password": []string{"different password value"}}.Encode() + rejected := perform(handler, http.MethodPost, "https://observatory.example/account/password/", strings.NewReader(mismatch), []*http.Cookie{cookie}, headers) + if rejected.Code != http.StatusUnprocessableEntity || !strings.Contains(rejected.Body.String(), passwordMatchFailure) || strings.Contains(rejected.Body.String(), temporary) || strings.Contains(rejected.Body.String(), replacement) { + t.Fatalf("mismatch status=%d body=%s", rejected.Code, rejected.Body.String()) + } + change := url.Values{"csrf_token": []string{csrf}, "current_password": []string{temporary}, "new_password": []string{replacement}, "confirm_password": []string{replacement}}.Encode() + privacyHeaders := http.Header{"Origin": []string{"null"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + changed := perform(handler, http.MethodPost, "https://observatory.example/account/password/", strings.NewReader(change), []*http.Cookie{cookie}, privacyHeaders) + if changed.Code != http.StatusSeeOther || changed.Header().Get("Location") != "/login/?password=changed" { + t.Fatalf("change status=%d location=%q body=%s", changed.Code, changed.Header().Get("Location"), changed.Body.String()) + } + oldSession := perform(handler, http.MethodGet, "https://observatory.example/app/", nil, []*http.Cookie{cookie}) + if oldSession.Code != http.StatusSeeOther || oldSession.Header().Get("Location") != "/login/" { + t.Fatalf("old session status=%d location=%q", oldSession.Code, oldSession.Header().Get("Location")) + } + if _, _, err = identities.Auth.Authenticate(t.Context(), "operator", temporary, time.Hour); err == nil { + t.Fatal("temporary password remained valid") + } + _, principal, err := identities.Auth.Authenticate(t.Context(), "operator", replacement, time.Hour) + if err != nil || principal.User.PasswordChangeRequired { + t.Fatalf("replacement principal=%+v err=%v", principal, err) + } + + newLogin := loginHTMLResponse(t, handler, "operator", replacement) + if newLogin.Code != http.StatusSeeOther || newLogin.Header().Get("Location") != "/app/" { + t.Fatalf("new login status=%d location=%q", newLogin.Code, newLogin.Header().Get("Location")) + } +} + +func TestAPITemporaryOperatorReceivesScopedRotationToken(t *testing.T) { + server, store, identities, _ := newRotationTestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + loginHeaders := http.Header{"Origin": []string{"https://observatory.example"}, "Content-Type": []string{"application/json"}} + login := perform(handler, http.MethodPost, "https://observatory.example/api/v1/session", strings.NewReader(`{"identifier":"operator","password":"temporary correct horse battery staple"}`), nil, loginHeaders) + var session struct { + PasswordChangeRequired bool `json:"password_change_required"` + PasswordChangeCSRF string `json:"password_change_csrf"` + } + if login.Code != http.StatusOK || json.Unmarshal(login.Body.Bytes(), &session) != nil || !session.PasswordChangeRequired || session.PasswordChangeCSRF == "" { + t.Fatalf("login status=%d session=%+v body=%s", login.Code, session, login.Body.String()) + } + cookies := login.Result().Cookies() + changeHeaders := http.Header{"Origin": []string{"https://observatory.example"}, "Content-Type": []string{"application/json"}, "X-CSRF-Token": []string{session.PasswordChangeCSRF}} + changed := perform(handler, http.MethodPost, "https://observatory.example/api/v1/account/password", strings.NewReader(`{"current_password":"temporary correct horse battery staple","new_password":"API replacement password value"}`), cookies, changeHeaders) + if changed.Code != http.StatusNoContent || changed.Body.Len() != 0 { + t.Fatalf("change status=%d body=%s", changed.Code, changed.Body.String()) + } + _, principal, err := identities.Auth.Authenticate(t.Context(), "operator", "API replacement password value", time.Hour) + if err != nil || principal.User.PasswordChangeRequired { + t.Fatalf("principal=%+v err=%v", principal, err) + } +} + +func TestLiveRefreshStreamIsAuthorizedAndCarriesNoTelemetry(t *testing.T) { + server, store, identities, bootstrap := newUITestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + unauthorized := perform(handler, http.MethodGet, "https://observatory.example/app/events?organization="+url.QueryEscape(bootstrap.Organization.ID), nil, nil) + if unauthorized.Code != http.StatusUnauthorized { + t.Fatalf("unauthorized stream status=%d", unauthorized.Code) + } + cookie := loginHTML(t, handler) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + request := httptest.NewRequest(http.MethodGet, "https://observatory.example/app/events?organization="+url.QueryEscape(bootstrap.Organization.ID), nil).WithContext(ctx) + request.AddCookie(cookie) + stream := newStreamRecorder() + done := make(chan struct{}) + go func() { + handler.ServeHTTP(stream, request) + close(done) + }() + waitFlush := func() { + t.Helper() + select { + case <-stream.flushed: + case <-ctx.Done(): + t.Fatal("stream did not flush before timeout") + } + } + waitFlush() + if stream.statusCode() != http.StatusOK || stream.Header().Get("Content-Type") != "text/event-stream; charset=utf-8" || stream.Header().Get("X-Accel-Buffering") != "no" || stream.bodyString() != "event: ready\ndata: {}\n\n" { + t.Fatalf("stream status=%d headers=%v body=%q", stream.statusCode(), stream.Header(), stream.bodyString()) + } + server.refresh.publish(bootstrap.Organization.ID) + waitFlush() + streamBody := stream.bodyString() + if streamBody != "event: ready\ndata: {}\n\nevent: refresh\ndata: {}\n\n" || strings.Contains(streamBody, bootstrap.Organization.ID) || strings.Contains(streamBody, "service") { + t.Fatalf("stream body=%q", streamBody) + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("stream did not stop after cancellation") + } +} + +func TestDashboardManagementIsScopedCSRFProtectedAndExportable(t *testing.T) { + server, store, identities, bootstrap := newUITestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + cookie := loginHTML(t, handler) + csrf, err := authhttp.CSRFToken(cookie.Value, "dashboards:manage") + if err != nil { + t.Fatal(err) + } + headers := http.Header{"Origin": []string{"https://observatory.example"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + + invalid := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{"invalid"}, + "name": []string{"Recent errors"}, "description": []string{"A bounded recent error view."}, + "query": []string{"logs | where status >= 500 | window 1h | limit 50"}, + }.Encode() + denied := perform(handler, http.MethodPost, "https://observatory.example/app/queries/", strings.NewReader(invalid), []*http.Cookie{cookie}, headers) + if denied.Code != http.StatusForbidden { + t.Fatalf("invalid CSRF status=%d", denied.Code) + } + + queryForm := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "name": []string{"Recent errors"}, "description": []string{"A bounded recent error view."}, + "query": []string{"logs | where status >= 500 | window 1h | limit 50"}, + }.Encode() + privacyHeaders := http.Header{"Origin": []string{"null"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + createdQuery := perform(handler, http.MethodPost, "https://observatory.example/app/queries/", strings.NewReader(queryForm), []*http.Cookie{cookie}, privacyHeaders) + if createdQuery.Code != http.StatusSeeOther || !strings.HasPrefix(createdQuery.Header().Get("Location"), "/app/?organization=") { + t.Fatalf("query status=%d location=%q body=%s", createdQuery.Code, createdQuery.Header().Get("Location"), createdQuery.Body.String()) + } + queries, err := store.SavedQueries(context.Background(), bootstrap.Organization.ID) + if err != nil || len(queries) != 1 { + t.Fatalf("queries=%+v err=%v", queries, err) + } + mismatchedDashboard := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "slug": []string{"invalid-stat"}, "name": []string{"Invalid stat"}, + "description": []string{"A non-summary query cannot become a statistic."}, "panel_title": []string{"Invalid"}, + "saved_query_id": []string{queries[0].ID}, "visualization": []string{"stat"}, + }.Encode() + mismatched := perform(handler, http.MethodPost, "https://observatory.example/app/dashboards/", strings.NewReader(mismatchedDashboard), []*http.Cookie{cookie}, headers) + if mismatched.Code != http.StatusUnprocessableEntity { + t.Fatalf("mismatched presentation status=%d body=%s", mismatched.Code, mismatched.Body.String()) + } + + dashboardForm := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "slug": []string{"recent-errors"}, "name": []string{"Recent errors"}, + "description": []string{"An accessible bounded error dashboard."}, "panel_title": []string{"Errors"}, + "saved_query_id": []string{queries[0].ID}, "visualization": []string{"table"}, + }.Encode() + createdDashboard := perform(handler, http.MethodPost, "https://observatory.example/app/dashboards/", strings.NewReader(dashboardForm), []*http.Cookie{cookie}, headers) + if createdDashboard.Code != http.StatusSeeOther || !strings.HasPrefix(createdDashboard.Header().Get("Location"), "/app/dashboards/recent-errors/") { + t.Fatalf("dashboard status=%d location=%q body=%s", createdDashboard.Code, createdDashboard.Header().Get("Location"), createdDashboard.Body.String()) + } + + target := "https://observatory.example/app/dashboards/recent-errors/?organization=" + url.QueryEscape(bootstrap.Organization.ID) + dashboard := perform(handler, http.MethodGet, target, nil, []*http.Cookie{cookie}) + if dashboard.Code != http.StatusOK || !strings.Contains(dashboard.Body.String(), "An accessible bounded error dashboard.") || !strings.Contains(dashboard.Body.String(), "Recent errors") || !strings.Contains(dashboard.Body.String(), ">Errors") { + t.Fatalf("dashboard status=%d body=%s", dashboard.Code, dashboard.Body.String()) + } + dashboardHead := perform(handler, http.MethodHead, target, nil, []*http.Cookie{cookie}) + if dashboardHead.Code != http.StatusOK || dashboardHead.Body.Len() != 0 || dashboardHead.Header().Get("Content-Length") != dashboard.Header().Get("Content-Length") { + t.Fatalf("dashboard HEAD status=%d length=%q", dashboardHead.Code, dashboardHead.Header().Get("Content-Length")) + } + storedDashboard, err := store.Dashboard(context.Background(), bootstrap.Organization.ID, "recent-errors") + if err != nil || storedDashboard.Revision != 1 || !strings.Contains(dashboard.Body.String(), "Update dashboard details") || !strings.Contains(dashboard.Body.String(), `name="expected_revision" value="1"`) { + t.Fatalf("stored dashboard=%+v err=%v body=%s", storedDashboard, err, dashboard.Body.String()) + } + revisionForm := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "dashboard_id": []string{storedDashboard.ID}, "expected_revision": []string{strconv.Itoa(storedDashboard.Revision)}, + "slug": []string{storedDashboard.Slug}, "name": []string{"Current errors"}, + "description": []string{"A revision-safe bounded error dashboard."}, + }.Encode() + revised := perform(handler, http.MethodPost, target, strings.NewReader(revisionForm), []*http.Cookie{cookie}, headers) + if revised.Code != http.StatusSeeOther { + t.Fatalf("revision status=%d body=%s", revised.Code, revised.Body.String()) + } + stale := perform(handler, http.MethodPost, target, strings.NewReader(revisionForm), []*http.Cookie{cookie}, headers) + if stale.Code != http.StatusConflict { + t.Fatalf("stale revision status=%d body=%s", stale.Code, stale.Body.String()) + } + storedDashboard, err = store.Dashboard(context.Background(), bootstrap.Organization.ID, "recent-errors") + if err != nil || storedDashboard.Revision != 2 || storedDashboard.Name != "Current errors" { + t.Fatalf("revised dashboard=%+v err=%v", storedDashboard, err) + } + addPanelForm := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "dashboard_id": []string{storedDashboard.ID}, "expected_revision": []string{strconv.Itoa(storedDashboard.Revision)}, + "panel_title": []string{"Recent failures"}, "saved_query_id": []string{queries[0].ID}, "visualization": []string{"table"}, + }.Encode() + addTarget := "https://observatory.example/app/dashboards/recent-errors/panels/?organization=" + url.QueryEscape(bootstrap.Organization.ID) + added := perform(handler, http.MethodPost, addTarget, strings.NewReader(addPanelForm), []*http.Cookie{cookie}, headers) + if added.Code != http.StatusSeeOther { + t.Fatalf("add panel status=%d body=%s", added.Code, added.Body.String()) + } + storedDashboard, err = store.Dashboard(context.Background(), bootstrap.Organization.ID, "recent-errors") + if err != nil || storedDashboard.Revision != 3 || len(storedDashboard.Panels) != 2 { + t.Fatalf("dashboard after add=%+v err=%v", storedDashboard, err) + } + addedPanel := storedDashboard.Panels[1] + updatePanelForm := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "dashboard_id": []string{storedDashboard.ID}, "expected_revision": []string{strconv.Itoa(storedDashboard.Revision)}, + "panel_title": []string{"Renamed failures"}, "saved_query_id": []string{queries[0].ID}, "visualization": []string{"table"}, + }.Encode() + updatePanelTarget := "https://observatory.example/app/dashboards/recent-errors/panels/" + url.PathEscape(addedPanel.ID) + "/?organization=" + url.QueryEscape(bootstrap.Organization.ID) + updatedPanel := perform(handler, http.MethodPost, updatePanelTarget, strings.NewReader(updatePanelForm), []*http.Cookie{cookie}, headers) + if updatedPanel.Code != http.StatusSeeOther { + t.Fatalf("update panel status=%d body=%s", updatedPanel.Code, updatedPanel.Body.String()) + } + storedDashboard, err = store.Dashboard(context.Background(), bootstrap.Organization.ID, "recent-errors") + if err != nil || storedDashboard.Revision != 4 || storedDashboard.Panels[1].Title != "Renamed failures" { + t.Fatalf("dashboard after panel update=%+v err=%v", storedDashboard, err) + } + mismatchedRevision := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "dashboard_id": []string{storedDashboard.ID}, "expected_revision": []string{strconv.Itoa(storedDashboard.Revision)}, + "panel_title": []string{"Invalid chart"}, "saved_query_id": []string{queries[0].ID}, "visualization": []string{"timeseries"}, + }.Encode() + mismatchedPanel := perform(handler, http.MethodPost, updatePanelTarget, strings.NewReader(mismatchedRevision), []*http.Cookie{cookie}, headers) + if mismatchedPanel.Code != http.StatusUnprocessableEntity { + t.Fatalf("mismatched panel status=%d body=%s", mismatchedPanel.Code, mismatchedPanel.Body.String()) + } + removePanelForm := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "dashboard_id": []string{storedDashboard.ID}, "expected_revision": []string{strconv.Itoa(storedDashboard.Revision)}, + }.Encode() + removeTarget := "https://observatory.example/app/dashboards/recent-errors/panels/" + url.PathEscape(addedPanel.ID) + "/remove/?organization=" + url.QueryEscape(bootstrap.Organization.ID) + removed := perform(handler, http.MethodPost, removeTarget, strings.NewReader(removePanelForm), []*http.Cookie{cookie}, headers) + if removed.Code != http.StatusSeeOther { + t.Fatalf("remove panel status=%d body=%s", removed.Code, removed.Body.String()) + } + storedDashboard, err = store.Dashboard(context.Background(), bootstrap.Organization.ID, "recent-errors") + if err != nil || storedDashboard.Revision != 5 || len(storedDashboard.Panels) != 1 { + t.Fatalf("dashboard after remove=%+v err=%v", storedDashboard, err) + } + + exportTarget := "https://observatory.example/app/dashboards/recent-errors/export.json?organization=" + url.QueryEscape(bootstrap.Organization.ID) + exported := perform(handler, http.MethodGet, exportTarget, nil, []*http.Cookie{cookie}) + if exported.Code != http.StatusOK || exported.Header().Get("Content-Type") != "application/json" || !strings.Contains(exported.Body.String(), `"version": 1`) || strings.Contains(exported.Body.String(), bootstrap.Organization.ID) || strings.Contains(exported.Body.String(), bootstrap.User.ID) { + t.Fatalf("export status=%d headers=%v body=%s", exported.Code, exported.Header(), exported.Body.String()) + } + exportedHead := perform(handler, http.MethodHead, exportTarget, nil, []*http.Cookie{cookie}) + if exportedHead.Code != http.StatusOK || exportedHead.Body.Len() != 0 || exportedHead.Header().Get("Content-Length") != exported.Header().Get("Content-Length") { + t.Fatalf("export HEAD status=%d length=%q", exportedHead.Code, exportedHead.Header().Get("Content-Length")) + } + unauthorized := perform(handler, http.MethodGet, exportTarget, nil, nil) + if unauthorized.Code != http.StatusUnauthorized || unauthorized.Body.String() == exported.Body.String() { + t.Fatalf("unauthorized export status=%d", unauthorized.Code) + } +} + +func TestIncidentRulesEvaluationInboxAndResponseAreScoped(t *testing.T) { + server, store, identities, bootstrap := newUITestServer(t) + defer store.Close() + defer identities.Close() + pushes := &recordingPushDispatcher{} + server.options.PushDispatcher = pushes + server.options.PushPublicKey = base64.RawURLEncoding.EncodeToString(append([]byte{4}, make([]byte, 64)...)) + handler := server.Handler() + cookie := loginHTML(t, handler) + now := server.now() + saved, err := store.SaveQuery(context.Background(), storage.SavedQueryInput{ + OrganizationID: bootstrap.Organization.ID, ActorUserID: bootstrap.User.ID, MaxRows: 100, + Name: "Recent failures", Description: "Recent HTTP failures.", Query: "logs | where status >= 500 | window 1h | limit 50", + }, now) + if err != nil { + t.Fatal(err) + } + token, err := store.CreateSource(context.Background(), "incident-source", model.Scope{OrganizationID: bootstrap.Organization.ID, ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "incident-source", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request", Attributes: map[string]string{"http.status_code": "503", "http.route": "/failed"}}}} + if _, err = store.Ingest(context.Background(), token, batch, now); err != nil { + t.Fatal(err) + } + if err = store.Recover(context.Background()); err != nil { + t.Fatal(err) + } + csrf, err := authhttp.CSRFToken(cookie.Value, "incidents:manage") + if err != nil { + t.Fatal(err) + } + headers := http.Header{"Origin": []string{"https://observatory.example"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + form := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "name": []string{"HTTP failures"}, "description": []string{"Open when a bounded saved query finds a failure."}, + "saved_query_id": []string{saved.ID}, "severity": []string{"critical"}, "minimum_matches": []string{"1"}, + "required_consecutive": []string{"1"}, "evaluation_interval": []string{"15s"}, + } + invalid := cloneValues(form) + invalid.Set("csrf_token", "invalid") + denied := perform(handler, http.MethodPost, "https://observatory.example/app/alert-rules/", strings.NewReader(invalid.Encode()), []*http.Cookie{cookie}, headers) + if denied.Code != http.StatusForbidden { + t.Fatalf("invalid CSRF status=%d body=%s", denied.Code, denied.Body.String()) + } + privacyHeaders := http.Header{"Origin": []string{"null"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + created := perform(handler, http.MethodPost, "https://observatory.example/app/alert-rules/", strings.NewReader(form.Encode()), []*http.Cookie{cookie}, privacyHeaders) + if created.Code != http.StatusSeeOther || !strings.HasPrefix(created.Header().Get("Location"), "/app/incidents/?organization=") { + t.Fatalf("create status=%d location=%q body=%s", created.Code, created.Header().Get("Location"), created.Body.String()) + } + + updates, remove, err := server.refresh.subscribe(bootstrap.Organization.ID) + if err != nil { + t.Fatal(err) + } + defer remove() + if evaluated, evaluationErr := server.EvaluateAlerts(context.Background()); evaluationErr != nil || evaluated != 1 { + t.Fatalf("evaluated=%d err=%v", evaluated, evaluationErr) + } + if len(pushes.organizations) != 1 || pushes.organizations[0] != bootstrap.Organization.ID { + t.Fatalf("push organizations=%v", pushes.organizations) + } + select { + case <-updates: + default: + t.Fatal("incident change did not publish a generic refresh") + } + incidents, err := store.Incidents(context.Background(), bootstrap.Organization.ID, false, 10) + if err != nil || len(incidents) != 1 || incidents[0].State != "firing" { + t.Fatalf("incidents=%+v err=%v", incidents, err) + } + + path := "https://observatory.example/app/incidents/?organization=" + url.QueryEscape(bootstrap.Organization.ID) + inbox := perform(handler, http.MethodGet, path, nil, []*http.Cookie{cookie}) + if inbox.Code != http.StatusOK || !strings.Contains(inbox.Body.String(), "What needs attention?") || !strings.Contains(inbox.Body.String(), "HTTP failures") || !strings.Contains(inbox.Body.String(), "critical · firing") || !strings.Contains(inbox.Body.String(), "data-cache-inbox") || !strings.Contains(inbox.Body.String(), "data-push-toggle") || !strings.Contains(inbox.Body.String(), `data-open-incident-count="1"`) || strings.Contains(inbox.Body.String(), "/failed") { + t.Fatalf("inbox status=%d body=%s", inbox.Code, inbox.Body.String()) + } + pushCSRF, err := authhttp.CSRFToken(cookie.Value, "push:manage") + if err != nil { + t.Fatal(err) + } + clientKey, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + authSecret := make([]byte, 16) + if _, err = rand.Read(authSecret); err != nil { + t.Fatal(err) + } + pushBody, _ := json.Marshal(map[string]any{"organization_id": bootstrap.Organization.ID, "endpoint": "https://push.example.test/send/browser", "keys": map[string]string{"p256dh": base64.RawURLEncoding.EncodeToString(clientKey.PublicKey().Bytes()), "auth": base64.RawURLEncoding.EncodeToString(authSecret)}}) + pushHeaders := make(http.Header) + pushHeaders.Set("Origin", "https://observatory.example") + pushHeaders.Set("Content-Type", "application/json") + pushHeaders.Set("X-CSRF-Token", pushCSRF) + invalidPushHeaders := pushHeaders.Clone() + invalidPushHeaders.Set("X-CSRF-Token", "invalid") + invalidPush := perform(handler, http.MethodPost, "https://observatory.example/api/v1/push/subscription", bytes.NewReader(pushBody), []*http.Cookie{cookie}, invalidPushHeaders) + if invalidPush.Code != http.StatusForbidden { + t.Fatalf("invalid push CSRF status=%d body=%s", invalidPush.Code, invalidPush.Body.String()) + } + crossOriginPushHeaders := pushHeaders.Clone() + crossOriginPushHeaders.Set("Origin", "https://attacker.example") + crossOriginPush := perform(handler, http.MethodPost, "https://observatory.example/api/v1/push/subscription", bytes.NewReader(pushBody), []*http.Cookie{cookie}, crossOriginPushHeaders) + if crossOriginPush.Code != http.StatusForbidden { + t.Fatalf("cross-origin push status=%d body=%s", crossOriginPush.Code, crossOriginPush.Body.String()) + } + registered := perform(handler, http.MethodPost, "https://observatory.example/api/v1/push/subscription", bytes.NewReader(pushBody), []*http.Cookie{cookie}, pushHeaders) + if registered.Code != http.StatusCreated || !strings.Contains(registered.Body.String(), `"id":"push_`) { + t.Fatalf("push registration status=%d body=%s", registered.Code, registered.Body.String()) + } + if subscriptions, listErr := store.PushSubscriptions(context.Background(), bootstrap.Organization.ID); listErr != nil || len(subscriptions) != 1 || subscriptions[0].UserID != bootstrap.User.ID { + t.Fatalf("push subscriptions=%+v err=%v", subscriptions, listErr) + } + statusBody, _ := json.Marshal(map[string]any{"organization_id": bootstrap.Organization.ID, "endpoint": "https://push.example.test/send/browser", "keys": map[string]string{"p256dh": "", "auth": ""}}) + pushStatus := perform(handler, http.MethodPost, "https://observatory.example/api/v1/push/subscription/status", bytes.NewReader(statusBody), []*http.Cookie{cookie}, pushHeaders) + if pushStatus.Code != http.StatusOK || !strings.Contains(pushStatus.Body.String(), `"subscribed":true`) { + t.Fatalf("push status=%d body=%s", pushStatus.Code, pushStatus.Body.String()) + } + privateEndpointBody := bytes.Replace(pushBody, []byte("https://push.example.test/send/browser"), []byte("https://127.0.0.1/send/browser"), 1) + rejected := perform(handler, http.MethodPost, "https://observatory.example/api/v1/push/subscription", bytes.NewReader(privateEndpointBody), []*http.Cookie{cookie}, pushHeaders) + if rejected.Code != http.StatusUnprocessableEntity || strings.Contains(rejected.Body.String(), "127.0.0.1") { + t.Fatalf("private endpoint status=%d body=%s", rejected.Code, rejected.Body.String()) + } + invalidCurveBody, _ := json.Marshal(map[string]any{"organization_id": bootstrap.Organization.ID, "endpoint": "https://push.example.test/send/invalid-curve", "keys": map[string]string{"p256dh": base64.RawURLEncoding.EncodeToString(append([]byte{4}, make([]byte, 64)...)), "auth": base64.RawURLEncoding.EncodeToString(authSecret)}}) + invalidCurve := perform(handler, http.MethodPost, "https://observatory.example/api/v1/push/subscription", bytes.NewReader(invalidCurveBody), []*http.Cookie{cookie}, pushHeaders) + if invalidCurve.Code != http.StatusUnprocessableEntity { + t.Fatalf("invalid curve status=%d body=%s", invalidCurve.Code, invalidCurve.Body.String()) + } + deleteBody, _ := json.Marshal(map[string]any{"organization_id": bootstrap.Organization.ID, "endpoint": "https://push.example.test/send/browser", "keys": map[string]string{"p256dh": "", "auth": ""}}) + deleted := perform(handler, http.MethodDelete, "https://observatory.example/api/v1/push/subscription", bytes.NewReader(deleteBody), []*http.Cookie{cookie}, pushHeaders) + if deleted.Code != http.StatusOK || !strings.Contains(deleted.Body.String(), `"remaining":false`) { + t.Fatalf("push deletion status=%d body=%s", deleted.Code, deleted.Body.String()) + } + offlineInbox := perform(handler, http.MethodGet, "https://observatory.example/app/incidents/offline/?organization="+url.QueryEscape(bootstrap.Organization.ID), nil, []*http.Cookie{cookie}) + if offlineInbox.Code != http.StatusOK || !strings.Contains(offlineInbox.Body.String(), "Saved incident inbox") || !strings.Contains(offlineInbox.Body.String(), "HTTP failures") { + t.Fatalf("offline inbox status=%d body=%s", offlineInbox.Code, offlineInbox.Body.String()) + } + for _, forbidden := range []string{incidents[0].ID, saved.Query, bootstrap.User.ID, "csrf_token", "/failed", "Acknowledge", "Resolve"} { + if strings.Contains(offlineInbox.Body.String(), forbidden) { + t.Fatalf("offline inbox exposed %q: %s", forbidden, offlineInbox.Body.String()) + } + } + unauthenticatedOffline := perform(handler, http.MethodGet, "https://observatory.example/app/incidents/offline/?organization="+url.QueryEscape(bootstrap.Organization.ID), nil, nil) + if unauthenticatedOffline.Code != http.StatusSeeOther || unauthenticatedOffline.Header().Get("Location") != "/login/" { + t.Fatalf("unauthenticated offline status=%d", unauthenticatedOffline.Code) + } + inboxHead := perform(handler, http.MethodHead, path, nil, []*http.Cookie{cookie}) + if inboxHead.Code != http.StatusOK || inboxHead.Body.Len() != 0 || inboxHead.Header().Get("Content-Length") != inbox.Header().Get("Content-Length") { + t.Fatalf("inbox HEAD status=%d length=%q body=%d", inboxHead.Code, inboxHead.Header().Get("Content-Length"), inboxHead.Body.Len()) + } + missingOrganization := perform(handler, http.MethodGet, "https://observatory.example/app/incidents/", nil, []*http.Cookie{cookie}) + if missingOrganization.Code != http.StatusBadRequest { + t.Fatalf("missing organization status=%d", missingOrganization.Code) + } + + action := url.Values{"organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, "action": []string{"acknowledge"}, "silence_duration": []string{""}}.Encode() + acknowledged := perform(handler, http.MethodPost, "https://observatory.example/app/incidents/"+incidents[0].ID+"/", strings.NewReader(action), []*http.Cookie{cookie}, headers) + if acknowledged.Code != http.StatusSeeOther { + t.Fatalf("acknowledge status=%d body=%s", acknowledged.Code, acknowledged.Body.String()) + } + current, err := store.Incidents(context.Background(), bootstrap.Organization.ID, false, 10) + if err != nil || len(current) != 1 || current[0].State != "acknowledged" || current[0].AcknowledgedBy != bootstrap.User.ID { + t.Fatalf("current=%+v err=%v", current, err) + } +} + +type recordingPushDispatcher struct{ organizations []string } + +func (dispatcher *recordingPushDispatcher) Enqueue(organizationID string) bool { + dispatcher.organizations = append(dispatcher.organizations, organizationID) + return true +} + +func cloneValues(input url.Values) url.Values { + result := make(url.Values, len(input)) + for key, values := range input { + result[key] = append([]string(nil), values...) + } + return result +} + +func TestAssistedQueryBuilderCreatesTypedTimeSeriesWithTableAlternative(t *testing.T) { + server, store, identities, bootstrap := newUITestServer(t) + defer store.Close() + defer identities.Close() + handler := server.Handler() + cookie := loginHTML(t, handler) + csrf, err := authhttp.CSRFToken(cookie.Value, "dashboards:manage") + if err != nil { + t.Fatal(err) + } + now := server.now() + token, err := store.CreateSource(context.Background(), "builder-source", model.Scope{OrganizationID: bootstrap.Organization.ID, ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "builder-source", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + {Timestamp: now.Add(-6 * time.Minute), Name: "application.http.request", Attributes: map[string]string{"http.status_code": "503", "http.route": "/failed"}}, + {Timestamp: now.Add(-1 * time.Minute), Name: "application.http.request", Attributes: map[string]string{"http.status_code": "500", "http.route": "/failed"}}, + }} + if _, err = store.Ingest(context.Background(), token, batch, now); err != nil { + t.Fatal(err) + } + if err = store.Recover(context.Background()); err != nil { + t.Fatal(err) + } + headers := http.Header{"Origin": []string{"https://observatory.example"}, "Content-Type": []string{"application/x-www-form-urlencoded"}} + builderForm := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "name": []string{"Errors over time"}, "description": []string{"Five-minute error counts."}, + "signal": []string{"logs"}, "filter_field": []string{"status"}, "filter_operator": []string{">="}, "filter_value": []string{"500"}, + "window": []string{"1h"}, "aggregate": []string{"count"}, "aggregate_field": []string{""}, "group_by": []string{"route"}, "bucket": []string{"5m"}, "limit": []string{"50"}, + }.Encode() + createdQuery := perform(handler, http.MethodPost, "https://observatory.example/app/queries/builder/", strings.NewReader(builderForm), []*http.Cookie{cookie}, headers) + if createdQuery.Code != http.StatusSeeOther { + t.Fatalf("builder status=%d body=%s", createdQuery.Code, createdQuery.Body.String()) + } + queries, err := store.SavedQueries(context.Background(), bootstrap.Organization.ID) + if err != nil || len(queries) != 1 { + t.Fatalf("queries=%+v err=%v", queries, err) + } + expectedText := `logs | where status >= "500" | window 1h | summarize count() by route, window(5m) | limit 50` + if queries[0].Query != expectedText || queries[0].AST.Signal != model.SignalLogs || len(queries[0].AST.Filters) != 1 || queries[0].AST.Filters[0].Value != "500" || queries[0].AST.Summary == nil || queries[0].AST.Bucket != 5*time.Minute { + t.Fatalf("saved query=%+v", queries[0]) + } + dashboardForm := url.Values{ + "organization_id": []string{bootstrap.Organization.ID}, "csrf_token": []string{csrf}, + "slug": []string{"error-rate"}, "name": []string{"Error rate"}, "description": []string{"A bounded error trend."}, + "panel_title": []string{"Errors by route"}, "saved_query_id": []string{queries[0].ID}, "visualization": []string{"timeseries"}, + }.Encode() + createdDashboard := perform(handler, http.MethodPost, "https://observatory.example/app/dashboards/", strings.NewReader(dashboardForm), []*http.Cookie{cookie}, headers) + if createdDashboard.Code != http.StatusSeeOther { + t.Fatalf("dashboard status=%d body=%s", createdDashboard.Code, createdDashboard.Body.String()) + } + target := "https://observatory.example/app/dashboards/error-rate/?organization=" + url.QueryEscape(bootstrap.Organization.ID) + dashboard := perform(handler, http.MethodGet, target, nil, []*http.Cookie{cookie}) + body := dashboard.Body.String() + if dashboard.Code != http.StatusOK || !strings.Contains(body, "Errors by route visual summary") || strings.Count(body, "") || !strings.Contains(body, "Errors by route") { + t.Fatalf("dashboard status=%d body=%s", dashboard.Code, body) + } +} + +func TestAssistedQueryBuilderEscapesStageSeparatorsAndRejectsInvalidCombinations(t *testing.T) { + values := url.Values{ + "signal": []string{"logs"}, "filter_field": []string{"name"}, "filter_operator": []string{"=="}, "filter_value": []string{"worker | limit 250"}, + "window": []string{"1h"}, "aggregate": []string{"none"}, "aggregate_field": []string{""}, "group_by": []string{""}, "bucket": []string{""}, "limit": []string{"50"}, + } + text, err := buildAssistedQuery(values, 1000) + if err != nil || strings.Contains(text, `"worker | limit 250"`) || !strings.Contains(text, `\u007c`) { + t.Fatalf("text=%q err=%v", text, err) + } + ast, err := query.Parse(text, 1000) + if err != nil || len(ast.Filters) != 1 || ast.Filters[0].Value != "worker | limit 250" || ast.Limit != 50 { + t.Fatalf("AST=%+v err=%v", ast, err) + } + values.Set("bucket", "5m") + if _, err = buildAssistedQuery(values, 1000); err == nil { + t.Fatal("time bucket without an aggregate was accepted") + } +} + +func TestResultChartIsBoundedAndFailsClosedForNegativeValues(t *testing.T) { + result := query.Result{Columns: []query.Column{{Field: "window_start", Type: schema.TypeTime}, {Field: "count", Type: schema.TypeInteger}}} + for index := range 60 { + label := fmt.Sprintf("2026-08-17T07:%02d:00Z", index) + value := strconv.Itoa(index) + result.Rows = append(result.Rows, query.Row{Values: []*string{&label, &value}}) + } + chart := resultChart("Requests", result) + if len(chart.Points) != 48 || chart.Points[47].Maximum != "47" || chart.Points[47].Value != "47" { + t.Fatalf("chart=%+v", chart) + } + negative := "-1" + result.Rows[0].Values[1] = &negative + if chart = resultChart("Requests", result); len(chart.Points) != 0 { + t.Fatalf("negative chart=%+v", chart) + } +} + +type streamRecorder struct { + mu sync.Mutex + header http.Header + status int + body bytes.Buffer + flushed chan struct{} +} + +func newStreamRecorder() *streamRecorder { + return &streamRecorder{header: make(http.Header), flushed: make(chan struct{}, 4)} +} + +func (recorder *streamRecorder) Header() http.Header { return recorder.header } + +func (recorder *streamRecorder) WriteHeader(status int) { + recorder.mu.Lock() + defer recorder.mu.Unlock() + if recorder.status == 0 { + recorder.status = status + } +} + +func (recorder *streamRecorder) Write(body []byte) (int, error) { + recorder.mu.Lock() + defer recorder.mu.Unlock() + if recorder.status == 0 { + recorder.status = http.StatusOK + } + return recorder.body.Write(body) +} + +func (recorder *streamRecorder) Flush() { + select { + case recorder.flushed <- struct{}{}: + default: + } +} + +func (recorder *streamRecorder) statusCode() int { + recorder.mu.Lock() + defer recorder.mu.Unlock() + return recorder.status +} + +func (recorder *streamRecorder) bodyString() string { + recorder.mu.Lock() + defer recorder.mu.Unlock() + return recorder.body.String() +} + +func newUITestServer(t *testing.T) (*Server, *storage.Store, *identity.Services, identity.BootstrapResult) { + t.Helper() + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + identities, err := identity.Open(root) + if err != nil { + store.Close() + t.Fatal(err) + } + bootstrap, err := identities.Bootstrap(context.Background(), identity.BootstrapInput{Username: "operator", Email: "operator@example.test", DisplayName: "Operator", Password: "correct horse battery staple"}) + if err != nil { + identities.Close() + store.Close() + t.Fatal(err) + } + server, err := New(store, identities, testOptions()) + if err != nil { + identities.Close() + store.Close() + t.Fatal(err) + } + server.now = func() time.Time { return time.Date(2026, 8, 17, 7, 30, 0, 0, time.UTC) } + return server, store, identities, bootstrap +} + +func newRotationTestServer(t *testing.T) (*Server, *storage.Store, *identity.Services, identity.BootstrapResult) { + t.Helper() + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := storage.Open(root) + if err != nil { + t.Fatal(err) + } + identities, err := identity.Open(root) + if err != nil { + store.Close() + t.Fatal(err) + } + bootstrap, err := identities.Bootstrap(t.Context(), identity.BootstrapInput{Username: "operator", Email: "operator@example.test", DisplayName: "Operator", Password: "temporary correct horse battery staple", RequirePasswordChange: true}) + if err != nil { + identities.Close() + store.Close() + t.Fatal(err) + } + server, err := New(store, identities, testOptions()) + if err != nil { + identities.Close() + store.Close() + t.Fatal(err) + } + server.now = func() time.Time { return time.Date(2026, 8, 18, 5, 30, 0, 0, time.UTC) } + return server, store, identities, bootstrap +} + +func loginHTML(t *testing.T, handler http.Handler) *http.Cookie { + t.Helper() + result := loginHTMLResponse(t, handler, "operator", "correct horse battery staple") + if result.Code != http.StatusSeeOther || result.Header().Get("Location") != "/app/" { + t.Fatalf("login status=%d location=%q body=%s", result.Code, result.Header().Get("Location"), result.Body.String()) + } + for _, cookie := range result.Result().Cookies() { + if cookie.Name == "__Host-observatory_session" && cookie.Secure && cookie.HttpOnly && cookie.SameSite == http.SameSiteStrictMode { + return cookie + } + } + t.Fatalf("login cookies=%+v", result.Result().Cookies()) + return nil +} + +func loginHTMLResponse(t *testing.T, handler http.Handler, identifier, password string) *httptest.ResponseRecorder { + t.Helper() + csrfCookie, csrfToken := loginFormCSRF(t, handler) + form := url.Values{"csrf_token": []string{csrfToken}, "identifier": []string{identifier}, "password": []string{password}}.Encode() + headers := http.Header{"Origin": []string{"https://observatory.example"}, "Content-Type": []string{"application/x-www-form-urlencoded; charset=utf-8"}} + return perform(handler, http.MethodPost, "https://observatory.example/login/", strings.NewReader(form), []*http.Cookie{csrfCookie}, headers) +} + +func loginFormCSRF(t *testing.T, handler http.Handler) (*http.Cookie, string) { + t.Helper() + page := perform(handler, http.MethodGet, "https://observatory.example/login/", nil, nil) + if page.Code != http.StatusOK { + t.Fatalf("login page status=%d body=%s", page.Code, page.Body.String()) + } + var csrfCookie *http.Cookie + for _, cookie := range page.Result().Cookies() { + if cookie.Name == loginCSRFCookieName { + csrfCookie = cookie + break + } + } + if csrfCookie == nil || !csrfCookie.Secure || !csrfCookie.HttpOnly || csrfCookie.SameSite != http.SameSiteStrictMode { + t.Fatalf("login CSRF cookie=%+v", csrfCookie) + } + const marker = `name="csrf_token" value="` + start := strings.Index(page.Body.String(), marker) + if start < 0 { + t.Fatalf("login page omitted CSRF token: %s", page.Body.String()) + } + start += len(marker) + end := strings.IndexByte(page.Body.String()[start:], '"') + if end < 0 { + t.Fatal("login page CSRF token is unterminated") + } + token := page.Body.String()[start : start+end] + if token == "" || token != csrfCookie.Value { + t.Fatal("login form and cookie CSRF tokens differ") + } + return csrfCookie, token +} + +func perform(handler http.Handler, method, target string, body io.Reader, cookies []*http.Cookie, headerSets ...http.Header) *httptest.ResponseRecorder { + request := httptest.NewRequest(method, target, body) + for _, cookie := range cookies { + request.AddCookie(cookie) + } + for _, headers := range headerSets { + for name, values := range headers { + for _, value := range values { + request.Header.Add(name, value) + } + } + } + result := httptest.NewRecorder() + handler.ServeHTTP(result, request) + return result +} diff --git a/internal/identity/identity.go b/internal/identity/identity.go new file mode 100644 index 0000000..c19fb6f --- /dev/null +++ b/internal/identity/identity.go @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package identity binds Observatory's application policy to the storage-neutral +// Gamertan Web Foundations authentication, organization, and access packages. +package identity + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "gamertan.com/web/access" + "gamertan.com/web/auth" + "gamertan.com/web/authsqlite" + "gamertan.com/web/organizations" + _ "modernc.org/sqlite" +) + +const ( + PlatformOperator = "platform.operator" + OrganizationOwner = "organization.owner" + OrganizationViewer = "organization.viewer" + IncidentResponder = "incident.responder" + + PermissionPlatformOperate = "platform.operate" + PermissionTelemetryQuery = "telemetry.query" + PermissionTelemetryReadSensitive = "telemetry.sensitive" + PermissionSourcesManage = "sources.manage" + PermissionSchemaManage = "schema.manage" + PermissionDashboardsRead = "dashboards.read" + PermissionDashboardsManage = "dashboards.manage" + PermissionIncidentsRead = "incidents.read" + PermissionIncidentsManage = "incidents.manage" + PermissionOrganizationAudit = "organization.audit.read" + PermissionOrganizationManage = "organization.manage" +) + +var ( + ErrAlreadyBootstrapped = errors.New("identity: platform is already bootstrapped") + ErrResourceNotFound = errors.New("identity: resource scope not found") +) + +type Services struct { + Store *authsqlite.Store + Auth *auth.Service + Organizations *organizations.Service + Access *access.Service + control *sql.DB + dataDir string +} + +func Open(dataDir string) (*Services, error) { + if !filepath.IsAbs(dataDir) || filepath.Clean(dataDir) != dataDir { + return nil, errors.New("identity: data directory must be absolute and clean") + } + info, err := os.Lstat(dataDir) + if err != nil { + return nil, fmt.Errorf("identity: inspect data directory: %w", err) + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return nil, errors.New("identity: data directory must be a private non-symlink directory") + } + controlPath := filepath.Join(dataDir, "control.sqlite") + store, err := authsqlite.Open(controlPath) + if err != nil { + return nil, err + } + authService, err := auth.New(store, auth.Options{}) + if err != nil { + store.Close() + return nil, err + } + organizationService, err := organizations.New(store, organizations.Options{}) + if err != nil { + store.Close() + return nil, err + } + accessService, err := access.New(store, AccessPolicy(), access.Options{}) + if err != nil { + store.Close() + return nil, err + } + control, err := sql.Open("sqlite", sqliteDSN(controlPath)) + if err != nil { + store.Close() + return nil, err + } + control.SetMaxOpenConns(1) + services := &Services{Store: store, Auth: authService, Organizations: organizationService, Access: accessService, control: control, dataDir: dataDir} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err = services.seed(ctx); err != nil { + services.Close() + return nil, err + } + return services, nil +} + +func (services *Services) Close() error { + var errs []error + if services.control != nil { + errs = append(errs, services.control.Close()) + } + if services.Store != nil { + errs = append(errs, services.Store.Close()) + } + return errors.Join(errs...) +} + +func PlatformPolicy() auth.PolicySeed { + return auth.PolicySeed{ + Roles: map[string]string{PlatformOperator: "Operate the Observatory service without implicit access to organization telemetry."}, + Permissions: map[string]string{PermissionPlatformOperate: "Operate platform-level service and migration controls."}, + RolePermissions: map[string][]string{PlatformOperator: {PermissionPlatformOperate}}, + } +} + +func AccessPolicy() access.Policy { + permissions := map[string]string{ + PermissionTelemetryQuery: "Query telemetry in an explicitly authorized resource scope.", + PermissionTelemetryReadSensitive: "Read fields classified as sensitive.", + PermissionSourcesManage: "Enroll, rotate, and revoke ingestion sources.", + PermissionSchemaManage: "Review field descriptors and projection changes.", + PermissionDashboardsRead: "Read saved queries and dashboards.", + PermissionDashboardsManage: "Create and change saved queries and dashboards.", + PermissionIncidentsRead: "Read incidents for an authorized scope.", + PermissionIncidentsManage: "Acknowledge, silence, and resolve incidents.", + PermissionOrganizationAudit: "Read organization-visible security and access audit events.", + PermissionOrganizationManage: "Manage organization membership invitations and teams.", + } + return access.Policy{ + Roles: map[string]string{ + OrganizationOwner: "Manage an organization and its Observatory resources.", + OrganizationViewer: "Read ordinary telemetry, dashboards, and incidents.", + IncidentResponder: "Read telemetry and respond to incidents without managing sources or access.", + }, + Permissions: permissions, + Grants: map[string][]string{ + OrganizationOwner: { + PermissionTelemetryQuery, PermissionTelemetryReadSensitive, + PermissionSourcesManage, PermissionSchemaManage, PermissionDashboardsRead, + PermissionDashboardsManage, PermissionIncidentsRead, + PermissionIncidentsManage, PermissionOrganizationAudit, + PermissionOrganizationManage, + }, + OrganizationViewer: {PermissionTelemetryQuery, PermissionDashboardsRead, PermissionIncidentsRead}, + IncidentResponder: {PermissionTelemetryQuery, PermissionDashboardsRead, PermissionIncidentsRead, PermissionIncidentsManage}, + }, + } +} + +func (services *Services) CancelUnusedInvitation(ctx context.Context, digest [32]byte) error { + result, err := services.control.ExecContext(ctx, `DELETE FROM gwf_organization_invitations WHERE token_hash=? AND used_at IS NULL`, digest[:]) + if err != nil { + return errors.New("identity: cancel invitation") + } + if changed, _ := result.RowsAffected(); changed != 1 { + return errors.New("identity: unused invitation was not found") + } + return nil +} + +func (services *Services) seed(ctx context.Context) error { + if err := services.Store.SeedPolicy(ctx, PlatformPolicy()); err != nil { + return fmt.Errorf("identity: seed platform policy: %w", err) + } + if err := services.Access.Seed(ctx); err != nil { + return fmt.Errorf("identity: seed organization access policy: %w", err) + } + return nil +} + +func (services *Services) ValidateResourceScope(ctx context.Context, scope access.Scope) error { + if err := scope.Validate(); err != nil { + return err + } + queryText := `SELECT COUNT(*) FROM gwf_organizations WHERE id=?` + arguments := []any{scope.OrganizationID} + switch { + case scope.ServiceID != "": + queryText = `SELECT COUNT(*) FROM gwf_application_services WHERE id=? AND environment_id=? AND project_id=? AND organization_id=?` + arguments = []any{scope.ServiceID, scope.EnvironmentID, scope.ProjectID, scope.OrganizationID} + case scope.EnvironmentID != "": + queryText = `SELECT COUNT(*) FROM gwf_environments WHERE id=? AND project_id=? AND organization_id=?` + arguments = []any{scope.EnvironmentID, scope.ProjectID, scope.OrganizationID} + case scope.ProjectID != "": + queryText = `SELECT COUNT(*) FROM gwf_projects WHERE id=? AND organization_id=?` + arguments = []any{scope.ProjectID, scope.OrganizationID} + } + var count int + if err := services.control.QueryRowContext(ctx, queryText, arguments...).Scan(&count); err != nil { + return fmt.Errorf("identity: validate resource scope: %w", err) + } + if count != 1 { + return ErrResourceNotFound + } + return nil +} + +// OrganizationsForUser returns only active organizations in which the user +// has a direct membership. Access grants remain the independent authority for +// every operation performed after selection. +func (services *Services) OrganizationsForUser(ctx context.Context, userID string) ([]organizations.Organization, error) { + memberships, err := services.Organizations.Memberships(ctx, userID) + if err != nil { + return nil, fmt.Errorf("identity: list organization memberships: %w", err) + } + result := make([]organizations.Organization, 0, len(memberships)) + for _, membership := range memberships { + if membership.Status != "active" { + continue + } + var organization organizations.Organization + var personal int + var createdAt int64 + err = services.control.QueryRowContext(ctx, `SELECT id,slug,name,personal,created_at FROM gwf_organizations WHERE id=?`, membership.OrganizationID).Scan(&organization.ID, &organization.Slug, &organization.Name, &personal, &createdAt) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrResourceNotFound + } + if err != nil { + return nil, fmt.Errorf("identity: read organization: %w", err) + } + organization.Personal = personal == 1 + organization.CreatedAt = time.Unix(createdAt, 0).UTC() + result = append(result, organization) + } + sort.Slice(result, func(i, j int) bool { + if result[i].Name == result[j].Name { + return result[i].ID < result[j].ID + } + return result[i].Name < result[j].Name + }) + return result, nil +} + +type BootstrapInput struct { + Username, Email, DisplayName, Password string + RequirePasswordChange bool +} + +type BootstrapResult struct { + User auth.User + Organization organizations.Organization + Binding access.Binding +} + +type UserProvisionResult struct { + User auth.User + Organization organizations.Organization + Binding access.Binding +} + +// ProvisionUser creates an active local user and the personal organization +// that owns their private work. Shared organization access still requires a +// separately authorized, expiring invitation. +func (services *Services) ProvisionUser(ctx context.Context, input auth.CreateUser) (UserProvisionResult, error) { + user, err := services.Auth.CreateUser(ctx, input) + if err != nil { + return UserProvisionResult{}, fmt.Errorf("identity: create user: %w", err) + } + organization, err := services.Organizations.CreatePersonalOrganization(ctx, user.ID, user.DisplayName) + if err != nil { + return UserProvisionResult{}, fmt.Errorf("identity: create personal organization: %w", err) + } + binding, err := services.Access.Grant(ctx, access.Grant{ + SubjectKind: access.User, SubjectID: user.ID, Role: OrganizationOwner, + Scope: access.Scope{OrganizationID: organization.ID}, GrantedBy: user.ID, + }) + if err != nil { + return UserProvisionResult{}, fmt.Errorf("identity: grant personal organization ownership: %w", err) + } + return UserProvisionResult{User: user, Organization: organization, Binding: binding}, nil +} + +func (services *Services) Bootstrap(ctx context.Context, input BootstrapInput) (BootstrapResult, error) { + lock, err := openBootstrapLock(filepath.Join(services.dataDir, ".bootstrap.lock")) + if err != nil { + return BootstrapResult{}, err + } + defer lock.Close() + var users int + if err = services.control.QueryRowContext(ctx, `SELECT COUNT(*) FROM gwf_users`).Scan(&users); err != nil { + return BootstrapResult{}, fmt.Errorf("identity: inspect bootstrap state: %w", err) + } + if users != 0 { + return BootstrapResult{}, ErrAlreadyBootstrapped + } + provisioned, err := services.ProvisionUser(ctx, auth.CreateUser{ + Username: input.Username, Email: input.Email, + DisplayName: input.DisplayName, Password: input.Password, + RequirePasswordChange: input.RequirePasswordChange, + }) + if err != nil { + return BootstrapResult{}, fmt.Errorf("identity: create first operator: %w", err) + } + now := time.Now().UTC() + if err = services.Store.GrantRole(ctx, provisioned.User.ID, PlatformOperator, now); err != nil { + return BootstrapResult{}, fmt.Errorf("identity: grant platform operator: %w", err) + } + return BootstrapResult{User: provisioned.User, Organization: provisioned.Organization, Binding: provisioned.Binding}, nil +} + +func openBootstrapLock(path string) (*os.File, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return nil, fmt.Errorf("identity: open bootstrap lock: %w", err) + } + if err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + file.Close() + return nil, errors.New("identity: another bootstrap operation is active") + } + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + file.Close() + return nil, errors.New("identity: bootstrap lock must be a private regular file") + } + return file, nil +} + +func sqliteDSN(path string) string { + return (&url.URL{Scheme: "file", Path: filepath.ToSlash(path), RawQuery: "_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_pragma=synchronous(FULL)"}).String() +} + +func ReadSecret(path string, requireRoot bool) (string, error) { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return "", errors.New("identity: secret path must be absolute and clean") + } + info, err := os.Lstat(path) + if err != nil { + return "", fmt.Errorf("identity: inspect secret file: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm() != 0o600 { + return "", errors.New("identity: secret must be a regular non-symlink file with mode 0600") + } + if requireRoot { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Uid != 0 { + return "", errors.New("identity: secret must be owned by root") + } + } + value, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("identity: read secret file: %w", err) + } + secret := strings.TrimSuffix(string(value), "\n") + secret = strings.TrimSuffix(secret, "\r") + if secret == "" || strings.ContainsAny(secret, "\x00\r\n") { + return "", errors.New("identity: secret file must contain one non-empty line") + } + return secret, nil +} + +func WriteSecret(path, secret string) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path || secret == "" || len(secret) > 1024 || strings.ContainsAny(secret, "\x00\r\n") { + return errors.New("identity: secret output is invalid") + } + parent := filepath.Dir(path) + info, err := os.Lstat(parent) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("identity: secret output directory must be an existing non-symlink directory") + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return fmt.Errorf("identity: create secret output: %w", err) + } + remove := true + defer func() { + _ = file.Close() + if remove { + _ = os.Remove(path) + } + }() + if err = file.Chmod(0o600); err == nil { + _, err = io.WriteString(file, secret+"\n") + } + if err == nil { + err = file.Sync() + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err != nil { + return errors.New("identity: persist secret output") + } + directory, err := os.Open(parent) + if err != nil { + return errors.New("identity: open secret output directory") + } + if err = directory.Sync(); err != nil { + directory.Close() + return errors.New("identity: persist secret output directory") + } + if err = directory.Close(); err != nil { + return errors.New("identity: close secret output directory") + } + remove = false + return nil +} + +// RemoveSecret removes only an exact private regular secret file and syncs its +// parent directory. It is used to clean up a generated bootstrap credential +// when bootstrap cannot commit an operator. +func RemoveSecret(path string, requireRoot bool) error { + if _, err := ReadSecret(path, requireRoot); err != nil { + return err + } + if err := os.Remove(path); err != nil { + return fmt.Errorf("identity: remove secret file: %w", err) + } + directory, err := os.Open(filepath.Dir(path)) + if err != nil { + return errors.New("identity: open secret output directory") + } + if err = directory.Sync(); err != nil { + directory.Close() + return errors.New("identity: persist secret output directory") + } + if err = directory.Close(); err != nil { + return errors.New("identity: close secret output directory") + } + return nil +} diff --git a/internal/identity/identity_test.go b/internal/identity/identity_test.go new file mode 100644 index 0000000..147079b --- /dev/null +++ b/internal/identity/identity_test.go @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package identity + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "gamertan.com/web/access" + "gamertan.com/web/auth" + "gamertan.com/web/organizations" +) + +func TestBootstrapSeparatesPlatformAndOrganizationAccess(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + services, err := Open(root) + if err != nil { + t.Fatal(err) + } + defer services.Close() + result, err := services.Bootstrap(context.Background(), BootstrapInput{ + Username: "operator", Email: "operator@example.test", + DisplayName: "First Operator", Password: "correct horse battery staple", + }) + if err != nil { + t.Fatal(err) + } + if !result.Organization.Personal || result.Binding.Role != OrganizationOwner { + t.Fatalf("result=%+v", result) + } + token, principal, err := services.Auth.Authenticate(context.Background(), "operator", "correct horse battery staple", time.Hour) + if err != nil || token == "" || !principal.Has(PermissionPlatformOperate) { + t.Fatalf("platform session: token_present=%t principal=%+v err=%v", token != "", principal, err) + } + decision, err := services.Access.Authorize(context.Background(), result.User.ID, access.Scope{OrganizationID: result.Organization.ID}, PermissionTelemetryQuery) + if err != nil || !decision.Allowed || decision.Role != OrganizationOwner { + t.Fatalf("query decision=%+v err=%v", decision, err) + } + decision, err = services.Access.Authorize(context.Background(), result.User.ID, access.Scope{OrganizationID: result.Organization.ID}, PermissionTelemetryReadSensitive) + if err != nil || !decision.Allowed { + t.Fatalf("sensitive decision=%+v err=%v", decision, err) + } + decision, err = services.Access.Authorize(context.Background(), result.User.ID, access.Scope{OrganizationID: result.Organization.ID}, PermissionSchemaManage) + if err != nil || !decision.Allowed { + t.Fatalf("schema decision=%+v err=%v", decision, err) + } + if _, err = services.Access.Authorize(context.Background(), result.User.ID, access.Scope{OrganizationID: result.Organization.ID}, PermissionPlatformOperate); err == nil { + t.Fatal("platform permission entered organization access policy") + } + project, err := services.Organizations.CreateProject(context.Background(), organizations.CreateProject{OrganizationID: result.Organization.ID, Slug: "eql-helper", Name: "EQL Helper"}) + if err != nil { + t.Fatal(err) + } + environment, err := services.Organizations.CreateEnvironment(context.Background(), organizations.CreateEnvironment{OrganizationID: result.Organization.ID, ProjectID: project.ID, Slug: "production", Name: "Production"}) + if err != nil { + t.Fatal(err) + } + application, err := services.Organizations.CreateApplicationService(context.Background(), organizations.CreateApplicationService{OrganizationID: result.Organization.ID, ProjectID: project.ID, EnvironmentID: environment.ID, Slug: "web", Name: "Web"}) + if err != nil { + t.Fatal(err) + } + scope := access.Scope{OrganizationID: result.Organization.ID, ProjectID: project.ID, EnvironmentID: environment.ID, ServiceID: application.ID} + if err = services.ValidateResourceScope(context.Background(), scope); err != nil { + t.Fatal(err) + } + scope.ServiceID = "missing1" + if err = services.ValidateResourceScope(context.Background(), scope); !errors.Is(err, ErrResourceNotFound) { + t.Fatalf("missing scope err=%v", err) + } +} + +func TestBootstrapIsSingleUse(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + services, err := Open(root) + if err != nil { + t.Fatal(err) + } + defer services.Close() + input := BootstrapInput{Username: "operator", Email: "operator@example.test", DisplayName: "First Operator", Password: "correct horse battery staple"} + if _, err = services.Bootstrap(context.Background(), input); err != nil { + t.Fatal(err) + } + if _, err = services.Bootstrap(context.Background(), BootstrapInput{Username: "second", Email: "second@example.test", DisplayName: "Second Operator", Password: "correct horse battery staple"}); !errors.Is(err, ErrAlreadyBootstrapped) { + t.Fatalf("second bootstrap err=%v", err) + } +} + +func TestBootstrapCanRequirePasswordChange(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + services, err := Open(root) + if err != nil { + t.Fatal(err) + } + defer services.Close() + result, err := services.Bootstrap(t.Context(), BootstrapInput{ + Username: "operator", Email: "operator@example.test", DisplayName: "First Operator", + Password: "temporary correct horse battery staple", RequirePasswordChange: true, + }) + if err != nil || !result.User.PasswordChangeRequired { + t.Fatalf("result=%+v err=%v", result, err) + } + _, principal, err := services.Auth.Authenticate(t.Context(), "operator", "temporary correct horse battery staple", time.Hour) + if err != nil || !principal.User.PasswordChangeRequired { + t.Fatalf("principal=%+v err=%v", principal, err) + } +} + +func TestRemoveSecretValidatesAndRemovesOnlyPrivateRegularFile(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "bootstrap-password") + if err := WriteSecret(path, "temporary secret"); err != nil { + t.Fatal(err) + } + if err := RemoveSecret(path, false); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("removed path err=%v", err) + } + unsafe := filepath.Join(root, "unsafe") + if err := os.WriteFile(unsafe, []byte("secret\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := RemoveSecret(unsafe, false); err == nil { + t.Fatal("world-readable secret was removed") + } + if _, err := os.Stat(unsafe); err != nil { + t.Fatalf("unsafe file changed: %v", err) + } +} + +func TestEvidenceRetentionPrunesOnlyExpiredAudit(t *testing.T) { + ctx := t.Context() + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + services, err := Open(root) + if err != nil { + t.Fatal(err) + } + defer services.Close() + owner, err := services.Bootstrap(ctx, BootstrapInput{Username: "operator", Email: "operator@example.test", DisplayName: "First Operator", Password: "correct horse battery staple"}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 5, 0, 0, 0, time.UTC) + for _, event := range []struct { + id string + created time.Time + }{{"audit-old", now.Add(-401 * 24 * time.Hour)}, {"audit-current", now.Add(-399 * 24 * time.Hour)}} { + if _, err = services.control.ExecContext(ctx, `INSERT INTO gwf_audit_events(id,actor_user_id,action,resource_type,resource_id,summary,created_at) VALUES(?,?,?,?,?,?,?)`, event.id, owner.User.ID, "session.test", "user", owner.User.ID, "Test event", event.created.Unix()); err != nil { + t.Fatal(err) + } + if _, err = services.control.ExecContext(ctx, `INSERT INTO gwf_access_audit_events(id,organization_id,actor_user_id,action,resource_type,resource_id,summary,created_at) VALUES(?,?,?,?,?,?,?,?)`, "access-"+event.id, owner.Organization.ID, owner.User.ID, "access.test", "organization", owner.Organization.ID, "Test event", event.created.Unix()); err != nil { + t.Fatal(err) + } + } + report, err := services.PruneEvidence(ctx, 400, now) + if err != nil { + t.Fatal(err) + } + if report.AuthenticationEvents != 1 || report.OrganizationEvents != 1 { + t.Fatalf("report=%+v", report) + } + for _, table := range []string{"gwf_audit_events", "gwf_access_audit_events"} { + var count int + if err = services.control.QueryRowContext(ctx, `SELECT COUNT(*) FROM `+table).Scan(&count); err != nil || count != 1 { + t.Fatalf("table=%s count=%d err=%v", table, count, err) + } + } +} + +func TestTeamsInvitationsRevocationAndBreakGlassRemainOrganizationScoped(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + services, err := Open(root) + if err != nil { + t.Fatal(err) + } + defer services.Close() + owner, err := services.Bootstrap(ctx, BootstrapInput{Username: "operator", Email: "operator@example.test", DisplayName: "First Operator", Password: "correct horse battery staple"}) + if err != nil { + t.Fatal(err) + } + member, err := services.Auth.CreateUser(ctx, auth.CreateUser{Username: "responder", Email: "responder@example.test", DisplayName: "Incident Responder", Password: "another correct horse battery staple"}) + if err != nil { + t.Fatal(err) + } + rawInvitation, invitation, err := services.Organizations.Invite(ctx, owner.Organization.ID, member.Email, owner.User.ID, 15*time.Minute) + if err != nil || rawInvitation == "" || invitation.OrganizationID != owner.Organization.ID { + t.Fatalf("invitation=%+v token_present=%t err=%v", invitation, rawInvitation != "", err) + } + if err = services.Organizations.AcceptInvitation(ctx, rawInvitation, member.ID); err != nil { + t.Fatal(err) + } + team, err := services.Organizations.CreateTeam(ctx, organizations.CreateTeam{OrganizationID: owner.Organization.ID, Slug: "responders", Name: "Incident Responders"}) + if err != nil { + t.Fatal(err) + } + if err = services.Organizations.AddTeamMember(ctx, team.ID, member.ID); err != nil { + t.Fatal(err) + } + binding, err := services.Access.Grant(ctx, access.Grant{SubjectKind: access.Team, SubjectID: team.ID, Role: IncidentResponder, Scope: access.Scope{OrganizationID: owner.Organization.ID}, GrantedBy: owner.User.ID}) + if err != nil { + t.Fatal(err) + } + decision, err := services.Access.Authorize(ctx, member.ID, access.Scope{OrganizationID: owner.Organization.ID}, PermissionIncidentsManage) + if err != nil || !decision.Allowed || decision.Source != "role" || decision.Role != IncidentResponder { + t.Fatalf("team decision=%+v err=%v", decision, err) + } + decision, err = services.Access.Authorize(ctx, owner.User.ID, access.Scope{OrganizationID: owner.Organization.ID}, PermissionOrganizationManage) + if err != nil || !decision.Allowed || decision.Role != OrganizationOwner { + t.Fatalf("owner organization-management decision=%+v err=%v", decision, err) + } + decision, err = services.Access.Authorize(ctx, member.ID, access.Scope{OrganizationID: owner.Organization.ID}, PermissionOrganizationManage) + if err != nil || decision.Allowed { + t.Fatalf("member organization-management decision=%+v err=%v", decision, err) + } + cancelToken, cancelInvitation, err := services.Organizations.Invite(ctx, owner.Organization.ID, "cancelled@example.test", owner.User.ID, 15*time.Minute) + if err != nil || cancelToken == "" { + t.Fatalf("cancel invitation=%+v token_present=%t err=%v", cancelInvitation, cancelToken != "", err) + } + if err = services.CancelUnusedInvitation(ctx, cancelInvitation.Digest); err != nil { + t.Fatal(err) + } + if err = services.Organizations.AcceptInvitation(ctx, cancelToken, member.ID); err == nil { + t.Fatal("cancelled invitation remained usable") + } + other, err := services.Organizations.CreatePersonalOrganization(ctx, member.ID, member.DisplayName) + if err != nil { + t.Fatal(err) + } + decision, err = services.Access.Authorize(ctx, owner.User.ID, access.Scope{OrganizationID: other.ID}, PermissionTelemetryQuery) + if err != nil || decision.Allowed { + t.Fatalf("cross-organization decision=%+v err=%v", decision, err) + } + decision, err = services.Access.Authorize(ctx, member.ID, access.Scope{OrganizationID: owner.Organization.ID}, PermissionTelemetryReadSensitive) + if err != nil || decision.Allowed { + t.Fatalf("unexpected sensitive decision=%+v err=%v", decision, err) + } + breakGlass, err := services.Access.ActivateBreakGlass(ctx, owner.Organization.ID, member.ID, PermissionTelemetryReadSensitive, "Investigate an active incident", "request-12345678", 15*time.Minute) + if err != nil { + t.Fatal(err) + } + decision, err = services.Access.Authorize(ctx, member.ID, access.Scope{OrganizationID: owner.Organization.ID}, PermissionTelemetryReadSensitive) + if err != nil || !decision.Allowed || decision.Source != "break_glass" { + t.Fatalf("break-glass decision=%+v err=%v", decision, err) + } + audit, err := services.Access.Audit(ctx, owner.Organization.ID, 10) + if err != nil || len(audit) != 1 || audit[0].Action != "break_glass.activate" || audit[0].ResourceID != owner.Organization.ID { + t.Fatalf("audit=%+v err=%v", audit, err) + } + if _, err = services.control.ExecContext(ctx, `UPDATE gwf_break_glass SET expires_at=? WHERE id=?`, time.Now().Add(-time.Minute).Unix(), breakGlass.ID); err != nil { + t.Fatal(err) + } + decision, err = services.Access.Authorize(ctx, member.ID, access.Scope{OrganizationID: owner.Organization.ID}, PermissionTelemetryReadSensitive) + if err != nil || decision.Allowed { + t.Fatalf("expired break-glass decision=%+v err=%v", decision, err) + } + if err = services.Store.Revoke(ctx, binding.ID, owner.User.ID, time.Now().UTC()); err != nil { + t.Fatal(err) + } + decision, err = services.Access.Authorize(ctx, member.ID, access.Scope{OrganizationID: owner.Organization.ID}, PermissionIncidentsManage) + if err != nil || decision.Allowed { + t.Fatalf("revoked team decision=%+v err=%v", decision, err) + } +} + +func TestReadSecretRejectsWeakFiles(t *testing.T) { + dir := t.TempDir() + valid := filepath.Join(dir, "password") + if err := os.WriteFile(valid, []byte("correct horse battery staple\n"), 0o600); err != nil { + t.Fatal(err) + } + secret, err := ReadSecret(valid, false) + if err != nil || secret != "correct horse battery staple" { + t.Fatalf("secret=%q err=%v", secret, err) + } + weak := filepath.Join(dir, "weak") + if err = os.WriteFile(weak, []byte("not private"), 0o644); err != nil { + t.Fatal(err) + } + if _, err = ReadSecret(weak, false); err == nil { + t.Fatal("world-readable secret accepted") + } + if runtime.GOOS != "windows" { + link := filepath.Join(dir, "link") + if err = os.Symlink(valid, link); err != nil { + t.Fatal(err) + } + if _, err = ReadSecret(link, false); err == nil { + t.Fatal("symlinked secret accepted") + } + } +} + +func TestWriteSecretIsPrivateExclusiveAndReadable(t *testing.T) { + path := filepath.Join(t.TempDir(), "invitation") + const secret = "single-use-invitation-token" + if err := WriteSecret(path, secret); err != nil { + t.Fatal(err) + } + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + t.Fatalf("info=%v err=%v", info, err) + } + got, err := ReadSecret(path, false) + if err != nil || got != secret { + t.Fatalf("secret=%q err=%v", got, err) + } + if err = WriteSecret(path, "replacement"); err == nil { + t.Fatal("existing secret was overwritten") + } + if err = WriteSecret(filepath.Join(filepath.Dir(path), "multiline"), "first\nsecond"); err == nil { + t.Fatal("multiline secret was accepted") + } + if runtime.GOOS != "windows" { + linkedParent := filepath.Join(filepath.Dir(path), "linked-parent") + if err = os.Symlink(filepath.Dir(path), linkedParent); err != nil { + t.Fatal(err) + } + if err = WriteSecret(filepath.Join(linkedParent, "through-link"), "secret"); err == nil { + t.Fatal("symlinked secret output directory was accepted") + } + } +} diff --git a/internal/identity/retention.go b/internal/identity/retention.go new file mode 100644 index 0000000..14c17a5 --- /dev/null +++ b/internal/identity/retention.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package identity + +import ( + "context" + "errors" + "time" +) + +type EvidencePruneReport struct { + AuthenticationEvents int64 `json:"authentication_events"` + OrganizationEvents int64 `json:"organization_events"` + ExpiredSessions int64 `json:"expired_sessions"` + ExpiredInvitations int64 `json:"expired_invitations"` + ExpiredBreakGlass int64 `json:"expired_break_glass"` +} + +// PruneEvidence applies the server default to platform authentication audit +// events and each organization's approved retention override to its visible +// access audit. Operational credentials are removed only after expiration. +func (services *Services) PruneEvidence(ctx context.Context, defaultDays int, now time.Time) (EvidencePruneReport, error) { + if services == nil || services.control == nil || defaultDays < 1 || defaultDays > 3650 || now.IsZero() { + return EvidencePruneReport{}, errors.New("identity: evidence retention input is invalid") + } + tx, err := services.control.BeginTx(ctx, nil) + if err != nil { + return EvidencePruneReport{}, errors.New("identity: begin evidence retention") + } + defer tx.Rollback() + report := EvidencePruneReport{} + cutoff := now.UTC().Add(-time.Duration(defaultDays) * 24 * time.Hour).Unix() + result, err := tx.ExecContext(ctx, `DELETE FROM gwf_audit_events WHERE created_at 3650 { + _ = rows.Close() + return report, errors.New("identity: organization audit retention is invalid") + } + organizations = append(organizations, organization) + } + if err = rows.Close(); err != nil { + return report, errors.New("identity: close organization audit retention") + } + for _, organization := range organizations { + organizationCutoff := now.UTC().Add(-time.Duration(organization.days) * 24 * time.Hour).Unix() + result, err = tx.ExecContext(ctx, `DELETE FROM gwf_access_audit_events WHERE organization_id=? AND created_at 1_000_000 || !validLowerHex(value.AgentEpoch, 32) || value.Sequence == 0 || value.Sequence > math.MaxInt64 || ValidateStreamID(value.StreamID) != nil || value.BatchSequence == 0 || value.BatchSequence > math.MaxInt64 || !validLowerHex(value.SegmentDigest, 64) { + return errors.New("alert transition identity is invalid") + } + if value.State != "matched" && value.State != "clear" && value.State != "error" { + return errors.New("alert transition state is invalid") + } + if now.IsZero() || value.WindowStart.IsZero() || value.WindowEnd.Before(value.WindowStart) || value.WindowEnd.Sub(value.WindowStart) > 24*time.Hour || value.ObservedAt.Before(value.WindowEnd) || value.ObservedAt.Before(now.Add(-7*24*time.Hour)) || value.ObservedAt.After(now.Add(10*time.Minute)) { + return errors.New("alert transition time range is invalid") + } + return nil +} + +func (value AlertTransition) Digest() (string, error) { + encoded, err := json.Marshal(value) + if err != nil { + return "", errors.New("encode alert transition digest") + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]), nil +} diff --git a/internal/model/alert_transition_test.go b/internal/model/alert_transition_test.go new file mode 100644 index 0000000..9930b20 --- /dev/null +++ b/internal/model/alert_transition_test.go @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package model + +import ( + "strings" + "testing" + "time" +) + +func TestAlertTransitionValidationAndDigest(t *testing.T) { + now := time.Date(2026, 8, 18, 22, 30, 0, 0, time.UTC) + value := AlertTransition{Version: AlertTransitionVersion, RuleID: "rule-a", RuleRevision: 1, AgentEpoch: strings.Repeat("a", 32), Sequence: 1, StreamID: "requests", BatchSequence: 2, SegmentDigest: strings.Repeat("b", 64), WindowStart: now.Add(-time.Minute), WindowEnd: now, State: "matched", ObservedAt: now} + if err := value.Validate(now); err != nil { + t.Fatal(err) + } + first, err := value.Digest() + if err != nil || len(first) != 64 { + t.Fatalf("digest=%q err=%v", first, err) + } + second, err := value.Digest() + if err != nil || first != second { + t.Fatalf("digest changed: %q %q err=%v", first, second, err) + } + invalid := []AlertTransition{ + {}, + func() AlertTransition { copy := value; copy.AgentEpoch = "not-hex"; return copy }(), + func() AlertTransition { copy := value; copy.Sequence = 0; return copy }(), + func() AlertTransition { copy := value; copy.Sequence = ^uint64(0); return copy }(), + func() AlertTransition { copy := value; copy.BatchSequence = ^uint64(0); return copy }(), + func() AlertTransition { copy := value; copy.State = "firing"; return copy }(), + func() AlertTransition { copy := value; copy.WindowStart = now.Add(-25 * time.Hour); return copy }(), + func() AlertTransition { copy := value; copy.ObservedAt = now.Add(-time.Second); return copy }(), + } + for index, candidate := range invalid { + if err := candidate.Validate(now); err == nil { + t.Fatalf("invalid transition %d accepted", index) + } + } +} diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 0000000..d4d3fd1 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package model + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "strings" + "time" + "unicode/utf8" +) + +// Digest returns the SHA-256 of the canonical JSON representation used by the +// native agent protocol. It binds an acknowledgement to the exact logical +// batch independently of either side's private compressed storage format. +func (b Batch) Digest() (string, error) { + encoded, err := json.Marshal(b) + if err != nil { + return "", errors.New("encode batch digest") + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]), nil +} + +const ( + BatchVersion = 1 + BatchEnvelopeVersion = 1 + MaxRecords = 5_000 + MaxAttributes = 64 + MaxAttributeKey = 128 + MaxAttributeValue = 4_096 + MaxName = 256 + MaxBody = 16_384 + MaxDistinctFields = 1_024 +) + +// BatchEnvelope is the bounded transport metadata for one native batch. The +// enrolled credential supplies source and tenant scope; timestamps are useful +// partition hints and deliberately do not participate in record-level +// deduplication. +type BatchEnvelope struct { + Version int + StreamID string + Sequence uint64 + Signal Signal + WireDigest string + BatchDigest string + RecordCount int + EncodedBytes int64 + FirstObservedAt time.Time + LastObservedAt time.Time +} + +type Signal string + +const ( + SignalLogs Signal = "logs" + SignalMetrics Signal = "metrics" + SignalTraces Signal = "traces" + SignalDeployments Signal = "deployments" +) + +type Batch struct { + Version int `json:"version"` + SourceID string `json:"source_id"` + StreamID string `json:"stream_id"` + Sequence uint64 `json:"sequence"` + ObservedAt time.Time `json:"observed_at"` + Signal Signal `json:"signal"` + Records []Observation `json:"records"` +} + +type Observation struct { + Timestamp time.Time `json:"timestamp"` + Name string `json:"name"` + Severity string `json:"severity,omitempty"` + Body string `json:"body,omitempty"` + Value *float64 `json:"value,omitempty"` + TraceID string `json:"trace_id,omitempty"` + SpanID string `json:"span_id,omitempty"` + CorrelationID string `json:"correlation_id,omitempty"` + Attributes map[string]string `json:"attributes,omitempty"` +} + +type Scope struct { + OrganizationID string `json:"organization_id"` + ProjectID string `json:"project_id"` + EnvironmentID string `json:"environment_id"` + ServiceID string `json:"service_id"` +} + +// Envelope returns transport metadata bound to the exact encoded body and the +// canonical logical batch. The two digests are intentionally distinct even +// when the current JSON encoder happens to produce identical bytes. +func (b Batch) Envelope(encoded []byte) (BatchEnvelope, error) { + if len(encoded) == 0 { + return BatchEnvelope{}, errors.New("encoded batch is empty") + } + batchDigest, err := b.Digest() + if err != nil { + return BatchEnvelope{}, err + } + wireDigest := sha256.Sum256(encoded) + first, last := b.ObservationRange() + return BatchEnvelope{ + Version: BatchEnvelopeVersion, + StreamID: b.StreamID, + Sequence: b.Sequence, + Signal: b.Signal, + WireDigest: hex.EncodeToString(wireDigest[:]), + BatchDigest: batchDigest, + RecordCount: len(b.Records), + EncodedBytes: int64(len(encoded)), + FirstObservedAt: first, + LastObservedAt: last, + }, nil +} + +func (b Batch) ObservationRange() (time.Time, time.Time) { + if len(b.Records) == 0 { + return time.Time{}, time.Time{} + } + first, last := b.Records[0].Timestamp.UTC(), b.Records[0].Timestamp.UTC() + for _, record := range b.Records[1:] { + observed := record.Timestamp.UTC() + if observed.Before(first) { + first = observed + } + if observed.After(last) { + last = observed + } + } + return first, last +} + +func (e BatchEnvelope) Validate(maxEncodedBytes int64) error { + if e.Version != BatchEnvelopeVersion { + return errors.New("unsupported batch envelope version") + } + if err := ValidateStreamID(e.StreamID); err != nil { + return err + } + if e.Sequence == 0 { + return errors.New("sequence must be positive") + } + if !e.Signal.valid() { + return errors.New("unsupported signal") + } + if !validLowerHex(e.WireDigest, sha256.Size*2) || !validLowerHex(e.BatchDigest, sha256.Size*2) { + return errors.New("batch envelope digest is invalid") + } + if e.RecordCount < 1 || e.RecordCount > MaxRecords { + return errors.New("batch envelope record count is invalid") + } + if e.EncodedBytes < 1 || e.EncodedBytes > maxEncodedBytes { + return errors.New("batch envelope byte count is invalid") + } + if e.FirstObservedAt.IsZero() || e.LastObservedAt.IsZero() || e.LastObservedAt.Before(e.FirstObservedAt) { + return errors.New("batch envelope time range is invalid") + } + return nil +} + +// Match proves that the decoded batch and exact transport bytes agree with +// the agent-supplied envelope. Tenant scope remains absent by design. +func (e BatchEnvelope) Match(batch Batch, encoded []byte) error { + if err := e.Validate(int64(len(encoded))); err != nil || e.EncodedBytes != int64(len(encoded)) { + return errors.New("batch envelope does not match encoded body") + } + expected, err := batch.Envelope(encoded) + if err != nil { + return err + } + if e != expected { + return errors.New("batch envelope does not match encoded body") + } + return nil +} + +func (b Batch) Validate(now time.Time) error { + if b.Version != BatchVersion { + return fmt.Errorf("unsupported batch version %d", b.Version) + } + if err := validateID("source_id", b.SourceID); err != nil { + return err + } + if err := validateID("stream_id", b.StreamID); err != nil { + return err + } + if b.Sequence == 0 { + return errors.New("sequence must be positive") + } + if !b.Signal.valid() { + return errors.New("unsupported signal") + } + if b.ObservedAt.IsZero() || b.ObservedAt.Before(now.Add(-7*24*time.Hour)) || b.ObservedAt.After(now.Add(10*time.Minute)) { + return errors.New("observed_at outside accepted clock-skew window") + } + if len(b.Records) == 0 || len(b.Records) > MaxRecords { + return fmt.Errorf("records must contain between 1 and %d items", MaxRecords) + } + distinctFields := map[string]struct{}{} + for i, record := range b.Records { + if err := record.validate(b.Signal, now); err != nil { + return fmt.Errorf("record %d: %w", i, err) + } + for field := range record.Attributes { + distinctFields[field] = struct{}{} + if len(distinctFields) > MaxDistinctFields { + return fmt.Errorf("batch contains more than %d distinct attribute fields", MaxDistinctFields) + } + } + } + return nil +} + +func (s Signal) valid() bool { + switch s { + case SignalLogs, SignalMetrics, SignalTraces, SignalDeployments: + return true + default: + return false + } +} + +func (o Observation) validate(signal Signal, now time.Time) error { + if o.Timestamp.IsZero() || o.Timestamp.Before(now.Add(-400*24*time.Hour)) || o.Timestamp.After(now.Add(10*time.Minute)) { + return errors.New("timestamp outside accepted window") + } + if err := validateText("name", o.Name, MaxName, false); err != nil { + return err + } + if err := validateText("body", o.Body, MaxBody, true); err != nil { + return err + } + if err := validateText("severity", o.Severity, 64, true); err != nil { + return err + } + if o.TraceID != "" && !validLowerHex(o.TraceID, 32) { + return errors.New("trace_id must be 16 bytes encoded as lowercase hexadecimal") + } + if o.SpanID != "" && !validLowerHex(o.SpanID, 16) { + return errors.New("span_id must be 8 bytes encoded as lowercase hexadecimal") + } + if err := validateText("correlation_id", o.CorrelationID, 128, true); err != nil { + return err + } + if signal == SignalMetrics && o.Value == nil { + return errors.New("metric requires value") + } + if o.Value != nil && (math.IsNaN(*o.Value) || math.IsInf(*o.Value, 0)) { + return errors.New("value must be finite") + } + if len(o.Attributes) > MaxAttributes { + return fmt.Errorf("too many attributes: maximum %d", MaxAttributes) + } + for key, value := range o.Attributes { + if err := validateText("attribute key", key, MaxAttributeKey, false); err != nil { + return err + } + if err := validateText("attribute value", value, MaxAttributeValue, true); err != nil { + return err + } + } + return nil +} + +func validLowerHex(value string, length int) bool { + if len(value) != length { + return false + } + for _, character := range value { + if !(character >= '0' && character <= '9' || character >= 'a' && character <= 'f') { + return false + } + } + return true +} + +func (s Scope) Validate() error { + for label, value := range map[string]string{ + "organization_id": s.OrganizationID, + "project_id": s.ProjectID, + "environment_id": s.EnvironmentID, + "service_id": s.ServiceID, + } { + if err := validateID(label, value); err != nil { + return err + } + } + return nil +} + +func validateID(label, value string) error { + if len(value) < 1 || len(value) > 128 { + return fmt.Errorf("%s length must be between 1 and 128", label) + } + for _, r := range value { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("._-", r)) { + return fmt.Errorf("%s contains an invalid character", label) + } + } + return nil +} + +func ValidateSourceID(value string) error { return validateID("source_id", value) } + +func ValidateStreamID(value string) error { return validateID("stream_id", value) } + +func validateText(label, value string, max int, empty bool) error { + if !utf8.ValidString(value) || strings.IndexByte(value, 0) >= 0 { + return fmt.Errorf("%s must be valid UTF-8 without NUL", label) + } + if !empty && value == "" { + return fmt.Errorf("%s is required", label) + } + if len(value) > max { + return fmt.Errorf("%s exceeds %d bytes", label, max) + } + return nil +} diff --git a/internal/model/model_test.go b/internal/model/model_test.go new file mode 100644 index 0000000..b64ffc7 --- /dev/null +++ b/internal/model/model_test.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package model + +import ( + "encoding/json" + "fmt" + "math" + "strings" + "testing" + "time" +) + +func TestBatchEnvelopeBindsTransportAndTimePartitionHints(t *testing.T) { + now := time.Date(2026, 8, 18, 20, 0, 0, 0, time.UTC) + batch := Batch{Version: BatchVersion, SourceID: "source", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: SignalLogs, Records: []Observation{{Timestamp: now, Name: "latest"}, {Timestamp: now.Add(-time.Hour), Name: "earliest"}}} + body, err := json.Marshal(batch) + if err != nil { + t.Fatal(err) + } + envelope, err := batch.Envelope(body) + if err != nil { + t.Fatal(err) + } + if envelope.RecordCount != 2 || envelope.EncodedBytes != int64(len(body)) || envelope.FirstObservedAt != now.Add(-time.Hour) || envelope.LastObservedAt != now || envelope.WireDigest != envelope.BatchDigest { + t.Fatalf("envelope=%+v", envelope) + } + if err = envelope.Match(batch, body); err != nil { + t.Fatal(err) + } + padded := append([]byte(" \n"), body...) + paddedEnvelope, err := batch.Envelope(padded) + if err != nil || paddedEnvelope.WireDigest == envelope.WireDigest || paddedEnvelope.BatchDigest != envelope.BatchDigest || paddedEnvelope.EncodedBytes != int64(len(padded)) { + t.Fatalf("padded=%+v err=%v", paddedEnvelope, err) + } + if err = envelope.Match(batch, padded); err == nil { + t.Fatal("transport mutation was accepted") + } + // Overlapping time ranges are valid metadata, not a uniqueness key. + batch.Sequence = 2 + batch.Records = []Observation{{Timestamp: now.Add(-30 * time.Minute), Name: "overlap"}} + body, _ = json.Marshal(batch) + if overlap, overlapErr := batch.Envelope(body); overlapErr != nil || overlap.FirstObservedAt != now.Add(-30*time.Minute) { + t.Fatalf("overlap=%+v err=%v", overlap, overlapErr) + } +} + +func TestBatchDigestIsCanonicalAndContentBound(t *testing.T) { + now := time.Date(2026, 8, 17, 18, 0, 0, 0, time.UTC) + left := Batch{Version: BatchVersion, SourceID: "source", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: SignalLogs, Records: []Observation{{Timestamp: now, Name: "request", Attributes: map[string]string{"z": "last", "a": "first"}}}} + right := left + right.Records = []Observation{{Timestamp: now, Name: "request", Attributes: map[string]string{"a": "first", "z": "last"}}} + leftDigest, err := left.Digest() + if err != nil { + t.Fatal(err) + } + rightDigest, err := right.Digest() + if err != nil { + t.Fatal(err) + } + if leftDigest != rightDigest || len(leftDigest) != 64 || strings.Trim(leftDigest, "0123456789abcdef") != "" { + t.Fatalf("left=%q right=%q", leftDigest, rightDigest) + } + right.Records[0].Name = "changed" + changed, err := right.Digest() + if err != nil || changed == leftDigest { + t.Fatalf("changed=%q err=%v", changed, err) + } +} + +func validBatch(now time.Time) Batch { + v := 1.5 + return Batch{Version: 1, SourceID: "src_1", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: SignalMetrics, Records: []Observation{{Timestamp: now, Name: "http.duration", Value: &v, Attributes: map[string]string{"route": "/"}}}} +} + +func TestBatchRejectsDistinctFieldCardinalityAbuse(t *testing.T) { + now := time.Now().UTC() + batch := Batch{Version: BatchVersion, SourceID: "source", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: SignalLogs} + for recordIndex := 0; recordIndex < 17; recordIndex++ { + record := Observation{Timestamp: now, Name: "record", Attributes: map[string]string{}} + for fieldIndex := 0; fieldIndex < MaxAttributes; fieldIndex++ { + record.Attributes[fmt.Sprintf("field.%d.%d", recordIndex, fieldIndex)] = "value" + } + batch.Records = append(batch.Records, record) + } + if err := batch.Validate(now); err == nil || !strings.Contains(err.Error(), "distinct attribute fields") { + t.Fatalf("cardinality abuse err=%v", err) + } +} + +func TestBatchValidation(t *testing.T) { + now := time.Now().UTC() + if err := validBatch(now).Validate(now); err != nil { + t.Fatal(err) + } + b := validBatch(now) + b.SourceID = "../../tenant" + if err := b.Validate(now); err == nil || !strings.Contains(err.Error(), "invalid character") { + t.Fatalf("expected path-like ID rejection, got %v", err) + } + b = validBatch(now) + b.Records[0].Attributes["secret"] = strings.Repeat("x", MaxAttributeValue+1) + if err := b.Validate(now); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("expected attribute bound, got %v", err) + } +} + +func TestBatchClockSkewAndRetentionWindowsFailClosed(t *testing.T) { + now := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC) + for name, mutate := range map[string]func(*Batch){ + "old batch": func(batch *Batch) { batch.ObservedAt = now.Add(-7*24*time.Hour - time.Nanosecond) }, + "future batch": func(batch *Batch) { batch.ObservedAt = now.Add(10*time.Minute + time.Nanosecond) }, + "old record": func(batch *Batch) { batch.Records[0].Timestamp = now.Add(-400*24*time.Hour - time.Nanosecond) }, + "future record": func(batch *Batch) { batch.Records[0].Timestamp = now.Add(10*time.Minute + time.Nanosecond) }, + "zero observed": func(batch *Batch) { batch.ObservedAt = time.Time{} }, + "zero timestamp": func(batch *Batch) { batch.Records[0].Timestamp = time.Time{} }, + } { + t.Run(name, func(t *testing.T) { + batch := validBatch(now) + mutate(&batch) + if err := batch.Validate(now); err == nil { + t.Fatal("out-of-window telemetry was accepted") + } + }) + } +} + +func TestObservationRejectsUnsafeIdentifiersAndNonFiniteValues(t *testing.T) { + now := time.Now().UTC() + nan := math.NaN() + base := Batch{Version: BatchVersion, SourceID: "source", StreamID: "stream", Sequence: 1, ObservedAt: now, Signal: SignalLogs, Records: []Observation{{Timestamp: now, Name: "record"}}} + for name, mutate := range map[string]func(*Observation){ + "trace": func(record *Observation) { record.TraceID = "ABC" }, + "span": func(record *Observation) { record.SpanID = strings.Repeat("g", 16) }, + "correlation": func(record *Observation) { record.CorrelationID = strings.Repeat("x", 129) }, + "value": func(record *Observation) { record.Value = &nan }, + } { + t.Run(name, func(t *testing.T) { + batch := base + batch.Records = append([]Observation(nil), base.Records...) + mutate(&batch.Records[0]) + if err := batch.Validate(now); err == nil { + t.Fatal("unsafe observation accepted") + } + }) + } +} diff --git a/internal/nativeprotocol/headers.go b/internal/nativeprotocol/headers.go new file mode 100644 index 0000000..a3b876e --- /dev/null +++ b/internal/nativeprotocol/headers.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package nativeprotocol + +import ( + "errors" + "net/http" + "strconv" + "time" + + "gamertan.com/observatory/internal/model" +) + +const ( + VersionHeader = "Observatory-Batch-Version" + StreamHeader = "Observatory-Stream-ID" + SequenceHeader = "Observatory-Sequence" + SignalHeader = "Observatory-Signal" + WireDigestHeader = "Observatory-Wire-SHA256" + BatchDigestHeader = "Observatory-Batch-SHA256" + RecordCountHeader = "Observatory-Record-Count" + EncodedBytesHeader = "Observatory-Encoded-Bytes" + FirstObservedHeader = "Observatory-First-Observed-At" + LastObservedHeader = "Observatory-Last-Observed-At" +) + +var envelopeHeaders = []string{ + VersionHeader, StreamHeader, SequenceHeader, SignalHeader, + WireDigestHeader, BatchDigestHeader, RecordCountHeader, + EncodedBytesHeader, FirstObservedHeader, LastObservedHeader, +} + +func SetHeaders(header http.Header, envelope model.BatchEnvelope) { + header.Set(VersionHeader, strconv.Itoa(envelope.Version)) + header.Set(StreamHeader, envelope.StreamID) + header.Set(SequenceHeader, strconv.FormatUint(envelope.Sequence, 10)) + header.Set(SignalHeader, string(envelope.Signal)) + header.Set(WireDigestHeader, envelope.WireDigest) + header.Set(BatchDigestHeader, envelope.BatchDigest) + header.Set(RecordCountHeader, strconv.Itoa(envelope.RecordCount)) + header.Set(EncodedBytesHeader, strconv.FormatInt(envelope.EncodedBytes, 10)) + header.Set(FirstObservedHeader, envelope.FirstObservedAt.UTC().Format(time.RFC3339Nano)) + header.Set(LastObservedHeader, envelope.LastObservedAt.UTC().Format(time.RFC3339Nano)) +} + +func ParseHeaders(header http.Header, maxEncodedBytes int64) (model.BatchEnvelope, error) { + values := make(map[string]string, len(envelopeHeaders)) + for _, name := range envelopeHeaders { + items := header.Values(name) + if len(items) != 1 || items[0] == "" { + return model.BatchEnvelope{}, errors.New("native batch envelope headers are incomplete") + } + values[name] = items[0] + } + version, versionErr := strconv.Atoi(values[VersionHeader]) + sequence, sequenceErr := strconv.ParseUint(values[SequenceHeader], 10, 64) + recordCount, recordErr := strconv.Atoi(values[RecordCountHeader]) + encodedBytes, bytesErr := strconv.ParseInt(values[EncodedBytesHeader], 10, 64) + first, firstErr := time.Parse(time.RFC3339Nano, values[FirstObservedHeader]) + last, lastErr := time.Parse(time.RFC3339Nano, values[LastObservedHeader]) + if versionErr != nil || sequenceErr != nil || recordErr != nil || bytesErr != nil || firstErr != nil || lastErr != nil { + return model.BatchEnvelope{}, errors.New("native batch envelope headers are invalid") + } + envelope := model.BatchEnvelope{ + Version: version, StreamID: values[StreamHeader], Sequence: sequence, + Signal: model.Signal(values[SignalHeader]), WireDigest: values[WireDigestHeader], + BatchDigest: values[BatchDigestHeader], RecordCount: recordCount, + EncodedBytes: encodedBytes, FirstObservedAt: first.UTC(), LastObservedAt: last.UTC(), + } + if err := envelope.Validate(maxEncodedBytes); err != nil { + return model.BatchEnvelope{}, errors.New("native batch envelope headers are invalid") + } + return envelope, nil +} diff --git a/internal/nativeprotocol/headers_test.go b/internal/nativeprotocol/headers_test.go new file mode 100644 index 0000000..cb84622 --- /dev/null +++ b/internal/nativeprotocol/headers_test.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package nativeprotocol + +import ( + "encoding/json" + "net/http" + "testing" + "time" + + "gamertan.com/observatory/internal/model" +) + +func TestEnvelopeHeadersRoundTripAndRejectAmbiguity(t *testing.T) { + now := time.Date(2026, 8, 18, 20, 0, 0, 0, time.UTC) + batch := model.Batch{Version: model.BatchVersion, SourceID: "source", StreamID: "logs", Sequence: 7, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now.Add(-time.Second), Name: "first"}, {Timestamp: now, Name: "last"}}} + body, err := json.Marshal(batch) + if err != nil { + t.Fatal(err) + } + envelope, err := batch.Envelope(body) + if err != nil { + t.Fatal(err) + } + header := make(http.Header) + SetHeaders(header, envelope) + parsed, err := ParseHeaders(header, 1<<20) + if err != nil || parsed != envelope { + t.Fatalf("parsed=%+v err=%v", parsed, err) + } + header.Add(SequenceHeader, "8") + if _, err = ParseHeaders(header, 1<<20); err == nil { + t.Fatal("duplicate security-relevant header was accepted") + } +} + +func FuzzParseEnvelopeHeaders(f *testing.F) { + f.Add("1", "logs", "1", "logs", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "1", "100", "2026-08-18T20:00:00Z", "2026-08-18T20:00:00Z") + f.Fuzz(func(t *testing.T, version, stream, sequence, signal, wire, batch, count, size, first, last string) { + header := make(http.Header) + for name, value := range map[string]string{VersionHeader: version, StreamHeader: stream, SequenceHeader: sequence, SignalHeader: signal, WireDigestHeader: wire, BatchDigestHeader: batch, RecordCountHeader: count, EncodedBytesHeader: size, FirstObservedHeader: first, LastObservedHeader: last} { + header.Set(name, value) + } + _, _ = ParseHeaders(header, 32<<20) + }) +} diff --git a/internal/otlp/otlp.go b/internal/otlp/otlp.go new file mode 100644 index 0000000..8fa84a2 --- /dev/null +++ b/internal/otlp/otlp.go @@ -0,0 +1,666 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package otlp + +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "strconv" + "strings" + "time" + "unicode/utf8" + + "gamertan.com/observatory/internal/model" + commonpb "go.opentelemetry.io/proto/otlp/common/v1" + logspb "go.opentelemetry.io/proto/otlp/logs/v1" + metricspb "go.opentelemetry.io/proto/otlp/metrics/v1" + resourcepb "go.opentelemetry.io/proto/otlp/resource/v1" + tracepb "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + +type Signal string + +const ( + Logs Signal = "logs" + Metrics Signal = "metrics" + Traces Signal = "traces" +) + +func (signal Signal) ModelSignal() model.Signal { + switch signal { + case Logs: + return model.SignalLogs + case Metrics: + return model.SignalMetrics + case Traces: + return model.SignalTraces + default: + return "" + } +} + +func (signal Signal) StreamID() string { return "otlp-" + string(signal) } + +func Decode(signal Signal, body []byte, now time.Time) ([]model.Observation, error) { + if len(body) == 0 || now.IsZero() { + return nil, errors.New("OTLP request is empty or missing collection time") + } + options := proto.UnmarshalOptions{DiscardUnknown: true, RecursionLimit: 64} + var records []model.Observation + var err error + switch signal { + case Logs: + request := new(logspb.LogsData) + if err = options.Unmarshal(body, request); err == nil { + records, err = decodeLogs(request) + } + case Metrics: + request := new(metricspb.MetricsData) + if err = options.Unmarshal(body, request); err == nil { + records, err = decodeMetrics(request) + } + case Traces: + request := new(tracepb.TracesData) + if err = options.Unmarshal(body, request); err == nil { + records, err = decodeTraces(request) + } + default: + return nil, errors.New("unsupported OTLP signal") + } + if err != nil { + return nil, errors.New("invalid OTLP protobuf payload") + } + if len(records) == 0 { + return nil, errors.New("OTLP request contains no supported records") + } + if len(records) > model.MaxRecords { + return nil, fmt.Errorf("OTLP request exceeds %d records", model.MaxRecords) + } + return records, nil +} + +func SuccessResponse(signal Signal) ([]byte, error) { + switch signal { + case Logs, Metrics, Traces: + // Each successful OTLP/HTTP protobuf response is an empty message when + // there is no partial-success detail. The canonical protobuf encoding of + // an empty message is an empty byte sequence. + return []byte{}, nil + default: + return nil, errors.New("unsupported OTLP signal") + } +} + +func decodeLogs(request *logspb.LogsData) ([]model.Observation, error) { + var records []model.Observation + for _, resourceLogs := range request.GetResourceLogs() { + resource, err := baseAttributes(resourceLogs.GetResource()) + if err != nil { + return nil, err + } + for _, scopeLogs := range resourceLogs.GetScopeLogs() { + base, err := scopeAttributes(resource, scopeLogs.GetScope()) + if err != nil { + return nil, err + } + for _, record := range scopeLogs.GetLogRecords() { + attributes, err := mergeAttributes(base, record.GetAttributes()) + if err != nil { + return nil, err + } + if record.GetDroppedAttributesCount() > 0 { + if err = addAttribute(attributes, "otel.dropped_attributes", strconv.FormatUint(uint64(record.GetDroppedAttributesCount()), 10)); err != nil { + return nil, err + } + } + body, err := anyValueText(record.GetBody(), model.MaxBody) + if err != nil { + return nil, err + } + name := record.GetEventName() + if name == "" { + name = "otlp.log" + } + when := record.GetTimeUnixNano() + if when == 0 { + when = record.GetObservedTimeUnixNano() + } + timestamp, err := timestamp(when) + if err != nil { + return nil, err + } + traceID, err := identifier(record.GetTraceId(), 16) + if err != nil { + return nil, err + } + spanID, err := identifier(record.GetSpanId(), 8) + if err != nil { + return nil, err + } + severity := record.GetSeverityText() + if severity == "" && record.GetSeverityNumber() != 0 { + severity = record.GetSeverityNumber().String() + } + records = append(records, model.Observation{Timestamp: timestamp, Name: name, Severity: severity, Body: body, TraceID: traceID, SpanID: spanID, CorrelationID: traceID, Attributes: attributes}) + if len(records) > model.MaxRecords { + return nil, errors.New("OTLP logs exceed record limit") + } + } + } + } + return records, nil +} + +func decodeMetrics(request *metricspb.MetricsData) ([]model.Observation, error) { + var records []model.Observation + for _, resourceMetrics := range request.GetResourceMetrics() { + resource, err := baseAttributes(resourceMetrics.GetResource()) + if err != nil { + return nil, err + } + for _, scopeMetrics := range resourceMetrics.GetScopeMetrics() { + base, err := scopeAttributes(resource, scopeMetrics.GetScope()) + if err != nil { + return nil, err + } + for _, metric := range scopeMetrics.GetMetrics() { + metricBase := cloneAttributes(base) + if metric.GetUnit() != "" { + if err = addAttribute(metricBase, "metric.unit", metric.GetUnit()); err != nil { + return nil, err + } + } + switch data := metric.Data.(type) { + case *metricspb.Metric_Gauge: + err = appendNumberPoints(&records, metric.GetName(), "gauge", "", metricBase, data.Gauge.GetDataPoints()) + case *metricspb.Metric_Sum: + err = appendNumberPoints(&records, metric.GetName(), "sum", data.Sum.GetAggregationTemporality().String(), metricBase, data.Sum.GetDataPoints()) + case *metricspb.Metric_Histogram: + err = appendHistogramPoints(&records, metric.GetName(), data.Histogram.GetAggregationTemporality().String(), metricBase, data.Histogram.GetDataPoints()) + case *metricspb.Metric_ExponentialHistogram: + err = appendExponentialHistogramPoints(&records, metric.GetName(), data.ExponentialHistogram.GetAggregationTemporality().String(), metricBase, data.ExponentialHistogram.GetDataPoints()) + case *metricspb.Metric_Summary: + err = appendSummaryPoints(&records, metric.GetName(), metricBase, data.Summary.GetDataPoints()) + default: + err = errors.New("OTLP metric has unsupported data") + } + if err != nil { + return nil, err + } + if len(records) > model.MaxRecords { + return nil, errors.New("OTLP metrics exceed record limit") + } + } + } + } + return records, nil +} + +func appendNumberPoints(records *[]model.Observation, name, kind, temporality string, base map[string]string, points []*metricspb.NumberDataPoint) error { + for _, point := range points { + if point.GetFlags()&uint32(metricspb.DataPointFlags_DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) != 0 { + continue + } + attributes, err := metricAttributes(base, point.GetAttributes(), kind, temporality, point.GetStartTimeUnixNano()) + if err != nil { + return err + } + var value float64 + switch number := point.Value.(type) { + case *metricspb.NumberDataPoint_AsDouble: + value = number.AsDouble + case *metricspb.NumberDataPoint_AsInt: + value = float64(number.AsInt) + if err = addAttribute(attributes, "metric.int64", strconv.FormatInt(number.AsInt, 10)); err != nil { + return err + } + default: + return errors.New("OTLP number point has no value") + } + if math.IsNaN(value) || math.IsInf(value, 0) { + return errors.New("OTLP metric value is not finite") + } + timestamp, err := timestamp(point.GetTimeUnixNano()) + if err != nil { + return err + } + *records = append(*records, model.Observation{Timestamp: timestamp, Name: name, Value: &value, Attributes: attributes}) + } + return nil +} + +func appendHistogramPoints(records *[]model.Observation, name, temporality string, base map[string]string, points []*metricspb.HistogramDataPoint) error { + for _, point := range points { + if point.GetFlags()&uint32(metricspb.DataPointFlags_DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) != 0 { + continue + } + attributes, err := metricAttributes(base, point.GetAttributes(), "histogram", temporality, point.GetStartTimeUnixNano()) + if err != nil { + return err + } + bounds := point.GetExplicitBounds() + if len(point.GetBucketCounts()) != len(bounds)+1 || !strictlyIncreasingFinite(bounds) { + return errors.New("OTLP histogram buckets are invalid") + } + for key, value := range map[string]any{"metric.bucket_counts": point.GetBucketCounts(), "metric.explicit_bounds": point.GetExplicitBounds(), "metric.count": point.GetCount()} { + if err = addJSONAttribute(attributes, key, value); err != nil { + return err + } + } + value := float64(point.GetCount()) + if point.Sum != nil { + value = point.GetSum() + if !finite(value) { + return errors.New("OTLP histogram sum is not finite") + } + if err = addAttribute(attributes, "metric.sum", strconv.FormatFloat(value, 'g', -1, 64)); err != nil { + return err + } + } + if point.Min != nil { + if !finite(point.GetMin()) { + return errors.New("OTLP histogram minimum is not finite") + } + if err = addAttribute(attributes, "metric.min", strconv.FormatFloat(point.GetMin(), 'g', -1, 64)); err != nil { + return err + } + } + if point.Max != nil { + if !finite(point.GetMax()) { + return errors.New("OTLP histogram maximum is not finite") + } + if err = addAttribute(attributes, "metric.max", strconv.FormatFloat(point.GetMax(), 'g', -1, 64)); err != nil { + return err + } + } + timestamp, err := timestamp(point.GetTimeUnixNano()) + if err != nil { + return err + } + *records = append(*records, model.Observation{Timestamp: timestamp, Name: name, Value: &value, Attributes: attributes}) + } + return nil +} + +func appendExponentialHistogramPoints(records *[]model.Observation, name, temporality string, base map[string]string, points []*metricspb.ExponentialHistogramDataPoint) error { + for _, point := range points { + if point.GetFlags()&uint32(metricspb.DataPointFlags_DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) != 0 { + continue + } + attributes, err := metricAttributes(base, point.GetAttributes(), "exponential_histogram", temporality, point.GetStartTimeUnixNano()) + if err != nil { + return err + } + if point.GetScale() < -10 || point.GetScale() > 20 || !finite(point.GetZeroThreshold()) || point.GetZeroThreshold() < 0 { + return errors.New("OTLP exponential histogram parameters are invalid") + } + values := map[string]any{ + "metric.count": point.GetCount(), "metric.scale": point.GetScale(), "metric.zero_count": point.GetZeroCount(), + "metric.zero_threshold": point.GetZeroThreshold(), "metric.positive": point.GetPositive(), "metric.negative": point.GetNegative(), + } + for key, value := range values { + if err = addJSONAttribute(attributes, key, value); err != nil { + return err + } + } + value := float64(point.GetCount()) + if point.Sum != nil { + value = point.GetSum() + if !finite(value) { + return errors.New("OTLP exponential histogram sum is not finite") + } + if err = addAttribute(attributes, "metric.sum", strconv.FormatFloat(value, 'g', -1, 64)); err != nil { + return err + } + } + timestamp, err := timestamp(point.GetTimeUnixNano()) + if err != nil { + return err + } + *records = append(*records, model.Observation{Timestamp: timestamp, Name: name, Value: &value, Attributes: attributes}) + } + return nil +} + +func appendSummaryPoints(records *[]model.Observation, name string, base map[string]string, points []*metricspb.SummaryDataPoint) error { + for _, point := range points { + if point.GetFlags()&uint32(metricspb.DataPointFlags_DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) != 0 { + continue + } + attributes, err := metricAttributes(base, point.GetAttributes(), "summary", "", point.GetStartTimeUnixNano()) + if err != nil { + return err + } + if err = addJSONAttribute(attributes, "metric.count", point.GetCount()); err != nil { + return err + } + quantiles := make([][2]float64, 0, len(point.GetQuantileValues())) + for _, quantile := range point.GetQuantileValues() { + if !finite(quantile.GetQuantile()) || quantile.GetQuantile() < 0 || quantile.GetQuantile() > 1 || !finite(quantile.GetValue()) { + return errors.New("OTLP summary quantile is invalid") + } + quantiles = append(quantiles, [2]float64{quantile.GetQuantile(), quantile.GetValue()}) + } + if err = addJSONAttribute(attributes, "metric.quantiles", quantiles); err != nil { + return err + } + value := point.GetSum() + if !finite(value) { + return errors.New("OTLP summary sum is not finite") + } + if err = addAttribute(attributes, "metric.sum", strconv.FormatFloat(value, 'g', -1, 64)); err != nil { + return err + } + timestamp, err := timestamp(point.GetTimeUnixNano()) + if err != nil { + return err + } + *records = append(*records, model.Observation{Timestamp: timestamp, Name: name, Value: &value, Attributes: attributes}) + } + return nil +} + +func decodeTraces(request *tracepb.TracesData) ([]model.Observation, error) { + var records []model.Observation + for _, resourceSpans := range request.GetResourceSpans() { + resource, err := baseAttributes(resourceSpans.GetResource()) + if err != nil { + return nil, err + } + for _, scopeSpans := range resourceSpans.GetScopeSpans() { + base, err := scopeAttributes(resource, scopeSpans.GetScope()) + if err != nil { + return nil, err + } + for _, span := range scopeSpans.GetSpans() { + attributes, err := mergeAttributes(base, span.GetAttributes()) + if err != nil { + return nil, err + } + traceID, err := identifier(span.GetTraceId(), 16) + if err != nil || traceID == "" { + return nil, errors.New("OTLP span has invalid trace ID") + } + spanID, err := identifier(span.GetSpanId(), 8) + if err != nil || spanID == "" { + return nil, errors.New("OTLP span has invalid span ID") + } + if parent, parentErr := identifier(span.GetParentSpanId(), 8); parentErr != nil { + return nil, parentErr + } else if parent != "" { + if err = addAttribute(attributes, "span.parent_id", parent); err != nil { + return nil, err + } + } + start, err := timestamp(span.GetStartTimeUnixNano()) + if err != nil { + return nil, err + } + end, err := timestamp(span.GetEndTimeUnixNano()) + if err != nil || end.Before(start) { + return nil, errors.New("OTLP span has invalid time range") + } + duration := float64(end.Sub(start).Nanoseconds()) + if err = addAttribute(attributes, "span.duration_ns", strconv.FormatInt(end.Sub(start).Nanoseconds(), 10)); err != nil { + return nil, err + } + if err = addAttribute(attributes, "span.kind", span.GetKind().String()); err != nil { + return nil, err + } + if status := span.GetStatus(); status != nil { + if err = addAttribute(attributes, "span.status", status.GetCode().String()); err != nil { + return nil, err + } + } + for key, count := range map[string]uint64{"span.events_count": uint64(len(span.GetEvents())), "span.links_count": uint64(len(span.GetLinks())), "otel.dropped_attributes": uint64(span.GetDroppedAttributesCount()), "otel.dropped_events": uint64(span.GetDroppedEventsCount()), "otel.dropped_links": uint64(span.GetDroppedLinksCount())} { + if count > 0 { + if err = addAttribute(attributes, key, strconv.FormatUint(count, 10)); err != nil { + return nil, err + } + } + } + records = append(records, model.Observation{Timestamp: start, Name: span.GetName(), Value: &duration, TraceID: traceID, SpanID: spanID, CorrelationID: traceID, Attributes: attributes}) + if len(records) > model.MaxRecords { + return nil, errors.New("OTLP traces exceed record limit") + } + } + } + } + return records, nil +} + +func baseAttributes(resource *resourcepb.Resource) (map[string]string, error) { + if resource == nil { + return map[string]string{}, nil + } + attributes, err := mergeAttributes(nil, resource.GetAttributes()) + if err != nil { + return nil, err + } + if resource.GetDroppedAttributesCount() > 0 { + err = addAttribute(attributes, "otel.resource.dropped_attributes", strconv.FormatUint(uint64(resource.GetDroppedAttributesCount()), 10)) + } + return attributes, err +} + +func scopeAttributes(base map[string]string, scope *commonpb.InstrumentationScope) (map[string]string, error) { + attributes := cloneAttributes(base) + if scope == nil { + return attributes, nil + } + if scope.GetName() != "" { + if err := addAttribute(attributes, "otel.scope.name", scope.GetName()); err != nil { + return nil, err + } + } + if scope.GetVersion() != "" { + if err := addAttribute(attributes, "otel.scope.version", scope.GetVersion()); err != nil { + return nil, err + } + } + for _, pair := range scope.GetAttributes() { + if pair.GetKey() == "" || deniedKey(pair.GetKey()) { + continue + } + value, err := anyValueText(pair.GetValue(), model.MaxAttributeValue) + if err != nil { + return nil, err + } + if err = addAttribute(attributes, "otel.scope.attribute."+pair.GetKey(), value); err != nil { + return nil, err + } + } + return attributes, nil +} + +func metricAttributes(base map[string]string, pairs []*commonpb.KeyValue, kind, temporality string, start uint64) (map[string]string, error) { + attributes, err := mergeAttributes(base, pairs) + if err != nil { + return nil, err + } + if err = addAttribute(attributes, "metric.kind", kind); err != nil { + return nil, err + } + if temporality != "" { + if err = addAttribute(attributes, "metric.temporality", temporality); err != nil { + return nil, err + } + } + if start != 0 { + if err = addAttribute(attributes, "metric.start_unix_nano", strconv.FormatUint(start, 10)); err != nil { + return nil, err + } + } + return attributes, nil +} + +func mergeAttributes(base map[string]string, pairs []*commonpb.KeyValue) (map[string]string, error) { + attributes := cloneAttributes(base) + for _, pair := range pairs { + key := pair.GetKey() + if key == "" || deniedKey(key) { + continue + } + value, err := anyValueText(pair.GetValue(), model.MaxAttributeValue) + if err != nil { + return nil, err + } + if err = addAttribute(attributes, key, value); err != nil { + return nil, err + } + } + return attributes, nil +} + +func cloneAttributes(source map[string]string) map[string]string { + copy := make(map[string]string, len(source)) + for key, value := range source { + copy[key] = value + } + return copy +} + +func addAttribute(attributes map[string]string, key, value string) error { + if key == "" || len(key) > model.MaxAttributeKey || !utf8.ValidString(key) || strings.IndexByte(key, 0) >= 0 || len(value) > model.MaxAttributeValue || !utf8.ValidString(value) || strings.IndexByte(value, 0) >= 0 { + return errors.New("OTLP attribute exceeds accepted bounds") + } + if _, exists := attributes[key]; !exists && len(attributes) >= model.MaxAttributes { + return errors.New("OTLP record exceeds attribute limit") + } + attributes[key] = value + return nil +} + +func addJSONAttribute(attributes map[string]string, key string, value any) error { + body, err := json.Marshal(value) + if err != nil || len(body) > model.MaxAttributeValue { + return errors.New("OTLP structured attribute exceeds accepted bounds") + } + return addAttribute(attributes, key, string(body)) +} + +func anyValueText(value *commonpb.AnyValue, maximum int) (string, error) { + converted, err := anyValue(value, 0) + if err != nil { + return "", err + } + if text, ok := converted.(string); ok { + if len(text) > maximum || !utf8.ValidString(text) || strings.IndexByte(text, 0) >= 0 { + return "", errors.New("OTLP string value exceeds accepted bounds") + } + return text, nil + } + body, err := json.Marshal(converted) + if err != nil || len(body) > maximum { + return "", errors.New("OTLP value exceeds accepted bounds") + } + return string(body), nil +} + +func anyValue(value *commonpb.AnyValue, depth int) (any, error) { + if value == nil { + return "", nil + } + if depth > 8 { + return nil, errors.New("OTLP value exceeds nesting limit") + } + switch content := value.Value.(type) { + case *commonpb.AnyValue_StringValue: + return content.StringValue, nil + case *commonpb.AnyValue_BoolValue: + return content.BoolValue, nil + case *commonpb.AnyValue_IntValue: + return content.IntValue, nil + case *commonpb.AnyValue_DoubleValue: + if math.IsNaN(content.DoubleValue) || math.IsInf(content.DoubleValue, 0) { + return nil, errors.New("OTLP value is not finite") + } + return content.DoubleValue, nil + case *commonpb.AnyValue_BytesValue: + if len(content.BytesValue) > model.MaxAttributeValue/2 { + return nil, errors.New("OTLP byte value exceeds accepted bounds") + } + return hex.EncodeToString(content.BytesValue), nil + case *commonpb.AnyValue_ArrayValue: + values := content.ArrayValue.GetValues() + if len(values) > 64 { + return nil, errors.New("OTLP array exceeds element limit") + } + result := make([]any, 0, len(values)) + for _, item := range values { + converted, err := anyValue(item, depth+1) + if err != nil { + return nil, err + } + result = append(result, converted) + } + return result, nil + case *commonpb.AnyValue_KvlistValue: + values := content.KvlistValue.GetValues() + if len(values) > 64 { + return nil, errors.New("OTLP key-value list exceeds element limit") + } + result := make(map[string]any, len(values)) + for _, pair := range values { + key := pair.GetKey() + if key == "" || deniedKey(key) { + continue + } + converted, err := anyValue(pair.GetValue(), depth+1) + if err != nil { + return nil, err + } + result[key] = converted + } + return result, nil + case *commonpb.AnyValue_StringValueStrindex: + return "", nil + default: + return "", nil + } +} + +func deniedKey(key string) bool { + normalized := strings.NewReplacer("-", "_", ".", "_", "/", "_").Replace(strings.ToLower(key)) + for _, denied := range []string{"authorization", "proxy_authorization", "cookie", "set_cookie", "password", "passwd", "secret", "api_key", "apikey", "access_token", "refresh_token", "client_secret"} { + if strings.Contains(normalized, denied) { + return true + } + } + return false +} + +func finite(value float64) bool { return !math.IsNaN(value) && !math.IsInf(value, 0) } + +func strictlyIncreasingFinite(values []float64) bool { + for index, value := range values { + if !finite(value) || index > 0 && value <= values[index-1] { + return false + } + } + return true +} + +func identifier(value []byte, bytes int) (string, error) { + if len(value) == 0 { + return "", nil + } + if len(value) != bytes { + return "", errors.New("OTLP identifier has invalid length") + } + return hex.EncodeToString(value), nil +} + +func timestamp(nanoseconds uint64) (time.Time, error) { + if nanoseconds == 0 { + return time.Time{}, errors.New("OTLP timestamp is required") + } + if nanoseconds > math.MaxInt64 { + return time.Time{}, errors.New("OTLP timestamp exceeds supported range") + } + return time.Unix(0, int64(nanoseconds)).UTC(), nil +} diff --git a/internal/otlp/otlp_test.go b/internal/otlp/otlp_test.go new file mode 100644 index 0000000..55bf244 --- /dev/null +++ b/internal/otlp/otlp_test.go @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package otlp + +import ( + "bytes" + "math" + "testing" + "time" + + commonpb "go.opentelemetry.io/proto/otlp/common/v1" + logspb "go.opentelemetry.io/proto/otlp/logs/v1" + metricspb "go.opentelemetry.io/proto/otlp/metrics/v1" + resourcepb "go.opentelemetry.io/proto/otlp/resource/v1" + tracepb "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + +func TestDecodeLogsDropsCredentialsAndPreservesTelemetry(t *testing.T) { + now := time.Date(2026, 8, 17, 3, 0, 0, 0, time.UTC) + request := &logspb.LogsData{ResourceLogs: []*logspb.ResourceLogs{{ + Resource: &resourcepb.Resource{Attributes: []*commonpb.KeyValue{ + stringAttribute("service.name", "eql"), + stringAttribute("http.request.header.authorization", "Bearer do-not-store"), + }}, + ScopeLogs: []*logspb.ScopeLogs{{ + Scope: &commonpb.InstrumentationScope{Name: "test", Version: "1"}, + LogRecords: []*logspb.LogRecord{{ + TimeUnixNano: uint64(now.UnixNano()), EventName: "http.request", SeverityText: "INFO", + Body: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: "complete"}}, + TraceId: bytes.Repeat([]byte{0x11}, 16), SpanId: bytes.Repeat([]byte{0x22}, 8), + Attributes: []*commonpb.KeyValue{stringAttribute("http.route", "/items/{id}")}, + }}, + }}, + }}} + records, err := Decode(Logs, marshal(t, request), now) + if err != nil { + t.Fatal(err) + } + if len(records) != 1 || records[0].Name != "http.request" || records[0].Body != "complete" || records[0].TraceID != "11111111111111111111111111111111" { + t.Fatalf("records=%+v", records) + } + if records[0].Attributes["service.name"] != "eql" || records[0].Attributes["http.route"] != "/items/{id}" || records[0].Attributes["otel.scope.name"] != "test" { + t.Fatalf("attributes=%+v", records[0].Attributes) + } + if _, exists := records[0].Attributes["http.request.header.authorization"]; exists { + t.Fatal("credential-bearing attribute was retained") + } +} + +func TestDecodeMetricsAndTraces(t *testing.T) { + now := time.Date(2026, 8, 17, 3, 0, 0, 0, time.UTC) + point := &metricspb.NumberDataPoint{ + TimeUnixNano: uint64(now.UnixNano()), + Value: &metricspb.NumberDataPoint_AsDouble{AsDouble: 12.5}, + Attributes: []*commonpb.KeyValue{stringAttribute("http.route", "/")}, + } + metric := &metricspb.Metric{Name: "http.server.duration", Unit: "ms", Data: &metricspb.Metric_Gauge{Gauge: &metricspb.Gauge{DataPoints: []*metricspb.NumberDataPoint{point}}}} + metricRequest := &metricspb.MetricsData{ResourceMetrics: []*metricspb.ResourceMetrics{{ScopeMetrics: []*metricspb.ScopeMetrics{{Metrics: []*metricspb.Metric{metric}}}}}} + metrics, err := Decode(Metrics, marshal(t, metricRequest), now) + if err != nil { + t.Fatal(err) + } + if len(metrics) != 1 || metrics[0].Value == nil || *metrics[0].Value != 12.5 || metrics[0].Attributes["metric.unit"] != "ms" { + t.Fatalf("metrics=%+v", metrics) + } + + span := &tracepb.Span{ + TraceId: bytes.Repeat([]byte{0x33}, 16), SpanId: bytes.Repeat([]byte{0x44}, 8), Name: "GET /", Kind: tracepb.Span_SPAN_KIND_SERVER, + StartTimeUnixNano: uint64(now.UnixNano()), EndTimeUnixNano: uint64(now.Add(7 * time.Millisecond).UnixNano()), Status: &tracepb.Status{Code: tracepb.Status_STATUS_CODE_OK}, + } + traceRequest := &tracepb.TracesData{ResourceSpans: []*tracepb.ResourceSpans{{ScopeSpans: []*tracepb.ScopeSpans{{Spans: []*tracepb.Span{span}}}}}} + traces, err := Decode(Traces, marshal(t, traceRequest), now) + if err != nil { + t.Fatal(err) + } + if len(traces) != 1 || traces[0].Name != "GET /" || traces[0].Value == nil || *traces[0].Value != float64((7*time.Millisecond).Nanoseconds()) || traces[0].Attributes["span.status"] != "STATUS_CODE_OK" { + t.Fatalf("traces=%+v", traces) + } +} + +func TestDecodeRejectsMalformedAndNonFiniteData(t *testing.T) { + now := time.Date(2026, 8, 17, 3, 0, 0, 0, time.UTC) + badPoint := &metricspb.NumberDataPoint{TimeUnixNano: uint64(now.UnixNano()), Value: &metricspb.NumberDataPoint_AsDouble{AsDouble: math.NaN()}} + badMetricValue := &metricspb.Metric{Name: "bad", Data: &metricspb.Metric_Gauge{Gauge: &metricspb.Gauge{DataPoints: []*metricspb.NumberDataPoint{badPoint}}}} + badMetric := &metricspb.MetricsData{ResourceMetrics: []*metricspb.ResourceMetrics{{ScopeMetrics: []*metricspb.ScopeMetrics{{Metrics: []*metricspb.Metric{badMetricValue}}}}}} + if _, err := Decode(Metrics, marshal(t, badMetric), now); err == nil { + t.Fatal("non-finite metric accepted") + } + badTrace := &tracepb.TracesData{ResourceSpans: []*tracepb.ResourceSpans{{ScopeSpans: []*tracepb.ScopeSpans{{Spans: []*tracepb.Span{{TraceId: []byte{1}, SpanId: bytes.Repeat([]byte{2}, 8), Name: "bad", StartTimeUnixNano: uint64(now.UnixNano()), EndTimeUnixNano: uint64(now.UnixNano())}}}}}}} + if _, err := Decode(Traces, marshal(t, badTrace), now); err == nil { + t.Fatal("invalid trace identifier accepted") + } + if _, err := Decode(Logs, []byte{0xff, 0xff}, now); err == nil { + t.Fatal("malformed protobuf accepted") + } +} + +func TestSuccessResponseIsDeterministicAndValid(t *testing.T) { + for _, signal := range []Signal{Logs, Metrics, Traces} { + first, err := SuccessResponse(signal) + if err != nil { + t.Fatal(err) + } + second, _ := SuccessResponse(signal) + if !bytes.Equal(first, second) { + t.Fatalf("%s response is not deterministic", signal) + } + if len(first) != 0 { + t.Fatalf("%s response has non-canonical empty encoding", signal) + } + } +} + +func FuzzDecode(f *testing.F) { + now := time.Date(2026, 8, 17, 3, 0, 0, 0, time.UTC) + seed, err := proto.Marshal(&logspb.LogsData{ResourceLogs: []*logspb.ResourceLogs{{ScopeLogs: []*logspb.ScopeLogs{{LogRecords: []*logspb.LogRecord{{TimeUnixNano: uint64(now.UnixNano()), EventName: "seed"}}}}}}}) + if err != nil { + f.Fatal(err) + } + f.Add(uint8(0), seed) + f.Add(uint8(1), []byte{0xff, 0x00}) + f.Fuzz(func(t *testing.T, selected uint8, body []byte) { + if len(body) > 1<<20 { + return + } + signal := []Signal{Logs, Metrics, Traces}[selected%3] + records, _ := Decode(signal, body, now) + if len(records) > 5_000 { + t.Fatalf("decoded %d records", len(records)) + } + }) +} + +func stringAttribute(key, value string) *commonpb.KeyValue { + return &commonpb.KeyValue{Key: key, Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: value}}} +} + +func marshal(t *testing.T, message proto.Message) []byte { + t.Helper() + body, err := proto.MarshalOptions{Deterministic: true}.Marshal(message) + if err != nil { + t.Fatal(err) + } + return body +} diff --git a/internal/query/match.go b/internal/query/match.go new file mode 100644 index 0000000..7375126 --- /dev/null +++ b/internal/query/match.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package query + +import ( + "math" + "regexp" + "strconv" + "strings" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/schema" +) + +type FieldLookup func(field string) (string, bool) + +// MatchesFilters applies the query's typed filters to a bounded record value +// lookup. Storage projections and the local agent evaluator share this path so +// edge and central comparisons cannot drift silently. +func MatchesFilters(ast AST, registry Registry, lookup FieldLookup) (bool, error) { + for _, filter := range ast.Filters { + field := CanonicalField(filter.Field) + descriptor, _ := ResolveDescriptor(ast.Signal, field, registry) + value, present := lookup(field) + if !present { + return false, nil + } + matched, err := CompareValue(value, filter.Value, filter.Op, descriptor.Type) + if err != nil { + return false, err + } + if !matched { + return false, nil + } + } + return true, nil +} + +func MatchObservation(observation model.Observation, ast AST, registry Registry) (bool, error) { + return MatchesFilters(ast, registry, func(field string) (string, bool) { + switch CanonicalField(field) { + case "timestamp": + return observation.Timestamp.UTC().Format(time.RFC3339Nano), !observation.Timestamp.IsZero() + case "name": + return observation.Name, observation.Name != "" + case "severity": + return observation.Severity, observation.Severity != "" + case "body": + return observation.Body, observation.Body != "" + case "value": + if observation.Value == nil { + return "", false + } + return strconv.FormatFloat(*observation.Value, 'g', -1, 64), true + case "trace_id": + return observation.TraceID, observation.TraceID != "" + case "span_id": + return observation.SpanID, observation.SpanID != "" + case "correlation_id": + return observation.CorrelationID, observation.CorrelationID != "" + default: + value, ok := observation.Attributes[CanonicalField(field)] + return value, ok + } + }) +} + +func CompareValue(left, right, operator string, valueType schema.Type) (bool, error) { + if operator == "=~" { + if valueType != schema.TypeString { + return false, ErrTypeMismatch + } + expression, err := regexp.Compile(right) + if err != nil { + return false, ErrTypeMismatch + } + return expression.MatchString(left), nil + } + var comparison int + switch valueType { + case schema.TypeInteger, schema.TypeFloat, schema.TypeDuration: + rightNumber, rightErr := strconv.ParseFloat(right, 64) + if rightErr != nil || math.IsNaN(rightNumber) || math.IsInf(rightNumber, 0) { + return false, ErrTypeMismatch + } + leftNumber, leftErr := strconv.ParseFloat(left, 64) + if leftErr != nil || math.IsNaN(leftNumber) || math.IsInf(leftNumber, 0) { + return false, nil + } + comparison = compareFloat(leftNumber, rightNumber) + case schema.TypeTime: + rightTime, rightErr := time.Parse(time.RFC3339Nano, right) + if rightErr != nil { + return false, ErrTypeMismatch + } + leftTime, leftErr := time.Parse(time.RFC3339Nano, left) + if leftErr != nil { + return false, nil + } + comparison = leftTime.Compare(rightTime) + case schema.TypeBoolean: + rightBool, rightErr := strconv.ParseBool(right) + if rightErr != nil { + return false, ErrTypeMismatch + } + leftBool, leftErr := strconv.ParseBool(left) + if leftErr != nil { + return false, nil + } + comparison = compareBool(leftBool, rightBool) + default: + comparison = strings.Compare(left, right) + } + switch operator { + case "==": + return comparison == 0, nil + case "!=": + return comparison != 0, nil + case ">": + return comparison > 0, nil + case ">=": + return comparison >= 0, nil + case "<": + return comparison < 0, nil + case "<=": + return comparison <= 0, nil + default: + return false, ErrTypeMismatch + } +} + +func compareFloat(left, right float64) int { + if left < right { + return -1 + } + if left > right { + return 1 + } + return 0 +} + +func compareBool(left, right bool) int { + if left == right { + return 0 + } + if !left { + return -1 + } + return 1 +} diff --git a/internal/query/match_test.go b/internal/query/match_test.go new file mode 100644 index 0000000..a8f2eda --- /dev/null +++ b/internal/query/match_test.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package query + +import ( + "testing" + "time" + + "gamertan.com/observatory/internal/model" +) + +func TestMatchObservationUsesCanonicalTypedFilters(t *testing.T) { + now := time.Date(2026, 8, 18, 23, 55, 0, 0, time.UTC) + observation := model.Observation{Timestamp: now, Name: "http.request", Severity: "error", Attributes: map[string]string{"http.status_code": "503", "http.route": "/failed"}} + ast, err := Parse(`logs | where status >= 500 | where route == "/failed" | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + matched, err := MatchObservation(observation, ast, nil) + if err != nil || !matched { + t.Fatalf("matched=%t err=%v", matched, err) + } + observation.Attributes["http.status_code"] = "200" + matched, err = MatchObservation(observation, ast, nil) + if err != nil || matched { + t.Fatalf("matched=%t err=%v", matched, err) + } + delete(observation.Attributes, "http.status_code") + matched, err = MatchObservation(observation, ast, nil) + if err != nil || matched { + t.Fatalf("missing field matched=%t err=%v", matched, err) + } +} + +func TestMatchObservationRejectsInvalidTypedFilterValue(t *testing.T) { + ast, err := Parse(`logs | where status >= nope | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + observation := model.Observation{Timestamp: time.Now().UTC(), Name: "http.request", Attributes: map[string]string{"http.status_code": "503"}} + if _, err = MatchObservation(observation, ast, nil); err != ErrTypeMismatch { + t.Fatalf("err=%v", err) + } +} diff --git a/internal/query/planner.go b/internal/query/planner.go new file mode 100644 index 0000000..81cecfa --- /dev/null +++ b/internal/query/planner.go @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package query + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/schema" +) + +var ErrSensitivePermissionRequired = errors.New("query requires sensitive-field permission") + +type Scope struct { + OrganizationID string `json:"organization_id"` + ProjectID string `json:"project_id,omitempty"` + EnvironmentID string `json:"environment_id,omitempty"` + ServiceID string `json:"service_id,omitempty"` + Sensitive bool `json:"sensitive"` +} + +type Budget struct { + MaxDuration time.Duration `json:"-"` + MaxRows int `json:"max_rows"` + MaxScannedBytes int64 `json:"max_scanned_bytes"` + MaxMemoryBytes int64 `json:"max_memory_bytes"` +} + +type FieldPlan struct { + Field string `json:"field"` + Descriptor schema.Descriptor `json:"descriptor"` + Indexed bool `json:"indexed"` + Unknown bool `json:"unknown"` +} + +type Explain struct { + AST AST `json:"ast"` + ProjectedSources []string `json:"projected_sources"` + Fields []FieldPlan `json:"fields"` + EstimatedScanBytes int64 `json:"estimated_scan_bytes"` + CacheEligible bool `json:"cache_eligible"` + RequiredPermissions []string `json:"required_permissions"` + Budget Budget `json:"budget"` +} + +type Registry interface { + Lookup(model.Signal, string) (schema.Descriptor, bool) +} + +type MapRegistry map[string]schema.Descriptor + +func (registry MapRegistry) Lookup(signal model.Signal, field string) (schema.Descriptor, bool) { + descriptor, ok := registry[string(signal)+":"+CanonicalField(field)] + return descriptor, ok +} + +func Plan(ast AST, scope Scope, registry Registry, estimatedScanBytes int64, budget Budget) (Explain, error) { + if err := Validate(ast, budget.MaxRows); err != nil { + return Explain{}, err + } + if !safeScope(scope) { + return Explain{}, errors.New("query scope is invalid") + } + if budget.MaxDuration < time.Millisecond || budget.MaxDuration > time.Minute || budget.MaxRows < 1 || budget.MaxScannedBytes < 1 || budget.MaxMemoryBytes < 1 { + return Explain{}, errors.New("query budget is invalid") + } + if estimatedScanBytes < 0 || estimatedScanBytes > budget.MaxScannedBytes { + return Explain{}, errors.New("estimated query scan exceeds budget") + } + fields := ReferencedFields(ast) + plans := make([]FieldPlan, 0, len(fields)) + requiresSensitive := false + cacheEligible := true + for _, field := range fields { + canonical := CanonicalField(field) + descriptor, unknown := ResolveDescriptor(ast.Signal, canonical, registry) + if err := descriptor.Validate(); err != nil { + return Explain{}, fmt.Errorf("field %s descriptor: %w", field, err) + } + if descriptor.Sensitivity == schema.SensitivitySensitive { + requiresSensitive = true + cacheEligible = false + } + plans = append(plans, FieldPlan{Field: field, Descriptor: descriptor, Indexed: descriptor.Index != schema.IndexNone, Unknown: unknown}) + } + if requiresSensitive && !scope.Sensitive { + return Explain{}, ErrSensitivePermissionRequired + } + for _, filter := range ast.Filters { + if filter.Op == "=~" { + cacheEligible = false + } + } + permissions := []string{"telemetry:query"} + if requiresSensitive { + permissions = append(permissions, "telemetry:sensitive") + } + source := "organization:" + scope.OrganizationID + "/signal:" + string(ast.Signal) + if scope.ProjectID != "" { + source += "/project:" + scope.ProjectID + } + if scope.EnvironmentID != "" { + source += "/environment:" + scope.EnvironmentID + } + if scope.ServiceID != "" { + source += "/service:" + scope.ServiceID + } + return Explain{AST: ast, ProjectedSources: []string{source}, Fields: plans, EstimatedScanBytes: estimatedScanBytes, CacheEligible: cacheEligible, RequiredPermissions: permissions, Budget: budget}, nil +} + +func ReferencedFields(ast AST) []string { + seen := map[string]bool{} + var fields []string + add := func(field string) { + if field != "" && !seen[field] { + seen[field] = true + fields = append(fields, field) + } + } + for _, filter := range ast.Filters { + add(filter.Field) + } + if ast.Sort != nil { + isAggregateAlias := false + if ast.Summary != nil { + for _, aggregate := range ast.Summary.Aggregates { + if aggregate.Alias == ast.Sort.Field { + isAggregateAlias = true + break + } + } + } + if !isAggregateAlias { + add(ast.Sort.Field) + } + } + if ast.Summary != nil { + for _, aggregate := range ast.Summary.Aggregates { + add(aggregate.Field) + } + for _, field := range ast.Summary.GroupBy { + add(field) + } + } + sort.Strings(fields) + return fields +} + +func CanonicalField(field string) string { + switch field { + case "service": + return "service.id" + case "project": + return "project.id" + case "environment": + return "environment.id" + case "route": + return "http.route" + case "status": + return "http.status_code" + case "duration": + return "duration_ns" + default: + return field + } +} + +func ResolveDescriptor(signal model.Signal, field string, registry Registry) (schema.Descriptor, bool) { + canonical := CanonicalField(field) + descriptor, ok := BuiltinDescriptor(signal, canonical) + if !ok && registry != nil { + descriptor, ok = registry.Lookup(signal, canonical) + } + if !ok { + return schema.Unknown(signal, canonical), true + } + return descriptor, false +} + +func BuiltinDescriptor(signal model.Signal, field string) (schema.Descriptor, bool) { + descriptors := map[string]schema.Descriptor{ + "service.id": descriptor(signal, "service.id", schema.TypeString, "Application service identifier.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""), + "project.id": descriptor(signal, "project.id", schema.TypeString, "Application project identifier.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""), + "environment.id": descriptor(signal, "environment.id", schema.TypeString, "Deployment environment identifier.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""), + "timestamp": descriptor(signal, "timestamp", schema.TypeTime, "Observation timestamp.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, "s"), + "name": descriptor(signal, "name", schema.TypeString, "Observation name.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""), + "severity": descriptor(signal, "severity", schema.TypeString, "Log severity.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""), + "value": descriptor(signal, "value", schema.TypeFloat, "Numeric observation value.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, ""), + "http.route": descriptor(signal, "http.route", schema.TypeString, "Application-normalized HTTP route.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""), + "http.status_code": descriptor(signal, "http.status_code", schema.TypeInteger, "HTTP response status code.", schema.SensitivityPublic, schema.CardinalityLow, schema.IndexExact, ""), + "duration_ns": descriptor(signal, "duration_ns", schema.TypeDuration, "Observed duration in nanoseconds.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, "ns"), + "state": descriptor(signal, "state", schema.TypeString, "Bounded metric state dimension.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""), + "period": descriptor(signal, "period", schema.TypeString, "Bounded measurement period dimension.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""), + "interface": descriptor(signal, "interface", schema.TypeString, "Configured network interface dimension.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""), + "direction": descriptor(signal, "direction", schema.TypeString, "Bounded input or output direction dimension.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""), + "filesystem": descriptor(signal, "filesystem", schema.TypeString, "Configured filesystem dimension.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""), + "process": descriptor(signal, "process", schema.TypeString, "Configured process dimension.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""), + "cgroup": descriptor(signal, "cgroup", schema.TypeString, "Configured cgroup dimension.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""), + "unit": descriptor(signal, "unit", schema.TypeString, "Metric unit supplied by a bounded collector.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""), + "trace_id": descriptor(signal, "trace_id", schema.TypeString, "Trace correlation identifier.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, ""), + "span_id": descriptor(signal, "span_id", schema.TypeString, "Span correlation identifier.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, ""), + "correlation_id": descriptor(signal, "correlation_id", schema.TypeString, "Cross-signal correlation identifier.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, ""), + "body": descriptor(signal, "body", schema.TypeString, "Optional retained log text.", schema.SensitivitySensitive, schema.CardinalityHigh, schema.IndexNone, ""), + } + descriptor, ok := descriptors[field] + if ok && signal != model.SignalMetrics { + switch field { + case "state", "period", "interface", "direction", "filesystem", "process", "cgroup", "unit": + return schema.Descriptor{}, false + } + } + if ok && signal == model.SignalMetrics { + switch field { + case "http.route", "http.status_code", "state", "period", "interface", "direction", "filesystem", "process", "cgroup", "unit": + descriptor.Retention = schema.RetentionMetric + } + } + return descriptor, ok +} + +func descriptor(signal model.Signal, field string, valueType schema.Type, meaning string, sensitivity schema.Sensitivity, cardinality schema.Cardinality, index schema.IndexPolicy, unit string) schema.Descriptor { + return schema.Descriptor{Version: schema.DescriptorVersion, Signal: signal, Field: field, Type: valueType, Unit: unit, Meaning: meaning, Sensitivity: sensitivity, Cardinality: cardinality, Index: index, Retention: schema.RetentionRaw, ProjectionVersion: 1} +} + +func safeScope(scope Scope) bool { + values := []string{scope.OrganizationID, scope.ProjectID, scope.EnvironmentID, scope.ServiceID} + if values[0] == "" { + return false + } + for _, value := range values { + if value == "" { + continue + } + if len(value) > 128 { + return false + } + for _, r := range value { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("._-", r)) { + return false + } + } + } + return true +} diff --git a/internal/query/planner_test.go b/internal/query/planner_test.go new file mode 100644 index 0000000..9bb24fe --- /dev/null +++ b/internal/query/planner_test.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package query + +import ( + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/schema" +) + +func TestPlanInjectsScopeAndReportsIndexes(t *testing.T) { + ast, err := Parse(`logs | where service == "eql" | where status >= 500 | window 24h | sort timestamp desc | limit 50`, 1000) + if err != nil { + t.Fatal(err) + } + explain, err := Plan(ast, Scope{OrganizationID: "personal-cole", ProjectID: "eql", EnvironmentID: "production"}, nil, 10<<20, Budget{MaxDuration: 5 * time.Second, MaxRows: 1000, MaxScannedBytes: 100 << 20, MaxMemoryBytes: 64 << 20}) + if err != nil { + t.Fatal(err) + } + if len(explain.ProjectedSources) != 1 || explain.ProjectedSources[0] != "organization:personal-cole/signal:logs/project:eql/environment:production" || len(explain.Fields) != 3 { + t.Fatalf("explain=%+v", explain) + } + for _, field := range explain.Fields { + if !field.Indexed || field.Unknown { + t.Fatalf("field=%+v", field) + } + } +} + +func TestPlanRequiresSensitivePermissionForUnknownAndBody(t *testing.T) { + for _, text := range []string{`logs | where vendor.unknown == "x" | limit 10`, `logs | where body =~ "error" | limit 10`} { + ast, err := Parse(text, 100) + if err != nil { + t.Fatal(err) + } + budget := Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20} + if _, err := Plan(ast, Scope{OrganizationID: "org"}, nil, 100, budget); err == nil { + t.Fatalf("expected sensitive rejection for %q", text) + } + explain, err := Plan(ast, Scope{OrganizationID: "org", Sensitive: true}, nil, 100, budget) + if err != nil { + t.Fatal(err) + } + if len(explain.RequiredPermissions) != 2 || explain.CacheEligible { + t.Fatalf("explain=%+v", explain) + } + } +} + +func TestPlanRejectsCrossTenantScopeAndScanBudget(t *testing.T) { + ast, _ := Parse(`metrics | limit 10`, 100) + budget := Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1000, MaxMemoryBytes: 1000} + if _, err := Plan(ast, Scope{OrganizationID: "../other"}, nil, 10, budget); err == nil { + t.Fatal("expected scope rejection") + } + if _, err := Plan(ast, Scope{OrganizationID: "org"}, nil, 1001, budget); err == nil { + t.Fatal("expected scan-budget rejection") + } +} + +func TestBuiltinMetricDimensionsAreSignalScopedAndRetentionAware(t *testing.T) { + state, ok := BuiltinDescriptor(model.SignalMetrics, "state") + if !ok || state.Retention != schema.RetentionMetric { + t.Fatalf("metric state descriptor=%+v ok=%t", state, ok) + } + if _, ok = BuiltinDescriptor(model.SignalLogs, "state"); ok { + t.Fatal("metric-only state dimension was exposed to logs") + } + + metricRoute, ok := BuiltinDescriptor(model.SignalMetrics, "http.route") + if !ok || metricRoute.Retention != schema.RetentionMetric { + t.Fatalf("metric route descriptor=%+v ok=%t", metricRoute, ok) + } + logRoute, ok := BuiltinDescriptor(model.SignalLogs, "http.route") + if !ok || logRoute.Retention != schema.RetentionRaw { + t.Fatalf("log route descriptor=%+v ok=%t", logRoute, ok) + } +} diff --git a/internal/query/query.go b/internal/query/query.go new file mode 100644 index 0000000..87c9f63 --- /dev/null +++ b/internal/query/query.go @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package query + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "gamertan.com/observatory/internal/model" +) + +const ( + ASTVersion = 1 + maxQueryWindow = 3650 * 24 * time.Hour +) + +type AST struct { + Version int `json:"version"` + Signal model.Signal `json:"signal"` + Filters []Filter `json:"filters,omitempty"` + Sort *Sort `json:"sort,omitempty"` + Summary *Summary `json:"summary,omitempty"` + Limit int `json:"limit"` + Window time.Duration `json:"-"` + WindowText string `json:"window,omitempty"` + Bucket time.Duration `json:"-"` + BucketText string `json:"bucket,omitempty"` +} + +type Filter struct { + Field string `json:"field"` + Op string `json:"op"` + Value string `json:"value"` +} + +type Sort struct { + Field string `json:"field"` + Descending bool `json:"descending"` +} + +type Summary struct { + Aggregates []Aggregate `json:"aggregates"` + GroupBy []string `json:"group_by,omitempty"` +} + +type Aggregate struct { + Function string `json:"function"` + Field string `json:"field,omitempty"` + Alias string `json:"alias"` +} + +func Parse(text string, maxLimit int) (AST, error) { + if len(text) == 0 || len(text) > 16_384 { + return AST{}, errors.New("query length outside accepted bounds") + } + parts := strings.Split(text, "|") + if len(parts) > 16 { + return AST{}, errors.New("too many query stages") + } + ast := AST{Version: ASTVersion, Signal: model.Signal(strings.TrimSpace(parts[0])), Limit: min(100, maxLimit)} + if ast.Signal != model.SignalLogs && ast.Signal != model.SignalMetrics && ast.Signal != model.SignalTraces && ast.Signal != model.SignalDeployments { + return AST{}, errors.New("query must begin with logs, metrics, traces, or deployments") + } + for _, raw := range parts[1:] { + stage := strings.TrimSpace(raw) + switch { + case strings.HasPrefix(stage, "where "): + filter, err := parseFilter(strings.TrimSpace(strings.TrimPrefix(stage, "where "))) + if err != nil { + return AST{}, err + } + ast.Filters = append(ast.Filters, filter) + case strings.HasPrefix(stage, "sort "): + fields := strings.Fields(strings.TrimSpace(strings.TrimPrefix(stage, "sort "))) + if len(fields) < 1 || len(fields) > 2 || !validField(fields[0]) { + return AST{}, errors.New("invalid sort stage") + } + desc := len(fields) == 2 && fields[1] == "desc" + if len(fields) == 2 && fields[1] != "asc" && fields[1] != "desc" { + return AST{}, errors.New("sort direction must be asc or desc") + } + ast.Sort = &Sort{Field: fields[0], Descending: desc} + case strings.HasPrefix(stage, "summarize "): + if ast.Summary != nil { + return AST{}, errors.New("query may contain only one summarize stage") + } + summary, bucket, bucketText, err := parseSummary(strings.TrimSpace(strings.TrimPrefix(stage, "summarize "))) + if err != nil { + return AST{}, err + } + ast.Summary = &summary + if bucket > 0 { + ast.Bucket, ast.BucketText = bucket, bucketText + } + case strings.HasPrefix(stage, "limit "): + n, err := strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(stage, "limit "))) + if err != nil || n < 1 || n > maxLimit { + return AST{}, fmt.Errorf("limit must be between 1 and %d", maxLimit) + } + ast.Limit = n + case strings.HasPrefix(stage, "window "): + windowText := strings.TrimSpace(strings.TrimPrefix(stage, "window ")) + d, err := time.ParseDuration(windowText) + if err != nil || d < time.Second || d > maxQueryWindow { + return AST{}, errors.New("window must be between 1s and 87600h") + } + ast.Window, ast.WindowText = d, windowText + default: + return AST{}, fmt.Errorf("unsupported query stage %q", stage) + } + } + if err := Validate(ast, maxLimit); err != nil { + return AST{}, err + } + return ast, nil +} + +func Validate(ast AST, maxLimit int) error { + if ast.Version != ASTVersion || (ast.Signal != model.SignalLogs && ast.Signal != model.SignalMetrics && ast.Signal != model.SignalTraces && ast.Signal != model.SignalDeployments) { + return errors.New("query AST identity is invalid") + } + if ast.Limit < 1 || ast.Limit > maxLimit { + return fmt.Errorf("limit must be between 1 and %d", maxLimit) + } + if ast.Window < 0 || ast.Window > maxQueryWindow || ast.Window > 0 && ast.Window < time.Second { + return errors.New("query window is invalid") + } + if ast.Bucket < 0 || ast.Bucket > maxQueryWindow || ast.Bucket > 0 && (ast.Bucket < time.Second || ast.Summary == nil) { + return errors.New("query summary bucket is invalid") + } + if len(ast.Filters) > 16 { + return errors.New("too many query filters") + } + for _, filter := range ast.Filters { + if !validField(filter.Field) || len(filter.Value) > 4096 { + return errors.New("query filter is invalid") + } + switch filter.Op { + case "!=", ">=", "<=", "==", ">", "<": + case "=~": + if len(filter.Value) > 512 { + return errors.New("regular expression exceeds 512 bytes") + } + if _, err := regexp.Compile(filter.Value); err != nil { + return errors.New("invalid regular expression") + } + default: + return errors.New("query filter operator is invalid") + } + } + if ast.Sort != nil && !validField(ast.Sort.Field) { + return errors.New("query sort is invalid") + } + if ast.Summary != nil { + if len(ast.Summary.Aggregates) < 1 || len(ast.Summary.Aggregates) > 16 || len(ast.Summary.GroupBy) > 16 { + return errors.New("query summary is invalid") + } + aliases := map[string]bool{} + for _, aggregate := range ast.Summary.Aggregates { + switch aggregate.Function { + case "count": + if aggregate.Field != "" { + return errors.New("count accepts no field") + } + case "min", "max", "sum", "avg", "p50", "p95", "p99": + if !validField(aggregate.Field) { + return errors.New("aggregate field is invalid") + } + default: + return errors.New("aggregate function is unsupported") + } + if !validField(aggregate.Alias) || aliases[aggregate.Alias] { + return errors.New("aggregate alias is invalid or duplicated") + } + aliases[aggregate.Alias] = true + } + for _, group := range ast.Summary.GroupBy { + if !validField(group) { + return errors.New("grouping field is invalid") + } + if aliases[group] { + return errors.New("grouping field conflicts with aggregate alias") + } + } + } + return nil +} + +func parseFilter(expr string) (Filter, error) { + for _, op := range []string{"!=", ">=", "<=", "==", "=~", ">", "<"} { + if i := strings.Index(expr, op); i > 0 { + field := strings.TrimSpace(expr[:i]) + value := strings.TrimSpace(expr[i+len(op):]) + if !validField(field) || value == "" || len(value) > 4096 { + return Filter{}, errors.New("invalid where stage") + } + if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' { + unquoted, err := strconv.Unquote(value) + if err != nil { + return Filter{}, errors.New("invalid quoted filter value") + } + value = unquoted + } + if op == "=~" { + if len(value) > 512 { + return Filter{}, errors.New("regular expression exceeds 512 bytes") + } + if _, err := regexp.Compile(value); err != nil { + return Filter{}, errors.New("invalid regular expression") + } + } + return Filter{Field: field, Op: op, Value: value}, nil + } + } + return Filter{}, errors.New("where stage requires a comparison") +} + +func parseSummary(stage string) (Summary, time.Duration, string, error) { + aggregateText, groupText, found := strings.Cut(stage, " by ") + aggregateParts := strings.Split(aggregateText, ",") + if len(aggregateParts) < 1 || len(aggregateParts) > 16 { + return Summary{}, 0, "", errors.New("summarize requires between 1 and 16 aggregates") + } + summary := Summary{} + aliases := map[string]bool{} + for _, raw := range aggregateParts { + expression := strings.TrimSpace(raw) + open := strings.IndexByte(expression, '(') + if open < 1 || !strings.HasSuffix(expression, ")") { + return Summary{}, 0, "", errors.New("invalid aggregate") + } + function := expression[:open] + field := strings.TrimSpace(expression[open+1 : len(expression)-1]) + switch function { + case "count": + if field != "" { + return Summary{}, 0, "", errors.New("count accepts no field") + } + case "min", "max", "sum", "avg", "p50", "p95", "p99": + if !validField(field) { + return Summary{}, 0, "", errors.New("aggregate field is invalid") + } + default: + return Summary{}, 0, "", errors.New("aggregate function is unsupported") + } + alias := function + if field != "" { + alias += "_" + strings.ReplaceAll(field, ".", "_") + } + if aliases[alias] { + return Summary{}, 0, "", errors.New("aggregate alias is duplicated") + } + aliases[alias] = true + summary.Aggregates = append(summary.Aggregates, Aggregate{Function: function, Field: field, Alias: alias}) + } + var window time.Duration + var windowText string + if found { + groups := strings.Split(groupText, ",") + if len(groups) > 16 { + return Summary{}, 0, "", errors.New("too many grouping fields") + } + for _, raw := range groups { + group := strings.TrimSpace(raw) + if strings.HasPrefix(group, "window(") && strings.HasSuffix(group, ")") { + if window > 0 { + return Summary{}, 0, "", errors.New("summary window is duplicated") + } + windowText = strings.TrimSpace(group[len("window(") : len(group)-1]) + parsed, err := time.ParseDuration(windowText) + if err != nil || parsed < time.Second || parsed > maxQueryWindow { + return Summary{}, 0, "", errors.New("summary window must be between 1s and 87600h") + } + window = parsed + continue + } + if !validField(group) { + return Summary{}, 0, "", errors.New("grouping field is invalid") + } + summary.GroupBy = append(summary.GroupBy, group) + } + } + return summary, window, windowText, nil +} + +func validField(field string) bool { + if field == "" || len(field) > 128 { + return false + } + for _, r := range field { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '.') { + return false + } + } + return true +} diff --git a/internal/query/query_test.go b/internal/query/query_test.go new file mode 100644 index 0000000..8ddacee --- /dev/null +++ b/internal/query/query_test.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package query + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestParseBoundedQuery(t *testing.T) { + ast, err := Parse(`logs | where service == "eql" | where status >= 500 | window 24h | sort timestamp desc | limit 50`, 1000) + if err != nil { + t.Fatal(err) + } + if ast.Limit != 50 || len(ast.Filters) != 2 || ast.WindowText != "24h" || ast.Sort == nil || !ast.Sort.Descending { + t.Fatalf("unexpected AST: %#v", ast) + } +} + +func TestTextAndVisualBuilderShareValidatedAST(t *testing.T) { + fromText, err := Parse(`metrics | where service == "eql" | window 1h | limit 25`, 1000) + if err != nil { + t.Fatal(err) + } + b, err := json.Marshal(fromText) + if err != nil { + t.Fatal(err) + } + var fromBuilder AST + if err := json.Unmarshal(b, &fromBuilder); err != nil { + t.Fatal(err) + } + fromBuilder.Window = fromText.Window + fromBuilder.Bucket = fromText.Bucket + if err := Validate(fromBuilder, 1000); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(fromText, fromBuilder) { + t.Fatalf("text=%+v builder=%+v", fromText, fromBuilder) + } + fromBuilder.Filters[0].Op = "SQL" + if err := Validate(fromBuilder, 1000); err == nil { + t.Fatal("expected hostile builder AST rejection") + } +} + +func TestParseRejectsSQLAndUnboundedLimit(t *testing.T) { + for _, input := range []string{"select * from logs", "logs | limit 1001", "logs | where route;drop == x"} { + if _, err := Parse(input, 1000); err == nil { + t.Fatalf("expected rejection for %q", input) + } + } +} + +func TestParseUnifiedSummaryAndSafeRegex(t *testing.T) { + ast, err := Parse(`logs | where route =~ "^/items/[0-9]+$" | summarize count(), p95(duration) by route, window(5m) | sort count desc | limit 50`, 1000) + if err != nil { + t.Fatal(err) + } + if ast.Summary == nil || len(ast.Summary.Aggregates) != 2 || len(ast.Summary.GroupBy) != 1 || ast.WindowText != "" || ast.BucketText != "5m" || ast.Summary.Aggregates[1].Alias != "p95_duration" { + t.Fatalf("ast=%+v", ast) + } + if _, err := Parse(`logs | where route =~ "["`, 1000); err == nil { + t.Fatal("expected invalid regular expression rejection") + } +} + +func TestParseSupportsApprovedTenYearColdLookback(t *testing.T) { + ast, err := Parse(`logs | window 87600h | limit 10`, 100) + if err != nil || ast.Window != maxQueryWindow { + t.Fatalf("ast=%+v err=%v", ast, err) + } + if _, err = Parse(`logs | window 87601h | limit 10`, 100); err == nil { + t.Fatal("lookback beyond retention ceiling was accepted") + } +} + +func FuzzParse(f *testing.F) { + for _, seed := range []string{ + `logs | where service == "eql" | limit 50`, + `metrics | summarize count(), p95(duration) by route, window(5m) | limit 50`, + `traces | where trace_id =~ "^[0-9a-f]{32}$" | window 1h | limit 10`, + `select * from logs`, + string([]byte{0, 1, 2, 3}), + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, text string) { + if len(text) > 20_000 { + return + } + ast, err := Parse(text, 1_000) + if err != nil { + return + } + if err = Validate(ast, 1_000); err != nil { + t.Fatalf("parser returned an invalid AST: %v", err) + } + if ast.Limit < 1 || ast.Limit > 1_000 || len(ast.Filters) > 16 { + t.Fatalf("accepted AST violates hard bounds: %+v", ast) + } + }) +} diff --git a/internal/query/result.go b/internal/query/result.go new file mode 100644 index 0000000..233588e --- /dev/null +++ b/internal/query/result.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package query + +import ( + "errors" + + "gamertan.com/observatory/internal/schema" +) + +const ResultVersion = 1 + +var ( + ErrBudgetExceeded = errors.New("query execution budget exceeded") + ErrTypeMismatch = errors.New("query value does not match its field type") +) + +type Column struct { + Field string `json:"field"` + Type schema.Type `json:"type"` + Unit string `json:"unit,omitempty"` +} + +// Row values align positionally with Result.Columns. Nil is a missing value; +// non-nil values use the canonical string form described by the column type. +type Row struct { + Values []*string `json:"values"` +} + +type Statistics struct { + ScannedRows int `json:"scanned_rows"` + MatchedRows int `json:"matched_rows"` + ScannedBytes int64 `json:"scanned_bytes"` + DurationNS int64 `json:"duration_ns"` + Truncated bool `json:"truncated"` + Approximate bool `json:"approximate,omitempty"` +} + +type Result struct { + Version int `json:"version"` + Explain Explain `json:"explain"` + Columns []Column `json:"columns"` + Rows []Row `json:"rows"` + Stats Statistics `json:"statistics"` +} diff --git a/internal/schema/schema.go b/internal/schema/schema.go new file mode 100644 index 0000000..d0886f6 --- /dev/null +++ b/internal/schema/schema.go @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package schema + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "gamertan.com/observatory/internal/model" +) + +const DescriptorVersion = 1 + +type Type string +type Sensitivity string +type Cardinality string +type IndexPolicy string +type RetentionClass string + +const ( + TypeString Type = "string" + TypeInteger Type = "integer" + TypeFloat Type = "float" + TypeBoolean Type = "boolean" + TypeDuration Type = "duration" + TypeTime Type = "time" + + SensitivityPublic Sensitivity = "public" + SensitivityInternal Sensitivity = "internal" + SensitivitySensitive Sensitivity = "sensitive" + + CardinalityLow Cardinality = "low" + CardinalityMedium Cardinality = "medium" + CardinalityHigh Cardinality = "high" + + IndexNone IndexPolicy = "none" + IndexExact IndexPolicy = "exact" + IndexRange IndexPolicy = "range" + + RetentionRaw RetentionClass = "raw" + RetentionMetric RetentionClass = "metric" + RetentionEvidence RetentionClass = "evidence" +) + +var fieldPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.]{0,127}$`) +var unitPattern = regexp.MustCompile(`^[A-Za-z0-9%/._-]{0,64}$`) + +type Descriptor struct { + Version int `json:"version"` + Signal model.Signal `json:"signal"` + Field string `json:"field"` + Type Type `json:"type"` + Unit string `json:"unit,omitempty"` + Meaning string `json:"meaning"` + Sensitivity Sensitivity `json:"sensitivity"` + Cardinality Cardinality `json:"cardinality"` + Index IndexPolicy `json:"index"` + Retention RetentionClass `json:"retention"` + ProjectionVersion int `json:"projection_version"` +} + +type Proposal struct { + Descriptor Descriptor `json:"descriptor"` + ObservedValues int64 `json:"observed_values"` + EstimatedBytes int64 `json:"estimated_bytes"` + ExampleQueries []string `json:"example_queries"` +} + +func (d Descriptor) Validate() error { + if d.Version != DescriptorVersion || !validSignal(d.Signal) || !fieldPattern.MatchString(d.Field) || !unitPattern.MatchString(d.Unit) { + return errors.New("descriptor identity is invalid") + } + switch d.Type { + case TypeString, TypeInteger, TypeFloat, TypeBoolean, TypeDuration, TypeTime: + default: + return errors.New("descriptor type is invalid") + } + if len(d.Meaning) < 1 || len(d.Meaning) > 512 || strings.IndexByte(d.Meaning, 0) >= 0 { + return errors.New("descriptor meaning is invalid") + } + switch d.Sensitivity { + case SensitivityPublic, SensitivityInternal, SensitivitySensitive: + default: + return errors.New("descriptor sensitivity is invalid") + } + switch d.Cardinality { + case CardinalityLow, CardinalityMedium, CardinalityHigh: + default: + return errors.New("descriptor cardinality is invalid") + } + switch d.Index { + case IndexNone, IndexExact, IndexRange: + default: + return errors.New("descriptor index policy is invalid") + } + if d.Cardinality == CardinalityHigh && d.Index == IndexExact { + return errors.New("high-cardinality exact indexes require a reviewed exception") + } + switch d.Retention { + case RetentionRaw, RetentionMetric, RetentionEvidence: + default: + return errors.New("descriptor retention class is invalid") + } + if d.ProjectionVersion < 1 { + return errors.New("projection version must be positive") + } + return nil +} + +func (p Proposal) Validate() error { + if err := p.Descriptor.Validate(); err != nil { + return err + } + if p.ObservedValues < 1 || p.EstimatedBytes < 0 || len(p.ExampleQueries) > 16 { + return errors.New("proposal evidence is invalid") + } + for _, example := range p.ExampleQueries { + if len(example) < 1 || len(example) > 4096 || strings.IndexByte(example, 0) >= 0 { + return fmt.Errorf("proposal example query is invalid") + } + } + return nil +} + +func Unknown(signal model.Signal, field string) Descriptor { + return Descriptor{Version: DescriptorVersion, Signal: signal, Field: field, Type: TypeString, Meaning: "Unreviewed field retained in raw evidence.", Sensitivity: SensitivitySensitive, Cardinality: CardinalityHigh, Index: IndexNone, Retention: RetentionRaw, ProjectionVersion: 1} +} + +func validSignal(signal model.Signal) bool { + switch signal { + case model.SignalLogs, model.SignalMetrics, model.SignalTraces, model.SignalDeployments: + return true + default: + return false + } +} diff --git a/internal/schema/schema_test.go b/internal/schema/schema_test.go new file mode 100644 index 0000000..7306624 --- /dev/null +++ b/internal/schema/schema_test.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package schema + +import ( + "testing" + + "gamertan.com/observatory/internal/model" +) + +func TestUnknownFieldsAreSensitiveAndUnindexed(t *testing.T) { + descriptor := Unknown(model.SignalLogs, "vendor.unreviewed") + if err := descriptor.Validate(); err != nil { + t.Fatal(err) + } + if descriptor.Sensitivity != SensitivitySensitive || descriptor.Index != IndexNone || descriptor.Cardinality != CardinalityHigh { + t.Fatalf("descriptor=%+v", descriptor) + } +} + +func TestDescriptorRejectsHighCardinalityExactIndex(t *testing.T) { + descriptor := Descriptor{Version: 1, Signal: model.SignalLogs, Field: "request.id", Type: TypeString, Meaning: "A request correlation identifier.", Sensitivity: SensitivityInternal, Cardinality: CardinalityHigh, Index: IndexExact, Retention: RetentionRaw, ProjectionVersion: 1} + if err := descriptor.Validate(); err == nil { + t.Fatal("expected unsafe index rejection") + } +} diff --git a/internal/segment/store.go b/internal/segment/store.go new file mode 100644 index 0000000..0dc65e3 --- /dev/null +++ b/internal/segment/store.go @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package segment + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "gamertan.com/observatory/internal/model" + "github.com/klauspost/compress/zstd" +) + +const ( + MaxDecodedSegment = 64 << 20 + MaxEncodedSegment = MaxDecodedSegment + 1<<20 + metadataReadBatch = 128 +) + +type Store struct { + root string +} + +type Committed struct { + Path string + Digest string + Compressed int64 + Uncompressed int64 +} + +type Entry struct { + OrganizationID string + Committed Committed + Batch model.Batch +} + +// Metadata identifies one immutable raw object without reading, checksumming, +// decompressing, or decoding its contents. It is safe to retain while walking +// a large store because it contains no telemetry records. +type Metadata struct { + OrganizationID string + SourceID string + StreamID string + Sequence uint64 + Path string + Digest string + Compressed int64 +} + +func New(root string) (*Store, error) { + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return nil, errors.New("segment root must be an absolute clean path") + } + if err := os.MkdirAll(root, 0o700); err != nil { + return nil, fmt.Errorf("create segment root: %w", err) + } + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return nil, errors.New("segment root must be a private non-symlink directory") + } + return &Store{root: root}, nil +} + +func (s *Store) Commit(scope model.Scope, batch model.Batch) (Committed, error) { + if err := scope.Validate(); err != nil { + return Committed{}, err + } + raw, err := json.Marshal(batch) + if err != nil { + return Committed{}, fmt.Errorf("encode batch: %w", err) + } + if len(raw) > MaxDecodedSegment { + return Committed{}, errors.New("decoded segment exceeds limit") + } + enc, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1), zstd.WithEncoderLevel(zstd.SpeedBetterCompression)) + if err != nil { + return Committed{}, fmt.Errorf("create compressor: %w", err) + } + compressed := enc.EncodeAll(raw, nil) + enc.Close() + if len(compressed) > MaxEncodedSegment { + return Committed{}, errors.New("encoded segment exceeds limit") + } + sum := sha256.Sum256(compressed) + digest := hex.EncodeToString(sum[:]) + dir := filepath.Join(s.root, "raw", scope.OrganizationID, batch.SourceID, batch.StreamID) + if err := ensurePrivateDirectoryChain(s.root, dir); err != nil { + return Committed{}, err + } + name := fmt.Sprintf("%020d-%s.zst", batch.Sequence, digest) + final := filepath.Join(dir, name) + if existing, err := readRegular(final, MaxEncodedSegment); err == nil { + if bytes.Equal(existing, compressed) { + return Committed{Path: final, Digest: digest, Compressed: int64(len(compressed)), Uncompressed: int64(len(raw))}, nil + } + return Committed{}, errors.New("existing segment digest collision") + } else if !errors.Is(err, os.ErrNotExist) { + return Committed{}, fmt.Errorf("inspect existing segment: %w", err) + } + tmp, err := os.CreateTemp(dir, ".segment-*") + if err != nil { + return Committed{}, fmt.Errorf("create segment temporary file: %w", err) + } + tmpName := tmp.Name() + cleanup := func() { _ = os.Remove(tmpName) } + defer cleanup() + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return Committed{}, fmt.Errorf("set segment mode: %w", err) + } + if _, err := tmp.Write(compressed); err != nil { + _ = tmp.Close() + return Committed{}, fmt.Errorf("write segment: %w", err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return Committed{}, fmt.Errorf("sync segment: %w", err) + } + if err := tmp.Close(); err != nil { + return Committed{}, fmt.Errorf("close segment: %w", err) + } + if err := os.Rename(tmpName, final); err != nil { + return Committed{}, fmt.Errorf("commit segment: %w", err) + } + d, err := os.Open(dir) + if err != nil { + return Committed{}, fmt.Errorf("open segment directory: %w", err) + } + if err := d.Sync(); err != nil { + _ = d.Close() + return Committed{}, fmt.Errorf("sync segment directory: %w", err) + } + if err := d.Close(); err != nil { + return Committed{}, fmt.Errorf("close segment directory: %w", err) + } + return Committed{Path: final, Digest: digest, Compressed: int64(len(compressed)), Uncompressed: int64(len(raw))}, nil +} + +func (s *Store) Read(path, expectedDigest string) (model.Batch, error) { + var batch model.Batch + cleanRoot := filepath.Clean(s.root) + string(os.PathSeparator) + cleanPath := filepath.Clean(path) + if !strings.HasPrefix(cleanPath, cleanRoot) { + return batch, errors.New("segment path escapes store") + } + if err := validatePrivateDirectoryChain(s.root, filepath.Dir(cleanPath)); err != nil { + return batch, err + } + b, err := readRegular(cleanPath, MaxEncodedSegment) + if err != nil { + return batch, fmt.Errorf("read segment: %w", err) + } + sum := sha256.Sum256(b) + if hex.EncodeToString(sum[:]) != expectedDigest { + return batch, errors.New("segment checksum mismatch") + } + dec, err := zstd.NewReader(bytes.NewReader(b), zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(MaxDecodedSegment)) + if err != nil { + return batch, fmt.Errorf("create decompressor: %w", err) + } + decompressed, err := io.ReadAll(io.LimitReader(dec, MaxDecodedSegment+1)) + dec.Close() + if err != nil { + return batch, fmt.Errorf("decompress segment: %w", err) + } + if len(decompressed) > MaxDecodedSegment { + return batch, errors.New("decoded segment exceeds limit") + } + decoder := json.NewDecoder(bytes.NewReader(decompressed)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&batch); err != nil { + return batch, fmt.Errorf("decode segment: %w", err) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return batch, errors.New("segment contains trailing JSON") + } + return batch, nil +} + +// ReadEntry validates and decodes exactly one object previously returned by +// WalkMetadata. Callers can therefore keep startup discovery bounded and pay +// the decompression cost only for an object that actually needs recovery. +func (s *Store) ReadEntry(metadata Metadata) (Entry, error) { + actual, err := s.metadata(metadata.Path) + if err != nil { + return Entry{}, err + } + if actual != metadata { + return Entry{}, errors.New("segment metadata changed before decode") + } + batch, err := s.Read(metadata.Path, metadata.Digest) + if err != nil { + return Entry{}, err + } + if batch.SourceID != metadata.SourceID || batch.StreamID != metadata.StreamID || batch.Sequence != metadata.Sequence { + return Entry{}, errors.New("segment path does not match batch identity") + } + raw, err := json.Marshal(batch) + if err != nil { + return Entry{}, fmt.Errorf("measure decoded segment: %w", err) + } + return Entry{ + OrganizationID: metadata.OrganizationID, + Committed: Committed{ + Path: metadata.Path, + Digest: metadata.Digest, + Compressed: metadata.Compressed, + Uncompressed: int64(len(raw)), + }, + Batch: batch, + }, nil +} + +// Delete removes one already-retired segment after verifying that the path, +// filename, file type, and content digest still identify the exact committed +// object. A missing object is an idempotent success for crash recovery. +func (s *Store) Delete(path, expectedDigest string) error { + if len(expectedDigest) != 64 { + return errors.New("segment deletion digest is invalid") + } + rawRoot := filepath.Join(filepath.Clean(s.root), "raw") + string(os.PathSeparator) + coldRoot := filepath.Join(filepath.Clean(s.root), "cold") + string(os.PathSeparator) + cleanPath := filepath.Clean(path) + if (!strings.HasPrefix(cleanPath, rawRoot) && !strings.HasPrefix(cleanPath, coldRoot)) || !strings.HasSuffix(filepath.Base(cleanPath), "-"+expectedDigest+".zst") { + return errors.New("segment deletion path is invalid") + } + if err := validatePrivateDirectoryChain(s.root, filepath.Dir(cleanPath)); err != nil { + return err + } + if _, err := os.Lstat(cleanPath); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("inspect retired segment: %w", err) + } + if _, err := s.Read(cleanPath, expectedDigest); err != nil { + return fmt.Errorf("verify retired segment: %w", err) + } + if err := os.Remove(cleanPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove retired segment: %w", err) + } + directory, err := os.Open(filepath.Dir(cleanPath)) + if err != nil { + return fmt.Errorf("open retired segment directory: %w", err) + } + if err = directory.Sync(); err != nil { + _ = directory.Close() + return fmt.Errorf("sync retired segment directory: %w", err) + } + if err = directory.Close(); err != nil { + return fmt.Errorf("close retired segment directory: %w", err) + } + return nil +} + +// MoveToCold atomically relocates one verified hot object beneath the cold +// archive. It is idempotent across a crash after rename: when the source is +// absent, the exact destination must already exist and match its digest. +func (s *Store) MoveToCold(path, target, expectedDigest string) error { + if len(expectedDigest) != 64 { + return errors.New("segment archive digest is invalid") + } + cleanPath, cleanTarget := filepath.Clean(path), filepath.Clean(target) + rawRoot := filepath.Join(filepath.Clean(s.root), "raw") + string(os.PathSeparator) + coldRoot := filepath.Join(filepath.Clean(s.root), "cold") + string(os.PathSeparator) + if !strings.HasPrefix(cleanPath, rawRoot) || !strings.HasPrefix(cleanTarget, coldRoot) || filepath.Base(cleanPath) != filepath.Base(cleanTarget) || !strings.HasSuffix(filepath.Base(cleanPath), "-"+expectedDigest+".zst") { + return errors.New("segment archive path is invalid") + } + if err := validatePrivateDirectoryChain(s.root, filepath.Dir(cleanPath)); err != nil { + return err + } + if err := ensurePrivateDirectoryChain(s.root, filepath.Dir(cleanTarget)); err != nil { + return err + } + if _, err := os.Lstat(cleanPath); errors.Is(err, os.ErrNotExist) { + if _, readErr := s.Read(cleanTarget, expectedDigest); readErr != nil { + return fmt.Errorf("recover archived segment: %w", readErr) + } + for _, directory := range []string{filepath.Dir(cleanTarget), filepath.Dir(cleanPath)} { + if syncErr := syncDirectory(directory); syncErr != nil { + return syncErr + } + } + return nil + } else if err != nil { + return fmt.Errorf("inspect hot segment: %w", err) + } + if _, err := os.Lstat(cleanTarget); err == nil { + return errors.New("cold segment destination already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect cold segment destination: %w", err) + } + if _, err := s.Read(cleanPath, expectedDigest); err != nil { + return fmt.Errorf("verify hot segment: %w", err) + } + if err := os.Rename(cleanPath, cleanTarget); err != nil { + return fmt.Errorf("archive segment: %w", err) + } + for _, directory := range []string{filepath.Dir(cleanTarget), filepath.Dir(cleanPath)} { + if err := syncDirectory(directory); err != nil { + return err + } + } + return nil +} + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return fmt.Errorf("open segment directory: %w", err) + } + if err = directory.Sync(); err != nil { + _ = directory.Close() + return fmt.Errorf("sync segment directory: %w", err) + } + if err = directory.Close(); err != nil { + return fmt.Errorf("close segment directory: %w", err) + } + return nil +} + +func validatePrivateDirectoryChain(root, directory string) error { + root = filepath.Clean(root) + directory = filepath.Clean(directory) + relative, err := filepath.Rel(root, directory) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) { + return errors.New("segment directory escapes store") + } + candidates := []string{root} + current := root + if relative != "." { + for _, component := range strings.Split(relative, string(os.PathSeparator)) { + if component == "" || component == "." || component == ".." { + return errors.New("segment directory path is invalid") + } + current = filepath.Join(current, component) + candidates = append(candidates, current) + } + } + for _, candidate := range candidates { + info, inspectErr := os.Lstat(candidate) + if inspectErr != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return errors.New("segment directory chain must be private and non-symlinked") + } + } + return nil +} + +func ensurePrivateDirectoryChain(root, directory string) error { + root = filepath.Clean(root) + directory = filepath.Clean(directory) + relative, err := filepath.Rel(root, directory) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) { + return errors.New("segment directory path is invalid") + } + current := root + for _, component := range strings.Split(relative, string(os.PathSeparator)) { + if component == "" || component == "." || component == ".." { + return errors.New("segment directory path is invalid") + } + current = filepath.Join(current, component) + info, inspectErr := os.Lstat(current) + if errors.Is(inspectErr, os.ErrNotExist) { + if inspectErr = os.Mkdir(current, 0o700); inspectErr != nil && !errors.Is(inspectErr, os.ErrExist) { + return fmt.Errorf("create segment directory: %w", inspectErr) + } + info, inspectErr = os.Lstat(current) + } + if inspectErr != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return errors.New("segment directory chain must be private and non-symlinked") + } + } + return nil +} + +func readRegular(path string, maximum int) ([]byte, error) { + before, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !before.Mode().IsRegular() || before.Mode()&os.ModeSymlink != 0 || before.Size() > int64(maximum) { + return nil, errors.New("segment is not a bounded regular file") + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + after, err := file.Stat() + if err != nil || !after.Mode().IsRegular() || !os.SameFile(before, after) { + return nil, errors.New("segment changed during open") + } + body, err := io.ReadAll(io.LimitReader(file, int64(maximum)+1)) + if err != nil { + return nil, err + } + if len(body) > maximum { + return nil, errors.New("encoded segment exceeds limit") + } + return body, nil +} + +// WalkMetadata derives bounded committed-object identities from the filesystem +// so a crash after rename but before control-database bookkeeping cannot hide +// a segment. It deliberately does not read or decode segment contents. +func (s *Store) WalkMetadata(visit func(Metadata) error) error { + if visit == nil { + return errors.New("segment metadata visitor is required") + } + base := filepath.Join(s.root, "raw") + if _, err := os.Lstat(base); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("inspect raw segment root: %w", err) + } + if err := validatePrivateDirectoryChain(s.root, base); err != nil { + return err + } + err := s.walkMetadataDirectory(base, 0, visit) + if err != nil { + return fmt.Errorf("walk raw segments: %w", err) + } + return nil +} + +func (s *Store) walkMetadataDirectory(directoryPath string, depth int, visit func(Metadata) error) error { + directory, err := os.Open(directoryPath) + if err != nil { + return fmt.Errorf("open raw segment directory: %w", err) + } + defer directory.Close() + info, err := directory.Stat() + if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 { + return errors.New("raw segment directory must be private") + } + for { + entries, readErr := directory.ReadDir(metadataReadBatch) + for _, entry := range entries { + path := filepath.Join(directoryPath, entry.Name()) + if entry.Type()&os.ModeSymlink != 0 { + return errors.New("raw segment tree contains a symlink") + } + if entry.IsDir() { + if depth >= 3 { + return errors.New("raw segment tree has invalid depth") + } + if err = s.walkMetadataDirectory(path, depth+1, visit); err != nil { + return err + } + continue + } + if !strings.HasSuffix(entry.Name(), ".zst") { + continue + } + metadata, metadataErr := s.metadata(path) + if metadataErr != nil { + return metadataErr + } + if visitErr := visit(metadata); visitErr != nil { + return visitErr + } + } + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return fmt.Errorf("read raw segment directory: %w", readErr) + } + } + if err = directory.Close(); err != nil { + return fmt.Errorf("close raw segment directory: %w", err) + } + return nil +} + +func (s *Store) metadata(path string) (Metadata, error) { + base := filepath.Join(s.root, "raw") + cleanPath := filepath.Clean(path) + relative, err := filepath.Rel(base, cleanPath) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) { + return Metadata{}, errors.New("segment path escapes raw store") + } + pathParts := strings.Split(filepath.ToSlash(relative), "/") + if len(pathParts) != 4 { + return Metadata{}, errors.New("raw segment path has invalid depth") + } + if err = model.ValidateSourceID(pathParts[0]); err != nil { + return Metadata{}, errors.New("raw segment organization is invalid") + } + if err = model.ValidateSourceID(pathParts[1]); err != nil { + return Metadata{}, errors.New("raw segment source is invalid") + } + if err = model.ValidateStreamID(pathParts[2]); err != nil { + return Metadata{}, errors.New("raw segment stream is invalid") + } + name := strings.TrimSuffix(pathParts[3], ".zst") + parts := strings.Split(name, "-") + if len(parts) != 2 || len(parts[0]) != 20 || len(parts[1]) != 64 || strings.ToLower(parts[1]) != parts[1] { + return Metadata{}, fmt.Errorf("invalid segment filename %q", pathParts[3]) + } + sequence, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil || sequence == 0 { + return Metadata{}, fmt.Errorf("invalid segment sequence %q", parts[0]) + } + if _, err = hex.DecodeString(parts[1]); err != nil { + return Metadata{}, errors.New("segment filename digest is invalid") + } + if err = validatePrivateDirectoryChain(s.root, filepath.Dir(cleanPath)); err != nil { + return Metadata{}, err + } + info, err := os.Lstat(cleanPath) + if err != nil { + return Metadata{}, fmt.Errorf("inspect raw segment: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 || info.Size() > MaxEncodedSegment { + return Metadata{}, errors.New("raw segment must be a private bounded regular file") + } + return Metadata{ + OrganizationID: pathParts[0], + SourceID: pathParts[1], + StreamID: pathParts[2], + Sequence: sequence, + Path: cleanPath, + Digest: parts[1], + Compressed: info.Size(), + }, nil +} + +// List retains the complete decoding API for explicit forensic callers and +// tests. Startup recovery uses WalkMetadata and decodes only missing work. +func (s *Store) List() ([]Entry, error) { + var entries []Entry + err := s.WalkMetadata(func(metadata Metadata) error { + entry, readErr := s.ReadEntry(metadata) + if readErr != nil { + return readErr + } + entries = append(entries, entry) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Committed.Path < entries[j].Committed.Path }) + return entries, nil +} diff --git a/internal/segment/store_test.go b/internal/segment/store_test.go new file mode 100644 index 0000000..94e3c73 --- /dev/null +++ b/internal/segment/store_test.go @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package segment + +import ( + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "gamertan.com/observatory/internal/model" +) + +func TestCommitReadAndCorruption(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + store, err := New(root) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request", Body: "safe"}}} + scope := model.Scope{OrganizationID: "org", ProjectID: "project", EnvironmentID: "prod", ServiceID: "site"} + committed, err := store.Commit(scope, batch) + if err != nil { + t.Fatal(err) + } + got, err := store.Read(committed.Path, committed.Digest) + if err != nil { + t.Fatal(err) + } + if got.Sequence != 1 || got.SourceID != "source" { + t.Fatalf("unexpected batch: %#v", got) + } + before, err := os.Stat(committed.Path) + if err != nil { + t.Fatal(err) + } + again, err := store.Commit(scope, batch) + if err != nil { + t.Fatal(err) + } + after, _ := os.Stat(again.Path) + if !before.ModTime().Equal(after.ModTime()) { + t.Fatal("idempotent commit changed segment mtime") + } + if err := os.WriteFile(committed.Path, []byte("corrupt"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := store.Read(committed.Path, committed.Digest); err == nil { + t.Fatal("expected checksum failure") + } +} + +func TestConcurrentCommitsCreateOnePrivateDirectoryChain(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + store, err := New(root) + if err != nil { + t.Fatal(err) + } + const writers = 64 + now := time.Now().UTC() + scope := model.Scope{OrganizationID: "org", ProjectID: "project", EnvironmentID: "prod", ServiceID: "site"} + start := make(chan struct{}) + errorsChannel := make(chan error, writers) + var wait sync.WaitGroup + for index := 0; index < writers; index++ { + wait.Add(1) + go func(sequence uint64) { + defer wait.Done() + <-start + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: sequence, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request"}}} + if _, commitErr := store.Commit(scope, batch); commitErr != nil { + errorsChannel <- commitErr + } + }(uint64(index + 1)) + } + close(start) + wait.Wait() + close(errorsChannel) + for commitErr := range errorsChannel { + t.Errorf("concurrent commit: %v", commitErr) + } + entries, err := store.List() + if err != nil { + t.Fatal(err) + } + if len(entries) != writers { + t.Fatalf("committed entries=%d want=%d", len(entries), writers) + } +} + +func TestListFindsCommittedSegments(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + store, err := New(root) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request"}}} + _, err = store.Commit(model.Scope{OrganizationID: "org", ProjectID: "project", EnvironmentID: "prod", ServiceID: "site"}, batch) + if err != nil { + t.Fatal(err) + } + entries, err := store.List() + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Batch.Sequence != 1 || entries[0].Committed.Digest == "" { + t.Fatalf("unexpected entries: %#v", entries) + } +} + +func TestWalkMetadataDoesNotDecodeSegmentContents(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + store, err := New(root) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request"}}} + committed, err := store.Commit(model.Scope{OrganizationID: "org", ProjectID: "project", EnvironmentID: "prod", ServiceID: "site"}, batch) + if err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(committed.Path) + if err != nil { + t.Fatal(err) + } + body[0] ^= 0xff + if err = os.WriteFile(committed.Path, body, 0o600); err != nil { + t.Fatal(err) + } + var metadata Metadata + if err = store.WalkMetadata(func(candidate Metadata) error { + metadata = candidate + return nil + }); err != nil { + t.Fatalf("metadata walk decoded content: %v", err) + } + if metadata.Path != committed.Path || metadata.Digest != committed.Digest || metadata.Compressed != committed.Compressed { + t.Fatalf("metadata=%+v committed=%+v", metadata, committed) + } + if _, err = store.ReadEntry(metadata); err == nil { + t.Fatal("corrupt segment decoded successfully") + } +} + +func TestInterruptedTemporarySegmentIsIgnoredUntilAtomicCommit(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + store, err := New(root) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, "raw", "organization", "source", "stream") + if err = os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + partial := filepath.Join(dir, ".segment-interrupted") + if err = os.WriteFile(partial, []byte("partial compressed bytes"), 0o600); err != nil { + t.Fatal(err) + } + entries, err := store.List() + if err != nil || len(entries) != 0 { + t.Fatalf("entries=%+v err=%v", entries, err) + } + if body, readErr := os.ReadFile(partial); readErr != nil || string(body) != "partial compressed bytes" { + t.Fatalf("partial=%q err=%v", body, readErr) + } +} + +func TestReadRejectsSymlinksAndOversizedEncodedSegments(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + store, err := New(root) + if err != nil { + t.Fatal(err) + } + if err = os.MkdirAll(filepath.Join(root, "raw"), 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(root, "raw", "target") + if err = os.WriteFile(target, []byte("safe"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "raw", "link") + if err = os.Symlink(target, link); err != nil { + t.Fatal(err) + } + if _, err = store.Read(link, "unused"); err == nil { + t.Fatal("symlink segment accepted") + } + oversized := filepath.Join(root, "raw", "oversized") + file, err := os.Create(oversized) + if err != nil { + t.Fatal(err) + } + if err = file.Truncate(MaxEncodedSegment + 1); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err = file.Close(); err != nil { + t.Fatal(err) + } + if _, err = store.Read(oversized, "unused"); err == nil { + t.Fatal("oversized encoded segment accepted") + } +} + +func TestCommitRejectsSymlinkedDirectoryChain(t *testing.T) { + base := t.TempDir() + root := filepath.Join(base, "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := New(root) + if err != nil { + t.Fatal(err) + } + if err = os.Mkdir(filepath.Join(root, "raw"), 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(base, "outside") + if err = os.Mkdir(target, 0o700); err != nil { + t.Fatal(err) + } + if err = os.Symlink(target, filepath.Join(root, "raw", "org")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request"}}} + if _, err = store.Commit(model.Scope{OrganizationID: "org", ProjectID: "project", EnvironmentID: "prod", ServiceID: "site"}, batch); err == nil { + t.Fatal("symlinked segment directory was accepted") + } + entries, err := os.ReadDir(target) + if err != nil || len(entries) != 0 { + t.Fatalf("symlink target was changed: entries=%v err=%v", entries, err) + } +} + +func TestMoveToColdIsVerifiedAndCrashIdempotent(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + store, err := New(root) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request"}}} + committed, err := store.Commit(model.Scope{OrganizationID: "org", ProjectID: "project", EnvironmentID: "prod", ServiceID: "site"}, batch) + if err != nil { + t.Fatal(err) + } + target := filepath.Join(root, "cold", "org", "logs", "source", "access", filepath.Base(committed.Path)) + if err = store.MoveToCold(committed.Path, target, committed.Digest); err != nil { + t.Fatal(err) + } + if _, err = os.Lstat(committed.Path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("hot segment still exists: %v", err) + } + if got, readErr := store.Read(target, committed.Digest); readErr != nil || got.Sequence != 1 { + t.Fatalf("cold batch=%+v err=%v", got, readErr) + } + if err = store.MoveToCold(committed.Path, target, committed.Digest); err != nil { + t.Fatalf("idempotent archive: %v", err) + } + if err = store.MoveToCold(target, committed.Path, committed.Digest); err == nil { + t.Fatal("reverse cold-to-hot move was accepted") + } +} + +func TestMoveToColdRejectsSymlinkedDestination(t *testing.T) { + base := t.TempDir() + root := filepath.Join(base, "data") + store, err := New(root) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request"}}} + committed, err := store.Commit(model.Scope{OrganizationID: "org", ProjectID: "project", EnvironmentID: "prod", ServiceID: "site"}, batch) + if err != nil { + t.Fatal(err) + } + outside := filepath.Join(base, "outside") + if err = os.Mkdir(outside, 0o700); err != nil { + t.Fatal(err) + } + if err = os.Symlink(outside, filepath.Join(root, "cold")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + target := filepath.Join(root, "cold", "org", "logs", "source", "access", filepath.Base(committed.Path)) + if err = store.MoveToCold(committed.Path, target, committed.Digest); err == nil { + t.Fatal("symlinked cold destination was accepted") + } + if _, err = os.Stat(committed.Path); err != nil { + t.Fatalf("hot segment changed: %v", err) + } + entries, err := os.ReadDir(outside) + if err != nil || len(entries) != 0 { + t.Fatalf("outside entries=%v err=%v", entries, err) + } +} diff --git a/internal/site/app.sando b/internal/site/app.sando new file mode 100644 index 0000000..3c3653c --- /dev/null +++ b/internal/site/app.sando @@ -0,0 +1,126 @@ + + + + + + + + +
+
+

Organization overview

Welcome, . Last refreshed .

+
+ + +
+
+ 1 { ?> + + + 0 { ?> + + + +
+ +
+

+

Recent

+

+
Query
+ +
+ +
+
+

Saved work

Queries and dashboards

+
+

Saved queries

+

No saved queries yet.

+

+
+

Dashboards

+

No dashboards yet.

+

panels

+
+
+
+
+

Incident response

Attention without noise

+

open incidents are visible in the authorized inbox. Alert rules execute the same bounded saved-query AST used everywhere else.

Open incident inbox

+
+ +
+

Organization workshop

Save a reusable view

Assisted controls and typed text enter the same parser and versioned AST. Every form is an ordinary server request; nothing is generated or executed on page load.

+
+
+

Build a query

+

Choose one bounded evidence source, an optional filter, and an optional summary. The saved result remains ordinary reviewable query text.

+ + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+

Write a query

+ + + + + + +
+
+

Create a one-panel dashboard

+ + + + + + + + + + + +
+
+
+ +
+ + + diff --git a/internal/site/app.sando.go b/internal/site/app.sando.go new file mode 100644 index 0000000..a48c1b4 --- /dev/null +++ b/internal/site/app.sando.go @@ -0,0 +1,497 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 ab6403781ccb2a659b749730243555dcb8a3f319720ad6a972252eb30847c5f4 + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func App(view AppView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/app.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:9:11 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (Head(view.Head))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:9:29 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n Skip to content\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:12:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteHeader(true, "overview", view.Organization.ID))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:12:60 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n

Organization overview

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:15:84 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Organization.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:15:109 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Welcome, "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:15:130 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.DisplayName)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:15:149 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ". Last refreshed "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:15:170 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.RefreshedAt)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:15:189 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ".

\n
\n \n \n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:21:8 + if len(view.Organizations) > 1 { +//line internal/site/app.sando:21:43 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:32:8 + } +//line internal/site/app.sando:32:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:33:8 + if view.PendingBatches > 0 { +//line internal/site/app.sando:33:39 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:37:8 + } +//line internal/site/app.sando:37:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n \n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:42:8 + for _, signal := range view.Signals { +//line internal/site/app.sando:42:48 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:44:32 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (signal.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:44:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

\n

Recent "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:45:24 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (signal.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:45:38 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

\n

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:46:16 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (signal.Description)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:46:37 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

\n
Query
"); __himesan_error != nil {
+				return __himesan_error
+			}
+//line internal/site/app.sando:47:57
+			if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (signal.Query)); __himesan_error != nil {
+				return __himesan_error
+			}
+//line internal/site/app.sando:47:72
+			if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:48:13 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (ResultTable(signal.Table))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:48:41 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:50:8 + } +//line internal/site/app.sando:50:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n

Saved work

Queries and dashboards

\n
\n

Saved queries

\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:56:12 + if len(view.SavedQueries) == 0 { +//line internal/site/app.sando:56:47 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

No saved queries yet.

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:56:78 + } +//line internal/site/app.sando:56:82 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:57:12 + for _, item := range view.SavedQueries { +//line internal/site/app.sando:57:55 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:57:72 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (item.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:57:84 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:57:96 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (item.Description)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:57:115 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil {
+				return __himesan_error
+			}
+//line internal/site/app.sando:57:134
+			if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (item.Query)); __himesan_error != nil {
+				return __himesan_error
+			}
+//line internal/site/app.sando:57:147
+			if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:57:173 + } +//line internal/site/app.sando:57:177 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Dashboards

\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:60:12 + if len(view.Dashboards) == 0 { +//line internal/site/app.sando:60:45 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

No dashboards yet.

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:60:73 + } +//line internal/site/app.sando:60:77 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:61:12 + for _, item := range view.Dashboards { +//line internal/site/app.sando:61:53 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:61:155 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (item.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:61:167 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:61:183 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (item.Description)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:61:202 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:61:213 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (item.PanelCount)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:61:231 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, " panels

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:61:255 + } +//line internal/site/app.sando:61:259 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n
\n
\n

Incident response

Attention without noise

\n

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:67:19 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.OpenIncidents)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:67:40 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, " open incidents are visible in the authorized inbox. Alert rules execute the same bounded saved-query AST used everywhere else.

Open incident inbox

\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:69:8 + if view.CanManage { +//line internal/site/app.sando:69:30 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Organization workshop

Save a reusable view

Assisted controls and typed text enter the same parser and versioned AST. Every form is an ordinary server request; nothing is generated or executed on page load.

\n
\n
\n

Build a query

\n

Choose one bounded evidence source, an optional filter, and an optional summary. The saved result remains ordinary reviewable query text.

\n \n \n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n \n
\n
\n

Write a query

\n \n \n \n \n \n \n
\n
\n

Create a one-panel dashboard

\n \n \n \n \n \n \n \n \n \n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:118:14 + if len(view.SavedQueries) == 0 { +//line internal/site/app.sando:118:49 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ""); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:118:108 + } else { +//line internal/site/app.sando:118:119 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ""); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:118:169 + } +//line internal/site/app.sando:118:173 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:122:8 + } +//line internal/site/app.sando:122:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:124:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteFooter())); __himesan_error != nil { + return __himesan_error + } +//line internal/site/app.sando:124:22 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/assets.go b/internal/site/assets.go new file mode 100644 index 0000000..65443f1 --- /dev/null +++ b/internal/site/assets.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package site contains Observatory's typed Sandwich Hime interface and its +// immutable browser assets. HTTP policy remains owned by internal/httpserver. +package site + +import ( + "crypto/sha256" + _ "embed" + "encoding/hex" +) + +//go:embed assets/site.css +var style []byte + +//go:embed assets/site.js +var script []byte + +//go:embed assets/observatory.svg +var icon []byte + +type Assets struct { + StylePath string + ScriptPath string + IconPath string +} + +func AssetPaths() Assets { + return Assets{StylePath: fingerprintedPath("site", "css", style), ScriptPath: fingerprintedPath("site", "js", script), IconPath: fingerprintedPath("observatory", "svg", icon)} +} + +func Asset(path string) (body []byte, contentType string, ok bool) { + assets := AssetPaths() + switch path { + case assets.StylePath: + return style, "text/css; charset=utf-8", true + case assets.ScriptPath: + return script, "text/javascript; charset=utf-8", true + case assets.IconPath: + return icon, "image/svg+xml", true + default: + return nil, "", false + } +} + +func fingerprintedPath(name, extension string, body []byte) string { + digest := sha256.Sum256(body) + return "/assets/" + name + "-" + hex.EncodeToString(digest[:8]) + "." + extension +} diff --git a/internal/site/assets/observatory.svg b/internal/site/assets/observatory.svg new file mode 100644 index 0000000..63f4fc1 --- /dev/null +++ b/internal/site/assets/observatory.svg @@ -0,0 +1,12 @@ + + + Gamertan Observatory + Three calm observation rings around one bright point. + + + + + + + + diff --git a/internal/site/assets/site.css b/internal/site/assets/site.css new file mode 100644 index 0000000..bcaadee --- /dev/null +++ b/internal/site/assets/site.css @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: AGPL-3.0-only */ +:root{color-scheme:dark;--ink:#f6f0e2;--muted:#bfb8aa;--paper:#111715;--panel:#18221f;--line:#4f665e;--accent:#b5e3c5;--focus:#f7c873;font:100%/1.6 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif} +*{box-sizing:border-box} +html{background:var(--paper);color:var(--ink)} +body{margin:0;min-width:18rem} +a{color:var(--accent);text-underline-offset:.2em} +a:hover{text-decoration-thickness:.16em} +button,input,select,textarea{font:inherit} +button,.button{display:inline-block;border:.12rem solid var(--ink);border-radius:.2rem;background:var(--ink);color:var(--paper);padding:.65rem 1rem;font-weight:700;text-decoration:none;cursor:pointer} +button.secondary,.secondary{background:transparent;color:var(--ink)} +:focus-visible{outline:.2rem solid var(--focus);outline-offset:.2rem} +.skip-link{position:absolute;left:1rem;top:-8rem;background:var(--ink);color:var(--paper);padding:.6rem;z-index:10}.skip-link:focus{top:1rem} +.site-header,.site-footer,main{width:min(92rem,100%);margin-inline:auto;padding-inline:clamp(1rem,4vw,3rem)} +.site-header{display:flex;align-items:center;justify-content:space-between;gap:1rem;padding-block:1.25rem;border-bottom:.08rem solid var(--line)} +.site-header nav{display:flex;gap:1rem}.wordmark{color:var(--ink);font-weight:800;text-decoration:none;letter-spacing:.02em} +.site-footer{margin-top:4rem;padding-block:2rem;border-top:.08rem solid var(--line);color:var(--muted)} +.hero{padding-block:clamp(4rem,12vw,9rem);max-width:72rem}.hero h1{font-size:clamp(3rem,10vw,8rem);line-height:.9;letter-spacing:-.055em;max-width:11ch;margin:.15em 0}.lede{font-size:clamp(1.15rem,2vw,1.5rem);max-width:48rem;color:var(--muted)} +.eyebrow{text-transform:uppercase;letter-spacing:.14em;font-size:.78rem;font-weight:800;color:var(--accent)} +.section-grid{display:grid;grid-template-columns:minmax(14rem,1fr) minmax(0,2fr);gap:clamp(2rem,6vw,7rem);padding-block:clamp(3rem,7vw,6rem);border-top:.08rem solid var(--line)} +h1,h2,h3,h4{line-height:1.1;text-wrap:balance}h2{font-size:clamp(2rem,5vw,4.5rem);margin:.15em 0}h3{font-size:1.4rem}.prose{max-width:52rem}.cards{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:1rem}.cards article,.auth-card,.signal-card,.saved-grid>section,.live-status{background:var(--panel);border:.08rem solid var(--line);padding:clamp(1rem,3vw,2rem)} +.auth-shell{min-height:70vh;display:grid;place-items:center}.auth-card{width:min(34rem,100%);margin-block:4rem}.auth-card h1{font-size:clamp(2.3rem,8vw,4rem)} +form{display:grid;gap:.55rem}label{font-weight:700;margin-top:.6rem}input,select,textarea{width:100%;border:.1rem solid var(--line);background:var(--paper);color:var(--ink);padding:.7rem;border-radius:.2rem}textarea{min-height:7rem;resize:vertical}.form-error{border-left:.3rem solid #ff9b8f;padding:.7rem;background:#301b1b} +.app-intro{display:flex;align-items:end;justify-content:space-between;gap:2rem;padding-block:3rem}.app-intro h1{font-size:clamp(2.6rem,7vw,6rem);margin:.1em 0}.logout{display:block}.organization-nav{display:flex;flex-wrap:wrap;gap:.7rem 1rem;align-items:center;border-block:.08rem solid var(--line);padding-block:1rem}.organization-nav a[aria-current="page"]{color:var(--ink);font-weight:800} +.live-status{margin-block:1rem}.live-status p{margin:0}.signal-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1rem;padding-block:2rem}.signal-card{min-width:0}.signal-card h2{font-size:clamp(1.8rem,4vw,3rem)} +details{margin-block:1rem}pre{max-width:100%;overflow:auto;background:#0a0f0d;border:.08rem solid var(--line);padding:.8rem;white-space:pre-wrap;overflow-wrap:anywhere} +.table-scroll{max-width:100%;overflow:auto;border:.08rem solid var(--line)}table{width:100%;border-collapse:collapse;min-width:40rem}caption{text-align:left;padding:.7rem;font-weight:800}th,td{text-align:left;vertical-align:top;border-top:.06rem solid var(--line);padding:.55rem .7rem;overflow-wrap:anywhere}th{color:var(--accent)}.unit{font-weight:400;color:var(--muted)} +.saved-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1rem}.saved-grid article+article{border-top:.08rem solid var(--line);margin-top:1rem;padding-top:1rem} +.editor-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1rem}.editor-card{background:var(--panel);border:.08rem solid var(--line);padding:clamp(1rem,3vw,2rem)}.builder-card{grid-column:1/-1}.field-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:.6rem 1rem}.optional{color:var(--muted);font-size:.9em;font-weight:400}button:disabled{cursor:not-allowed;opacity:.55}.breadcrumbs{display:flex;flex-wrap:wrap;gap:.5rem;padding-top:1.5rem}.dashboard-panels{display:grid;gap:1rem}.stat{font-size:clamp(2rem,8vw,5rem);font-weight:800;line-height:1;margin:.5em 0} +.meter-figure{margin:1.5rem 0}.meter-figure figcaption{font-weight:800;margin-bottom:.75rem}.meter-chart{display:grid;gap:.7rem;list-style:none;margin:0;padding:0}.meter-chart li{display:grid;grid-template-columns:minmax(8rem,1fr) minmax(10rem,3fr) minmax(5rem,auto);gap:.8rem;align-items:center}.meter-label,.meter-value{overflow-wrap:anywhere}.meter-value{font-variant-numeric:tabular-nums;text-align:right}meter{width:100%;height:1.2rem;accent-color:var(--accent)} +.incident-list{display:grid;gap:1rem}.incident-card{background:var(--panel);border:.08rem solid var(--line);border-left:.35rem solid var(--accent);padding:clamp(1rem,3vw,2rem)}.incident-card[data-state="firing"]{border-left-color:#ff9b8f}.incident-card[data-state="resolved"]{opacity:.78}.incident-card dl,.saved-grid dl{display:grid;gap:.5rem;margin-block:1rem}.incident-card dl div,.saved-grid dl div{display:grid;grid-template-columns:minmax(8rem,1fr) minmax(0,2fr);gap:1rem}.incident-card dt,.saved-grid dt{font-weight:800}.incident-card dd,.saved-grid dd{margin:0}.incident-actions{display:flex;align-items:end;flex-wrap:wrap;gap:.75rem}.incident-actions form{display:flex;align-items:end;gap:.45rem}.incident-actions label{margin:0}.incident-actions select{width:auto;min-width:10rem} +.offline-control{display:grid;grid-template-columns:minmax(14rem,1fr) minmax(0,2fr);gap:clamp(2rem,6vw,7rem);margin-block:1rem;padding:clamp(1rem,3vw,2rem);border:.08rem solid var(--line);background:var(--panel)}.offline-control h2{font-size:clamp(1.5rem,3vw,2.5rem)}.offline-control [role="status"]{color:var(--muted)} +.explore-layout{display:grid;grid-template-columns:minmax(16rem,1fr) minmax(0,2fr);gap:clamp(2rem,6vw,7rem);padding-block:2rem;border-top:.08rem solid var(--line)}.explore-layout h2,.query-results h2{font-size:clamp(2rem,5vw,4rem)}.query-workbench textarea{min-height:14rem;font-family:ui-monospace,SFMono-Regular,Consolas,monospace}.field-help{color:var(--muted)}.quick-queries{display:flex;flex-wrap:wrap;gap:.6rem;margin-block:1.5rem}.quick-queries form{display:block}.query-results{padding-block:3rem;border-top:.08rem solid var(--line)}.query-results-heading{display:grid;grid-template-columns:minmax(14rem,1fr) minmax(0,2fr);gap:clamp(2rem,6vw,7rem);align-items:end}.query-stats{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:.75rem;margin:0}.query-stats div{border-left:.18rem solid var(--line);padding-left:.75rem}.query-stats dt{color:var(--muted);font-size:.82rem;font-weight:800;text-transform:uppercase;letter-spacing:.08em}.query-stats dd{margin:.15rem 0 0;font-size:1.15rem;font-variant-numeric:tabular-nums;font-weight:800} +@media(max-width:55rem){.section-grid,.cards,.signal-grid,.saved-grid,.editor-grid,.field-grid,.offline-control,.explore-layout,.query-results-heading{grid-template-columns:1fr}.query-stats{grid-template-columns:repeat(2,minmax(0,1fr))}.app-intro{align-items:start;flex-direction:column}.hero h1{letter-spacing:-.035em}.site-header{align-items:flex-start;flex-direction:column}.site-header nav{flex-wrap:wrap}.meter-chart li{grid-template-columns:1fr}.meter-value{text-align:left}.incident-card dl div{grid-template-columns:1fr;gap:.1rem}.incident-actions,.incident-actions form{align-items:stretch;flex-direction:column}.incident-actions select{width:100%}} +@media(max-width:24rem){:root{font-size:95%}.hero h1{font-size:2.75rem;letter-spacing:-.025em}.site-header,.site-footer,main{padding-inline:.8rem}} +@media(prefers-reduced-motion:reduce){*,*::before,*::after{scroll-behavior:auto!important;animation:none!important;transition:none!important}} +@media(forced-colors:active){button,.button,.auth-card,.signal-card,.saved-grid>section,.editor-card,.live-status,.incident-card,.offline-control,.query-stats div{border:1px solid CanvasText}.eyebrow,a{color:LinkText}} +@media print{*{color:#000!important;background:#fff!important}.site-header nav,.logout,.live-status,.button,script{display:none!important}.site-header,.site-footer,.section-grid{border-color:#777}.signal-grid,.section-grid{display:block}.signal-card{break-inside:avoid;margin-block:1rem}.table-scroll{overflow:visible}table{min-width:0;font-size:9pt}} diff --git a/internal/site/assets/site.js b/internal/site/assets/site.js new file mode 100644 index 0000000..654206a --- /dev/null +++ b/internal/site/assets/site.js @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: AGPL-3.0-only +(() => { + "use strict"; + const serviceWorker = "serviceWorker" in navigator + ? navigator.serviceWorker.register("/service-worker.js", {scope: "/"}) + : Promise.reject(new Error("service workers unavailable")); + + const messageWorker = async message => { + const registration = await serviceWorker; + const worker = registration.active || registration.waiting || registration.installing; + if (!worker) throw new Error("service worker unavailable"); + return await new Promise((resolve, reject) => { + const channel = new MessageChannel(); + const timeout = window.setTimeout(() => reject(new Error("service worker timeout")), 5000); + channel.port1.onmessage = event => { + window.clearTimeout(timeout); + event.data && event.data.ok ? resolve(event.data) : reject(new Error("service worker request failed")); + }; + worker.postMessage(message, [channel.port2]); + }); + }; + + const decodeURLBase64 = value => { + const normalized = value.replace(/-/g, "+").replace(/_/g, "/"); + const padded = normalized + "=".repeat((4 - normalized.length % 4) % 4); + const decoded = window.atob(padded); + return Uint8Array.from(decoded, character => character.charCodeAt(0)); + }; + + const pushRequest = async (button, action, subscription) => { + const serialized = subscription.toJSON(); + const status = action === "status"; + const response = await fetch(status ? "/api/v1/push/subscription/status" : "/api/v1/push/subscription", { + method: action === "delete" ? "DELETE" : "POST", + credentials: "same-origin", + cache: "no-store", + headers: {"Content-Type": "application/json", "X-CSRF-Token": button.dataset.pushCsrf}, + body: JSON.stringify({ + organization_id: button.dataset.pushOrganization, + endpoint: subscription.endpoint, + keys: action === "save" ? serialized.keys : {p256dh: "", auth: ""} + }) + }); + if (!response.ok) throw new Error("push subscription request failed"); + return await response.json(); + }; + + for (const button of document.querySelectorAll("[data-cache-inbox]")) { + const status = document.querySelector("[data-cache-status]"); + serviceWorker.then(() => { + button.disabled = false; + if (status) status.textContent = "No private incident copy has been saved by this control yet."; + }).catch(() => { + if (status) status.textContent = "Offline saving is unavailable in this browser."; + }); + button.addEventListener("click", async () => { + button.disabled = true; + if (status) status.textContent = "Saving a private, read-only incident snapshot…"; + try { + await messageWorker({type: "cache-inbox", source: button.dataset.offlineSource, target: button.dataset.offlineTarget}); + if (status) status.textContent = "This incident inbox is available offline on this browser."; + } catch (_) { + if (status) status.textContent = "The offline incident snapshot could not be saved."; + } finally { + button.disabled = false; + } + }); + } + + for (const form of document.querySelectorAll('form[action="/logout/"]')) { + form.addEventListener("submit", async event => { + if (form.dataset.privateCacheCleared === "true") return; + event.preventDefault(); + try { await messageWorker({type: "clear-private"}); } catch (_) {} + form.dataset.privateCacheCleared = "true"; + form.requestSubmit(); + }); + } + + for (const button of document.querySelectorAll("[data-push-toggle]")) { + const status = document.querySelector("[data-push-status]"); + const ready = serviceWorker.then(async registration => { + if (!("PushManager" in window) || !("Notification" in window)) throw new Error("push unavailable"); + const existing = await registration.pushManager.getSubscription(); + const registered = existing ? (await pushRequest(button, "status", existing)).subscribed === true : false; + button.dataset.pushRegistered = registered ? "true" : "false"; + button.textContent = registered ? "Disable private incident nudges" : "Enable private incident nudges"; + button.disabled = false; + if (status) status.textContent = registered ? "This browser is subscribed for this organization." : "This browser is not subscribed for this organization."; + return registration; + }); + ready.catch(() => { if (status) status.textContent = "Web Push is unavailable in this browser."; }); + button.addEventListener("click", async () => { + button.disabled = true; + try { + const registration = await ready; + let subscription = await registration.pushManager.getSubscription(); + if (subscription && button.dataset.pushRegistered === "true") { + const result = await pushRequest(button, "delete", subscription); + if (!result.remaining) await subscription.unsubscribe(); + button.dataset.pushRegistered = "false"; + button.textContent = "Enable private incident nudges"; + if (status) status.textContent = "This browser is no longer subscribed for this organization."; + } else { + let createdNow = false; + if (!subscription) { + const permission = await Notification.requestPermission(); + if (permission !== "granted") throw new Error("notification permission not granted"); + subscription = await registration.pushManager.subscribe({userVisibleOnly: true, applicationServerKey: decodeURLBase64(button.dataset.pushPublicKey)}); + createdNow = true; + } + try { + await pushRequest(button, "save", subscription); + } catch (error) { + if (createdNow) await subscription.unsubscribe(); + throw error; + } + button.dataset.pushRegistered = "true"; + button.textContent = "Disable private incident nudges"; + if (status) status.textContent = "This browser will receive generic incident nudges."; + } + } catch (_) { + if (status) status.textContent = "The browser push setting could not be changed."; + } finally { + button.disabled = false; + } + }); + } + + const incidentCount = document.querySelector("[data-open-incident-count]"); + if (incidentCount && "setAppBadge" in navigator) { + const count = Number.parseInt(incidentCount.dataset.openIncidentCount, 10); + if (Number.isSafeInteger(count) && count >= 0) { + const update = count === 0 && "clearAppBadge" in navigator ? navigator.clearAppBadge() : navigator.setAppBadge(count); + Promise.resolve(update).catch(() => {}); + } + } + + for (const status of document.querySelectorAll(".live-status[data-events-url]")) { + const source = new EventSource(status.dataset.eventsUrl); + source.addEventListener("refresh", () => { + status.hidden = false; + }); + window.addEventListener("pagehide", () => source.close(), {once: true}); + } +})(); diff --git a/internal/site/assets_test.go b/internal/site/assets_test.go new file mode 100644 index 0000000..bd0f7a9 --- /dev/null +++ b/internal/site/assets_test.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package site + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +func TestAssetsAreContentAddressedAndBounded(t *testing.T) { + paths := AssetPaths() + for _, test := range []struct { + path, suffix, contentType string + body []byte + }{{paths.StylePath, ".css", "text/css; charset=utf-8", style}, {paths.ScriptPath, ".js", "text/javascript; charset=utf-8", script}, {paths.IconPath, ".svg", "image/svg+xml", icon}} { + t.Run(test.suffix, func(t *testing.T) { + if len(test.body) == 0 || len(test.body) > 64<<10 || !strings.HasSuffix(test.path, test.suffix) { + t.Fatalf("path=%q bytes=%d", test.path, len(test.body)) + } + digest := sha256.Sum256(test.body) + if !strings.Contains(test.path, hex.EncodeToString(digest[:8])) { + t.Fatalf("asset path %q does not contain content digest", test.path) + } + body, contentType, ok := Asset(test.path) + if !ok || contentType != test.contentType || string(body) != string(test.body) { + t.Fatalf("asset lookup ok=%v type=%q", ok, contentType) + } + }) + } + if _, _, ok := Asset("/assets/site.css"); ok { + t.Fatal("unversioned asset path was accepted") + } + css := string(style) + for _, required := range []string{"@media(max-width:24rem)", "@media(prefers-reduced-motion:reduce)", "@media(forced-colors:active)", "@media print", ".table-scroll", ".explore-layout", ".query-stats"} { + if !strings.Contains(css, required) { + t.Fatalf("responsive CSS missing %q", required) + } + } + if strings.Contains(strings.ToLower(css), "position:sticky") { + t.Fatal("mobile-hostile sticky table behavior returned") + } +} diff --git a/internal/site/dashboard.sando b/internal/site/dashboard.sando new file mode 100644 index 0000000..29bc5f1 --- /dev/null +++ b/internal/site/dashboard.sando @@ -0,0 +1,109 @@ + + + + + + + + +
+ +

· dashboard

Export JSON

+
+

This dashboard has no panels yet.

+ +
+

+

+ 0 { ?> +
+
+
    + +
    2. + +
+
+ +
Query
+ +
+ +
+ +
+

Dashboard workshop

Revise this view

Every change includes revision . If someone else saves first, Observatory asks you to reload instead of overwriting their work.

+
+
+

Dashboard details

+ + + + + + + + +
+ +
+

Panel

+
+ + + + + + + + + + +
+
+ Remove this panel +

This removes the panel from the dashboard definition. The saved query remains available.

+
+ + + + + +
+
+
+ + +
+

Add a panel

+ + + + + + + + + + +
+ +
+
+ +
+ + + diff --git a/internal/site/dashboard.sando.go b/internal/site/dashboard.sando.go new file mode 100644 index 0000000..a2d9fdd --- /dev/null +++ b/internal/site/dashboard.sando.go @@ -0,0 +1,735 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 5a4729bb7867878ea92e49dd8dc66f388fe8fd255ab32292283ca573d17153d9 + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func Dashboard(view DashboardView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/dashboard.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:9:11 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (Head(view.Head))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:9:29 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n Skip to content\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:12:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteHeader(true, "dashboards", view.Organization.ID))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:12:62 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n \n

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:15:59 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Organization.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:15:84 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, " · dashboard

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:15:109 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:15:121 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:15:133 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Description)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:15:152 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Export JSON

\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:17:8 + if len(view.Panels) == 0 { +//line internal/site/dashboard.sando:17:37 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

This dashboard has no panels yet.

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:17:80 + } +//line internal/site/dashboard.sando:17:84 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:18:8 + for _, panel := range view.Panels { +//line internal/site/dashboard.sando:18:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:20:32 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (panel.Visualization)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:20:54 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:20:66 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (panel.Title)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:20:80 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:21:12 + if panel.Stat != "" { +//line internal/site/dashboard.sando:21:36 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:21:87 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (panel.Stat)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:21:100 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:21:107 + } +//line internal/site/dashboard.sando:21:111 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:22:12 + if len(panel.Chart.Points) > 0 { +//line internal/site/dashboard.sando:22:47 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:24:27 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (panel.Chart.Label)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:24:47 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
\n
    \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:26:14 + for _, point := range panel.Chart.Points { +//line internal/site/dashboard.sando:26:59 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
  1. "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:27:47 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (point.Label)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:27:61 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ""); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:27:141 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (point.Display)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:27:157 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ""); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:27:195 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (point.Display)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:27:211 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
    2. \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:28:14 + } +//line internal/site/dashboard.sando:28:18 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:31:12 + } +//line internal/site/dashboard.sando:31:16 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
Query
"); __himesan_error != nil {
+				return __himesan_error
+			}
+//line internal/site/dashboard.sando:32:57
+			if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (panel.Query)); __himesan_error != nil {
+				return __himesan_error
+			}
+//line internal/site/dashboard.sando:32:71
+			if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:33:13 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (ResultTable(panel.Table))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:33:40 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:35:8 + } +//line internal/site/dashboard.sando:35:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:37:8 + if view.CanManage { +//line internal/site/dashboard.sando:37:30 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Dashboard workshop

Revise this view

Every change includes revision "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:39:136 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Revision)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:39:152 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ". If someone else saves first, Observatory asks you to reload instead of overwriting their work.

\n
\n
\n

Dashboard details

\n \n \n \n \n \n \n \n \n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:52:12 + for index, panel := range view.Panels { +//line internal/site/dashboard.sando:52:54 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Panel "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:54:56 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (index + 1)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:54:68 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

\n
\n \n \n \n \n \n \n \n \n \n \n
\n
\n Remove this panel\n

This removes the panel from the dashboard definition. The saved query remains available.

\n
\n \n \n \n \n \n
\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:87:12 + } +//line internal/site/dashboard.sando:87:16 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:88:12 + if len(view.Panels) < 16 { +//line internal/site/dashboard.sando:88:41 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Add a panel

\n \n \n \n \n \n \n \n \n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:100:14 + if len(view.SavedQueries) == 0 { +//line internal/site/dashboard.sando:100:49 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ""); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:100:101 + } else { +//line internal/site/dashboard.sando:100:112 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ""); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:100:155 + } +//line internal/site/dashboard.sando:100:159 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:102:12 + } +//line internal/site/dashboard.sando:102:16 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:105:8 + } +//line internal/site/dashboard.sando:105:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:107:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteFooter())); __himesan_error != nil { + return __himesan_error + } +//line internal/site/dashboard.sando:107:22 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/explore.sando b/internal/site/explore.sando new file mode 100644 index 0000000..ac745a5 --- /dev/null +++ b/internal/site/explore.sando @@ -0,0 +1,60 @@ + + + + + + + + +
+ +
+

Query workshop ·

Follow the evidence.

Welcome, . Write one bounded query, run it on the server, and inspect both the result and its cost.

+
+ +
+
+

Typed query

+

Ask a bounded question

+

Queries stay out of the URL and access log. The same parser, authorization scope, sensitivity rules, and execution limits used by the API are applied here.

+
+
+
+
+
+
+
+
+ + + +

Begin with logs, metrics, traces, or deployments. Every query has bounded time, rows, bytes, and memory.

+ +
+
+ + +
+

Authorized result

Query results

+
+
Scanned rows
+
Matched rows
+
Scanned bytes
+
Execution
+
+
+

Result limit reached. Refine the query or deliberately choose a different bounded limit.

+

Approximate result. This answer uses an authorized aggregate projection.

+ +
+ +
+ + + diff --git a/internal/site/explore.sando.go b/internal/site/explore.sando.go new file mode 100644 index 0000000..f062455 --- /dev/null +++ b/internal/site/explore.sando.go @@ -0,0 +1,269 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 970611f5a27597a2257f7ce7f38bc1184b99fa0801b9b0e3d09ea8bd53986827 + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func Explore(view ExploreView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/explore.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:9:11 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (Head(view.Head))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:9:29 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n Skip to content\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:12:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteHeader(true, "explore", view.Organization.ID))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:12:59 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n \n
\n

Query workshop · "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:16:53 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Organization.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:16:78 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Follow the evidence.

Welcome, "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:16:146 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.DisplayName)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:16:165 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ". Write one bounded query, run it on the server, and inspect both the result and its cost.

\n
\n \n
\n
\n

Typed query

\n

Ask a bounded question

\n

Queries stay out of the URL and access log. The same parser, authorization scope, sensitivity rules, and execution limits used by the API are applied here.

\n
\n
\n
\n
\n
\n
\n
\n
\n \n \n \n

Begin with logs, metrics, traces, or deployments. Every query has bounded time, rows, bytes, and memory.

\n \n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:41:8 + if view.ErrorMessage != "" { +//line internal/site/explore.sando:41:39 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:41:78 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.ErrorMessage)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:41:98 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:41:105 + } +//line internal/site/explore.sando:41:109 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:42:8 + if view.Executed { +//line internal/site/explore.sando:42:29 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Authorized result

Query results

\n
\n
Scanned rows
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:46:45 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Stats.ScannedRows)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:46:70 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
\n
Matched rows
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:47:45 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Stats.MatchedRows)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:47:70 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
\n
Scanned bytes
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:48:46 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Stats.ScannedBytes)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:48:72 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
\n
Execution
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:49:42 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Stats.Duration)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:49:64 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:52:10 + if view.Stats.Truncated { +//line internal/site/explore.sando:52:38 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Result limit reached. Refine the query or deliberately choose a different bounded limit.

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:52:173 + } +//line internal/site/explore.sando:52:177 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:53:10 + if view.Stats.Approximate { +//line internal/site/explore.sando:53:40 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Approximate result. This answer uses an authorized aggregate projection.

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:53:159 + } +//line internal/site/explore.sando:53:163 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:54:11 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (ResultTable(view.Table))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:54:37 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:56:8 + } +//line internal/site/explore.sando:56:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:58:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteFooter())); __himesan_error != nil { + return __himesan_error + } +//line internal/site/explore.sando:58:22 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/head.sando b/internal/site/head.sando new file mode 100644 index 0000000..a166551 --- /dev/null +++ b/internal/site/head.sando @@ -0,0 +1,14 @@ + + + + + + + + + +<?= view.Title ?> diff --git a/internal/site/head.sando.go b/internal/site/head.sando.go new file mode 100644 index 0000000..9c755a8 --- /dev/null +++ b/internal/site/head.sando.go @@ -0,0 +1,69 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 131f6248343bf4c87cd3a2a525a2050cee13ad29a2e178d5ef4e996c65590dc9 + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func Head(view HeadView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/head.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/head.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n\n\n\n\n\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/head.sando:14:12 + if __himesan_error := __himesan_sando.WriteRCDATA(__himesan_writer, (view.Title)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/head.sando:14:25 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/incidents.sando b/internal/site/incidents.sando new file mode 100644 index 0000000..a60c7b8 --- /dev/null +++ b/internal/site/incidents.sando @@ -0,0 +1,63 @@ + + + + + + + + +
+ +
+

Incident inbox ·

What needs attention?

Welcome, . Rules use bounded saved queries; this inbox keeps the human response durable and reviewable.

+
+ +

Keep a careful offline copy

This is opt-in. The saved copy excludes response controls, CSRF material, query text, telemetry values, actors, and project, environment, or service identifiers.

Offline saving requires service-worker support.

+

Ask this browser to nudge you

Optional Web Push sends one fixed message through the browser vendor: “Gamertan Observatory needs your attention.” It never includes an organization, host, service, severity, rule, incident identifier, count, or telemetry text.

Push requires service-worker and notification support.

+
+

Durable state

Incidents

+
+

No incidents have been opened.

+ +
+

·

+
Started
Updated
Silenced until
+ +
+
+
+
+
+ +
+ +
+
+
+

Bounded evaluation

Alert rules

+
+

No alert rules yet.

+

Interval
Status
EnabledDisabled
Last evaluated
Last result
Query unavailable
+
+
+ +
+

Organization workshop

Create an alert rule

A rule counts rows returned by one saved query. Consecutive matches allow brief noise to remain pending before an incident fires.

+
+ + + + +
+ +
+
+ +
+ + + diff --git a/internal/site/incidents.sando.go b/internal/site/incidents.sando.go new file mode 100644 index 0000000..034aaf8 --- /dev/null +++ b/internal/site/incidents.sando.go @@ -0,0 +1,529 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 df123546e9ef420a7ce574a528a984505176d15c8e6233582468bde1e5bc9ea6 + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func IncidentInbox(view IncidentInboxView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/incidents.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:9:11 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (Head(view.Head))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:9:29 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n Skip to content\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:12:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteHeader(true, "incidents", view.Organization.ID))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:12:61 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n \n
\n

Incident inbox · "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:16:53 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Organization.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:16:78 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

What needs attention?

Welcome, "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:16:148 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.DisplayName)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:16:167 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ". Rules use bounded saved queries; this inbox keeps the human response durable and reviewable.

\n
\n \n

Keep a careful offline copy

This is opt-in. The saved copy excludes response controls, CSRF material, query text, telemetry values, actors, and project, environment, or service identifiers.

Offline saving requires service-worker support.

\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:20:8 + if view.PushPublicKey != "" { +//line internal/site/incidents.sando:20:40 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Ask this browser to nudge you

Optional Web Push sends one fixed message through the browser vendor: “Gamertan Observatory needs your attention.” It never includes an organization, host, service, severity, rule, incident identifier, count, or telemetry text.

Push requires service-worker and notification support.

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:20:829 + } +//line internal/site/incidents.sando:20:833 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Durable state

Incidents

\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:24:10 + if len(view.Incidents) == 0 { +//line internal/site/incidents.sando:24:42 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

No incidents have been opened.

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:24:82 + } +//line internal/site/incidents.sando:24:86 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:25:10 + for _, incident := range view.Incidents { +//line internal/site/incidents.sando:25:54 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:27:39 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.Severity)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:27:59 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, " · "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:27:67 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.State)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:27:84 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:27:96 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.Title)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:27:113 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

\n
Started
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:28:44 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.StartedAt)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:28:65 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
Updated
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:28:105 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.UpdatedAt)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:28:126 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:28:140 + if incident.SilencedUntil != "" { +//line internal/site/incidents.sando:28:176 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
Silenced until
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:28:212 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.SilencedUntil)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:28:237 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:28:251 + } +//line internal/site/incidents.sando:28:255 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:29:14 + if view.CanManage && incident.State != "resolved" { +//line internal/site/incidents.sando:29:68 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n
\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:35:14 + } +//line internal/site/incidents.sando:35:18 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:37:10 + } +//line internal/site/incidents.sando:37:14 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n
\n

Bounded evaluation

Alert rules

\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:43:10 + if len(view.Rules) == 0 { +//line internal/site/incidents.sando:43:38 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

No alert rules yet.

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:43:67 + } +//line internal/site/incidents.sando:43:71 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:10 + for _, rule := range view.Rules { +//line internal/site/incidents.sando:44:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:78 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (rule.Severity)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:94 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:106 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (rule.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:118 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:130 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (rule.Description)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:149 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Interval
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:187 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (rule.Interval)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:203 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
Status
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:241 + if rule.Enabled { +//line internal/site/incidents.sando:44:261 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "Enabled"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:271 + } else { +//line internal/site/incidents.sando:44:282 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "Disabled"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:293 + } +//line internal/site/incidents.sando:44:297 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:311 + if rule.LastEvaluatedAt != "" { +//line internal/site/incidents.sando:44:345 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
Last evaluated
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:381 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (rule.LastEvaluatedAt)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:404 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:418 + } +//line internal/site/incidents.sando:44:425 + if rule.LastError != "" { +//line internal/site/incidents.sando:44:453 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
Last result
Query unavailable
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:513 + } +//line internal/site/incidents.sando:44:517 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:44:535 + } +//line internal/site/incidents.sando:44:539 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:47:8 + if view.CanManage { +//line internal/site/incidents.sando:47:30 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Organization workshop

Create an alert rule

A rule counts rows returned by one saved query. Consecutive matches allow brief noise to remain pending before an incident fires.

\n
\n \n \n \n \n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:56:12 + if len(view.SavedQueries) == 0 { +//line internal/site/incidents.sando:56:47 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ""); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:56:107 + } else { +//line internal/site/incidents.sando:56:118 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ""); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:56:169 + } +//line internal/site/incidents.sando:56:173 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:59:8 + } +//line internal/site/incidents.sando:59:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:61:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteFooter())); __himesan_error != nil { + return __himesan_error + } +//line internal/site/incidents.sando:61:22 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/landing.sando b/internal/site/landing.sando new file mode 100644 index 0000000..006ee6c --- /dev/null +++ b/internal/site/landing.sando @@ -0,0 +1,35 @@ + + + + + + + + +
+
+

Logs · metrics · traces · deployments

+

Keep the evidence close.

+

Gamertan Observatory is a self-hosted observability platform being built for small, carefully operated Linux systems. It keeps raw evidence replayable, tenant boundaries explicit, and queries understandable.

+

Open Observatory

+
+
+

Working model

One view without one giant trust boundary.

+
+

Evidence first

Checksummed raw segments are committed before projections acknowledge an ingest. Indexes and dashboards remain rebuildable views.

+

Scoped by the server

Organization and resource identity come from enrolled sources and authenticated grants—not from telemetry payloads or query text.

+

Bounded by default

Ingestion, storage, queries, subscribers, and fields have explicit limits. Sensitive data requires a separate permission.

+
+
+
+

Current status

A private development preview.

+

The foundations are under active development and are not a public release yet. Logs, metrics, traces, deployment records, organizations, typed queries, schema activation, and persisted dashboard definitions are being integrated before the first preview.

The interface remains useful without JavaScript. Progressive live updates add a small convenience; they do not own navigation, querying, or incident history.

+
+
+ + + diff --git a/internal/site/landing.sando.go b/internal/site/landing.sando.go new file mode 100644 index 0000000..dd14bd3 --- /dev/null +++ b/internal/site/landing.sando.go @@ -0,0 +1,53 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 7a68dcec190a95c15b15d7bba50a9ba2a067a270c7d1f00b0349f4fd3784ba3b + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func Landing(view LandingView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/landing.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/landing.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/landing.sando:9:11 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (Head(view.Head))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/landing.sando:9:29 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n Skip to content\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/landing.sando:12:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteHeader(false, "", ""))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/landing.sando:12:35 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n

Logs · metrics · traces · deployments

\n

Keep the evidence close.

\n

Gamertan Observatory is a self-hosted observability platform being built for small, carefully operated Linux systems. It keeps raw evidence replayable, tenant boundaries explicit, and queries understandable.

\n

Open Observatory

\n
\n
\n

Working model

One view without one giant trust boundary.

\n
\n

Evidence first

Checksummed raw segments are committed before projections acknowledge an ingest. Indexes and dashboards remain rebuildable views.

\n

Scoped by the server

Organization and resource identity come from enrolled sources and authenticated grants—not from telemetry payloads or query text.

\n

Bounded by default

Ingestion, storage, queries, subscribers, and fields have explicit limits. Sensitive data requires a separate permission.

\n
\n
\n
\n

Current status

A private development preview.

\n

The foundations are under active development and are not a public release yet. Logs, metrics, traces, deployment records, organizations, typed queries, schema activation, and persisted dashboard definitions are being integrated before the first preview.

The interface remains useful without JavaScript. Progressive live updates add a small convenience; they do not own navigation, querying, or incident history.

\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/landing.sando:33:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteFooter())); __himesan_error != nil { + return __himesan_error + } +//line internal/site/landing.sando:33:22 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/login.sando b/internal/site/login.sando new file mode 100644 index 0000000..7d19891 --- /dev/null +++ b/internal/site/login.sando @@ -0,0 +1,31 @@ + + + + + + + + +
+
+

Private workshop

+

Sign in to Observatory

+

Use the local account created by observatory admin bootstrap.

+ +
+ + + + + + +
+
+
+ + + diff --git a/internal/site/login.sando.go b/internal/site/login.sando.go new file mode 100644 index 0000000..1455867 --- /dev/null +++ b/internal/site/login.sando.go @@ -0,0 +1,81 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 dd75f02614b8688552456737ff2bfd5e9625897254989ab8ae2098b5fd5fccc9 + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func Login(view LoginView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/login.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:9:11 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (Head(view.Head))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:9:29 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n Skip to content\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:12:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteHeader(false, "", ""))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:12:35 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n

Private workshop

\n

Sign in to Observatory

\n

Use the local account created by observatory admin bootstrap.

\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:18:10 + if view.ErrorMessage != "" { +//line internal/site/login.sando:18:41 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:18:80 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.ErrorMessage)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:18:100 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:18:107 + } +//line internal/site/login.sando:18:111 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n \n \n \n \n \n \n
\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:29:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteFooter())); __himesan_error != nil { + return __himesan_error + } +//line internal/site/login.sando:29:22 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/offline.sando b/internal/site/offline.sando new file mode 100644 index 0000000..51eb5e6 --- /dev/null +++ b/internal/site/offline.sando @@ -0,0 +1,7 @@ + + +

Offline

The evidence is still safe.

Observatory cannot reach the server right now. A previously saved incident inbox remains available from its usual address; otherwise, reconnect before viewing private organization data.

Try Observatory again

diff --git a/internal/site/offline.sando.go b/internal/site/offline.sando.go new file mode 100644 index 0000000..9d6edd7 --- /dev/null +++ b/internal/site/offline.sando.go @@ -0,0 +1,53 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 00fe79c601abcfc98b766fbb7332552b31cc92968251972bd24715f990812f09 + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func Offline(view OfflineView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/offline.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline.sando:7:42 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (Head(view.Head))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline.sando:7:60 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "Skip to content"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline.sando:7:130 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteHeader(false, "", ""))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline.sando:7:158 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Offline

The evidence is still safe.

Observatory cannot reach the server right now. A previously saved incident inbox remains available from its usual address; otherwise, reconnect before viewing private organization data.

Try Observatory again

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline.sando:7:561 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteFooter())); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline.sando:7:576 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/offline_incidents.sando b/internal/site/offline_incidents.sando new file mode 100644 index 0000000..b62ccbd --- /dev/null +++ b/internal/site/offline_incidents.sando @@ -0,0 +1,24 @@ + + + + + + + + +
+ +

Offline snapshot ·

Saved incident inbox

Captured . Response actions are intentionally unavailable offline.

+

Read-only

Incidents

+

No open incidents were present in this snapshot.

+

·

Started
Updated
Silenced until
+
+

Privacy boundary

This copy stays on this browser.

It contains the organization name, incident titles, and states, but no response controls, CSRF tokens, query text, telemetry values, actor identifiers, or project, environment, or service identifiers. Signing out while online clears saved private snapshots.

Reconnect and refresh

+
+ + + diff --git a/internal/site/offline_incidents.sando.go b/internal/site/offline_incidents.sando.go new file mode 100644 index 0000000..ac4232b --- /dev/null +++ b/internal/site/offline_incidents.sando.go @@ -0,0 +1,169 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 47e648ceed0893244cf41692e7a3d7ecc66a0cba09554e45b0d3dda3567bdfd7 + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func OfflineIncidentInbox(view OfflineIncidentView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/offline_incidents.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:9:11 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (Head(view.Head))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:9:29 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n Skip to content\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:12:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteHeader(true, "incidents", view.Organization.ID))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:12:61 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n \n

Offline snapshot · "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:15:121 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Organization.Name)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:15:146 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Saved incident inbox

Captured "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:15:223 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.CapturedAt)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:15:241 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ". Response actions are intentionally unavailable offline.

\n

Read-only

Incidents

\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:17:10 + if len(view.Incidents) == 0 { +//line internal/site/offline_incidents.sando:17:42 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

No open incidents were present in this snapshot.

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:17:100 + } +//line internal/site/offline_incidents.sando:17:104 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:10 + for _, incident := range view.Incidents { +//line internal/site/offline_incidents.sando:18:54 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:143 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.Severity)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:163 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, " · "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:171 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.State)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:188 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:200 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.Title)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:217 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

Started
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:255 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.StartedAt)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:276 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
Updated
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:316 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.UpdatedAt)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:337 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:351 + if incident.SilencedUntil != "" { +//line internal/site/offline_incidents.sando:18:387 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
Silenced until
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:423 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (incident.SilencedUntil)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:448 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:462 + } +//line internal/site/offline_incidents.sando:18:466 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:18:484 + } +//line internal/site/offline_incidents.sando:18:488 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Privacy boundary

This copy stays on this browser.

It contains the organization name, incident titles, and states, but no response controls, CSRF tokens, query text, telemetry values, actor identifiers, or project, environment, or service identifiers. Signing out while online clears saved private snapshots.

Reconnect and refresh

\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:22:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteFooter())); __himesan_error != nil { + return __himesan_error + } +//line internal/site/offline_incidents.sando:22:22 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/password.sando b/internal/site/password.sando new file mode 100644 index 0000000..785897d --- /dev/null +++ b/internal/site/password.sando @@ -0,0 +1,33 @@ + + + + + + + + +
+
+

One careful first step

+

Choose your password

+

The bootstrap credential is single-purpose. Replace it now; Observatory will end every session and ask you to sign in again.

+ +
+ + + + + + + + +
+
+
+ + + diff --git a/internal/site/password.sando.go b/internal/site/password.sando.go new file mode 100644 index 0000000..1c5e2fb --- /dev/null +++ b/internal/site/password.sando.go @@ -0,0 +1,81 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 75a8b350f1635bc7a9ba4feb50aa4bc3b151783a3ebee72fc0e1947f7787c320 + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func Password(view PasswordView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/password.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:9:11 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (Head(view.Head))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:9:29 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n Skip to content\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:12:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteHeader(false, "", ""))); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:12:35 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n
\n

One careful first step

\n

Choose your password

\n

The bootstrap credential is single-purpose. Replace it now; Observatory will end every session and ask you to sign in again.

\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:18:10 + if view.ErrorMessage != "" { +//line internal/site/password.sando:18:41 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:18:80 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.ErrorMessage)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:18:100 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "

"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:18:107 + } +//line internal/site/password.sando:18:111 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n \n \n \n \n \n \n \n \n
\n
\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:31:7 + if __himesan_error := __himesan_sando.Render(__himesan_render_context, __himesan_writer, (SiteFooter())); __himesan_error != nil { + return __himesan_error + } +//line internal/site/password.sando:31:22 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n\n\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/pwa.go b/internal/site/pwa.go new file mode 100644 index 0000000..e338899 --- /dev/null +++ b/internal/site/pwa.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package site + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" +) + +func WebManifest() []byte { + assets := AssetPaths() + return []byte(fmt.Sprintf(`{"id":"/app/","name":"Gamertan Observatory","short_name":"Observatory","description":"A self-hosted, organization-aware observability workshop.","start_url":"/app/","scope":"/","display":"standalone","background_color":"#111715","theme_color":"#111715","icons":[{"src":%q,"sizes":"any","type":"image/svg+xml","purpose":"any maskable"}]}`+"\n", assets.IconPath)) +} + +func ServiceWorker() []byte { + assets := AssetPaths() + revisionSource := append(append(append([]byte(nil), style...), script...), icon...) + revision := sha256.Sum256(revisionSource) + return []byte(fmt.Sprintf(`/* SPDX-License-Identifier: AGPL-3.0-only */ +"use strict"; +const SHELL_CACHE = %q; +const PRIVATE_CACHE = "observatory-private-v1"; +const PUSH_MESSAGE = %q; +const PUSH_ICON = %q; +const SHELL = [%q,%q,%q,%q]; +self.addEventListener("install", event => { + event.waitUntil(caches.open(SHELL_CACHE).then(cache => cache.addAll(SHELL))); + self.skipWaiting(); +}); +self.addEventListener("activate", event => { + event.waitUntil((async () => { + for (const name of await caches.keys()) { + if (name.startsWith("observatory-shell-") && name !== SHELL_CACHE) await caches.delete(name); + } + await self.clients.claim(); + })()); +}); +self.addEventListener("fetch", event => { + const request = event.request; + if (request.method !== "GET") return; + const url = new URL(request.url); + if (url.origin !== self.location.origin) return; + if (request.mode === "navigate") { + event.respondWith((async () => { + try { + const response = await fetch(request); + if (response.status < 500) return response; + const saved = await caches.open(PRIVATE_CACHE).then(cache => cache.match(request)); + return saved || response; + } catch (_) { + const saved = await caches.open(PRIVATE_CACHE).then(cache => cache.match(request)); + return saved || await caches.match("/offline/"); + } + })()); + return; + } + if (SHELL.includes(url.pathname)) { + event.respondWith(caches.match(request).then(saved => saved || fetch(request))); + } +}); +self.addEventListener("message", event => { + const reply = value => { if (event.ports[0]) event.ports[0].postMessage(value); }; + if (!event.data || typeof event.data.type !== "string") return; + if (event.data.type === "clear-private") { + event.waitUntil(caches.delete(PRIVATE_CACHE).then(() => reply({ok:true}))); + return; + } + if (event.data.type !== "cache-inbox") return; + event.waitUntil((async () => { + try { + const source = new URL(event.data.source, self.location.origin); + const target = new URL(event.data.target, self.location.origin); + const valid = source.origin === self.location.origin && target.origin === self.location.origin && + source.pathname === "/app/incidents/offline/" && target.pathname === "/app/incidents/" && + source.search === target.search && source.searchParams.size === 1 && source.searchParams.has("organization"); + if (!valid) throw new Error("invalid inbox cache request"); + const response = await fetch(source.href, {credentials:"include",cache:"no-store"}); + if (!response.ok || !(response.headers.get("content-type") || "").startsWith("text/html")) throw new Error("offline inbox unavailable"); + const cache = await caches.open(PRIVATE_CACHE); + await cache.put(new Request(target.href, {method:"GET"}), response); + reply({ok:true}); + } catch (_) { reply({ok:false}); } + })()); +}); +self.addEventListener("push", event => { + event.waitUntil(self.registration.showNotification(PUSH_MESSAGE, {icon:PUSH_ICON,badge:PUSH_ICON,tag:"observatory-attention",renotify:true,data:{url:"/app/"}})); +}); +self.addEventListener("notificationclick", event => { + event.notification.close(); + event.waitUntil((async () => { + for (const client of await self.clients.matchAll({type:"window",includeUncontrolled:true})) { + if (new URL(client.url).origin === self.location.origin) { + await client.navigate("/app/"); + return client.focus(); + } + } + return self.clients.openWindow("/app/"); + })()); +}); +`, "observatory-shell-"+hex.EncodeToString(revision[:8]), "Gamertan Observatory needs your attention.", assets.IconPath, "/offline/", assets.StylePath, assets.ScriptPath, assets.IconPath)) +} diff --git a/internal/site/pwa_test.go b/internal/site/pwa_test.go new file mode 100644 index 0000000..ae61575 --- /dev/null +++ b/internal/site/pwa_test.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package site + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestManifestAndServiceWorkerUseExactContentAddressedShell(t *testing.T) { + var manifest struct { + ID string `json:"id"` + Name string `json:"name"` + StartURL string `json:"start_url"` + Scope string `json:"scope"` + Display string `json:"display"` + Icons []struct { + Source string `json:"src"` + Sizes string `json:"sizes"` + Type string `json:"type"` + Purpose string `json:"purpose"` + } `json:"icons"` + } + if err := json.Unmarshal(WebManifest(), &manifest); err != nil { + t.Fatal(err) + } + assets := AssetPaths() + if manifest.ID != "/app/" || manifest.Name != "Gamertan Observatory" || manifest.StartURL != "/app/" || manifest.Scope != "/" || manifest.Display != "standalone" || len(manifest.Icons) != 1 || manifest.Icons[0].Source != assets.IconPath || manifest.Icons[0].Sizes != "any" || manifest.Icons[0].Type != "image/svg+xml" || manifest.Icons[0].Purpose != "any maskable" { + t.Fatalf("manifest=%+v", manifest) + } + + worker := string(ServiceWorker()) + for _, required := range []string{"/offline/", assets.StylePath, assets.ScriptPath, assets.IconPath, "cache-inbox", "clear-private", "credentials:\"include\"", "source.pathname === \"/app/incidents/offline/\"", "target.pathname === \"/app/incidents/\"", "Gamertan Observatory needs your attention.", `self.addEventListener("push"`, `self.addEventListener("notificationclick"`, `data:{url:"/app/"}`} { + if !strings.Contains(worker, required) { + t.Fatalf("service worker missing %q", required) + } + } + for _, forbidden := range []string{"query_text", "csrf_token", "telemetry", "incident.title", "organization_id", "service_id", "severity", "rule_id", "console.log", "eval("} { + if strings.Contains(worker, forbidden) { + t.Fatalf("service worker contains forbidden %q", forbidden) + } + } + if strings.Count(worker, "Gamertan Observatory needs your attention.") != 1 { + t.Fatal("service worker did not contain exactly one fixed notification message") + } + if first, second := string(ServiceWorker()), string(ServiceWorker()); first != second { + t.Fatal("service worker output is not deterministic") + } +} diff --git a/internal/site/result_table.sando b/internal/site/result_table.sando new file mode 100644 index 0000000..e4e71ee --- /dev/null +++ b/internal/site/result_table.sando @@ -0,0 +1,24 @@ + + +
+ + + + + + + + + + + + + + + +
()
+
diff --git a/internal/site/result_table.sando.go b/internal/site/result_table.sando.go new file mode 100644 index 0000000..4cbb4a4 --- /dev/null +++ b/internal/site/result_table.sando.go @@ -0,0 +1,141 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 bf3381b912a1264323672beac6aeed2a2ddb770904404f9597645ac09d2981ae + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func ResultTable(view TableView) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/result_table.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n \n \n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:11:8 + for _, column := range view.Columns { +//line internal/site/result_table.sando:11:48 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:13:8 + } +//line internal/site/result_table.sando:13:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n \n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:16:8 + for _, row := range view.Rows { +//line internal/site/result_table.sando:16:42 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:17:14 + for _, value := range row.Values { +//line internal/site/result_table.sando:17:51 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ""); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:17:75 + } +//line internal/site/result_table.sando:17:79 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:18:8 + } +//line internal/site/result_table.sando:18:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:19:8 + if len(view.Rows) == 0 { +//line internal/site/result_table.sando:19:35 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:21:8 + } +//line internal/site/result_table.sando:21:12 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n \n
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:9:18 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Caption)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:9:33 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:12:27 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (column.Label)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:12:45 + if column.Unit != "" { +//line internal/site/result_table.sando:12:70 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, " ("); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:12:95 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (column.Unit)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:12:109 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, ")"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:12:120 + } +//line internal/site/result_table.sando:12:124 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:17:59 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (value)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:17:67 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:20:54 + if __himesan_error := __himesan_sando.WriteText(__himesan_writer, (view.Empty)); __himesan_error != nil { + return __himesan_error + } +//line internal/site/result_table.sando:20:67 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "
\n
\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/site_footer.sando b/internal/site/site_footer.sando new file mode 100644 index 0000000..356e9d8 --- /dev/null +++ b/internal/site/site_footer.sando @@ -0,0 +1,9 @@ + + +
+

Self-hosted, evidence-minded observability. Built and tended by Gamertan.

+
diff --git a/internal/site/site_footer.sando.go b/internal/site/site_footer.sando.go new file mode 100644 index 0000000..6692227 --- /dev/null +++ b/internal/site/site_footer.sando.go @@ -0,0 +1,29 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 df9cf0c1ef86d0317606ba4eccb618cfe2d0925ef83dd572df78beaa6d2c254f + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func SiteFooter() __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/site_footer.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/site_footer.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n

Self-hosted, evidence-minded observability. Built and tended by Gamertan.

\n
\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/site_header.sando b/internal/site/site_header.sando new file mode 100644 index 0000000..1cd1d8a --- /dev/null +++ b/internal/site/site_header.sando @@ -0,0 +1,20 @@ + + +
+ + Gamertan Observatory + + + Gamertan Observatory + + +
diff --git a/internal/site/site_header.sando.go b/internal/site/site_header.sando.go new file mode 100644 index 0000000..f46e4e3 --- /dev/null +++ b/internal/site/site_header.sando.go @@ -0,0 +1,191 @@ +// Code generated by himesan; DO NOT EDIT. +// himesan:compiler v1.0.0-beta.2 +// himesan:runtime-abi sando.v1 +// himesan:source-sha256 5e30e1ad98c586ffdc566d3688d3288d918faf5645804cd18ef6233a1e250e6b + +package site + +import ( + __himesan_context "context" + __himesan_sando "gamertan.com/sandwich-hime/sando" + __himesan_io "io" +) + +var _ = __himesan_sando.ABISandoV1 + +func SiteHeader(authenticated bool, current string, organizationID string) __himesan_sando.Component { + return __himesan_sando.ComponentFunc(func(__himesan_render_context __himesan_context.Context, __himesan_writer __himesan_io.Writer) error { + _ = __himesan_render_context +//line internal/site/site_header.sando:5:3 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n"); __himesan_error != nil { + return __himesan_error + } +//line internal/site/site_header.sando:6:46 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/site_header.sando:8:6 + if authenticated { +//line internal/site/site_header.sando:8:27 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n Gamertan Observatory\n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/site_header.sando:16:6 + } else { +//line internal/site/site_header.sando:16:17 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n Gamertan Observatory\n \n "); __himesan_error != nil { + return __himesan_error + } +//line internal/site/site_header.sando:19:6 + } +//line internal/site/site_header.sando:19:10 + if __himesan_error := __himesan_sando.WriteString(__himesan_writer, "\n
\n"); __himesan_error != nil { + return __himesan_error + } + return nil + }) +} diff --git a/internal/site/views.go b/internal/site/views.go new file mode 100644 index 0000000..83f3f8f --- /dev/null +++ b/internal/site/views.go @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package site + +type HeadView struct { + Title string + Description string + CanonicalURL string + Assets Assets +} + +type LandingView struct{ Head HeadView } + +type LoginView struct { + Head HeadView + CSRFToken string + ErrorMessage string +} + +type PasswordView struct { + Head HeadView + CSRFToken string + ErrorMessage string +} + +type OrganizationOption struct { + ID string + Name string + Selected bool +} + +type TableColumn struct { + Label string + Unit string +} + +type TableRow struct{ Values []string } + +type TableView struct { + Caption string + Columns []TableColumn + Rows []TableRow + Empty string +} + +type SignalView struct { + ID string + Name string + Description string + Query string + Table TableView +} + +type SavedQuerySummary struct { + ID string + Name string + Description string + Query string +} + +type DashboardSummary struct { + Slug string + Name string + Description string + PanelCount int +} + +type AppView struct { + Head HeadView + DisplayName string + Organizations []OrganizationOption + Organization OrganizationOption + Signals []SignalView + SavedQueries []SavedQuerySummary + Dashboards []DashboardSummary + CSRFToken string + ManageCSRF string + CanManage bool + EventsURL string + RefreshedAt string + ProjectionLag string + PendingBatches int + IncidentsURL string + OpenIncidents int +} + +type QueryStatsView struct { + ScannedRows int + MatchedRows int + ScannedBytes string + Duration string + Truncated bool + Approximate bool +} + +type ExploreView struct { + Head HeadView + DisplayName string + Organization OrganizationOption + Query string + CSRFToken string + EventsURL string + Executed bool + ErrorMessage string + Table TableView + Stats QueryStatsView +} + +type IncidentSummary struct { + ID string + Title string + State string + Severity string + StartedAt string + UpdatedAt string + SilencedUntil string +} + +type AlertRuleSummary struct { + Name string + Description string + Severity string + Enabled bool + Interval string + LastEvaluatedAt string + LastError string +} + +type IncidentInboxView struct { + Head HeadView + DisplayName string + Organization OrganizationOption + Incidents []IncidentSummary + Rules []AlertRuleSummary + SavedQueries []SavedQuerySummary + CanManage bool + ManageCSRF string + EventsURL string + OfflineURL string + CacheKey string + OpenCount int + PushPublicKey string + PushCSRF string +} + +type OfflineIncidentView struct { + Head HeadView + Organization OrganizationOption + Incidents []IncidentSummary + CapturedAt string +} + +type OfflineView struct{ Head HeadView } + +type PanelView struct { + ID string + SavedQueryID string + Title string + Visualization string + Query string + Stat string + Chart ChartView + Table TableView +} + +type ChartPoint struct { + Label string + Value string + Maximum string + Display string +} + +type ChartView struct { + Label string + Points []ChartPoint +} + +type DashboardView struct { + Head HeadView + DisplayName string + Organization OrganizationOption + ID string + Slug string + Revision int + Name string + Description string + ExportURL string + Panels []PanelView + SavedQueries []SavedQuerySummary + CanManage bool + ManageCSRF string +} diff --git a/internal/spool/spool.go b/internal/spool/spool.go new file mode 100644 index 0000000..2dc561f --- /dev/null +++ b/internal/spool/spool.go @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package spool + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "gamertan.com/observatory/internal/model" + "github.com/klauspost/compress/zstd" +) + +type Spool struct { + root string + maxBytes int64 + maxAge time.Duration +} + +type Entry struct { + Path string + Digest string + StreamID string + Sequence uint64 + Size int64 + ModTime time.Time +} + +type envelope struct { + Version int `json:"version"` + Batch model.Batch `json:"batch"` + Checkpoint json.RawMessage `json:"checkpoint,omitempty"` +} + +const maxEncodedBatchBytes = 64 << 20 + +func Open(root string, maxBytes int64, maxAge time.Duration) (*Spool, error) { + if !filepath.IsAbs(root) || filepath.Clean(root) != root || maxBytes < 1<<20 || maxBytes > 5<<30 || maxAge < time.Hour || maxAge > 72*time.Hour { + return nil, errors.New("invalid spool configuration") + } + if err := os.MkdirAll(filepath.Join(root, "pending"), 0o700); err != nil { + return nil, fmt.Errorf("create spool: %w", err) + } + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return nil, errors.New("spool root must be a private non-symlink directory") + } + return &Spool{root: root, maxBytes: maxBytes, maxAge: maxAge}, nil +} + +func (s *Spool) Put(batch model.Batch, now time.Time) (Entry, error) { + return s.PutWithCheckpoint(batch, nil, now) +} + +func (s *Spool) PutWithCheckpoint(batch model.Batch, checkpoint []byte, now time.Time) (Entry, error) { + if err := batch.Validate(now); err != nil { + return Entry{}, err + } + if len(checkpoint) > 4096 || len(checkpoint) != 0 && !json.Valid(checkpoint) { + return Entry{}, errors.New("spool checkpoint is invalid") + } + raw, err := json.Marshal(envelope{Version: 1, Batch: batch, Checkpoint: checkpoint}) + if err != nil { + return Entry{}, err + } + if len(raw) > maxEncodedBatchBytes { + return Entry{}, errors.New("spool batch exceeds encoded size limit") + } + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) + if err != nil { + return Entry{}, err + } + compressed := encoder.EncodeAll(raw, nil) + encoder.Close() + entries, err := s.List(now) + if err != nil { + return Entry{}, err + } + var used int64 + for _, entry := range entries { + used += entry.Size + } + if used+int64(len(compressed)) > s.maxBytes { + return Entry{}, errors.New("agent spool quota exhausted") + } + sum := sha256.Sum256(compressed) + digest := hex.EncodeToString(sum[:]) + dir := filepath.Join(s.root, "pending", batch.StreamID) + if err := os.MkdirAll(dir, 0o700); err != nil { + return Entry{}, err + } + final := filepath.Join(dir, fmt.Sprintf("%020d-%s.zst", batch.Sequence, digest)) + if info, err := os.Lstat(final); err == nil { + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return Entry{}, errors.New("existing spool batch is not a regular file") + } + existing, err := os.ReadFile(final) + if err != nil || !bytes.Equal(existing, compressed) { + return Entry{}, errors.New("existing spool batch does not match content") + } + return Entry{Path: final, Digest: digest, StreamID: batch.StreamID, Sequence: batch.Sequence, Size: info.Size(), ModTime: info.ModTime()}, nil + } else if !errors.Is(err, os.ErrNotExist) { + return Entry{}, err + } + tmp, err := os.CreateTemp(dir, ".batch-*") + if err != nil { + return Entry{}, err + } + name := tmp.Name() + defer os.Remove(name) + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return Entry{}, err + } + if _, err := tmp.Write(compressed); err != nil { + _ = tmp.Close() + return Entry{}, err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return Entry{}, err + } + if err := tmp.Close(); err != nil { + return Entry{}, err + } + if err := os.Rename(name, final); err != nil { + return Entry{}, err + } + if err := syncDir(dir); err != nil { + return Entry{}, err + } + return Entry{Path: final, Digest: digest, StreamID: batch.StreamID, Sequence: batch.Sequence, Size: int64(len(compressed)), ModTime: now}, nil +} + +func (s *Spool) List(now time.Time) ([]Entry, error) { + base := filepath.Join(s.root, "pending") + var entries []Entry + err := filepath.WalkDir(base, func(path string, item os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if item.Type()&os.ModeSymlink != 0 { + return errors.New("spool contains a symlink") + } + if item.IsDir() || !strings.HasSuffix(item.Name(), ".zst") { + return nil + } + if !item.Type().IsRegular() { + return errors.New("spool batch is not a regular file") + } + parts := strings.Split(strings.TrimSuffix(item.Name(), ".zst"), "-") + if len(parts) != 2 || len(parts[0]) != 20 || len(parts[1]) != 64 { + return errors.New("spool contains an invalid batch filename") + } + sequence, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return err + } + info, err := item.Info() + if err != nil { + return err + } + if info.Size() < 1 || info.Size() > maxEncodedBatchBytes { + return errors.New("spool batch exceeds compressed size limit") + } + if now.Sub(info.ModTime()) > s.maxAge { + return errors.New("agent spool contains data older than its outage budget") + } + entries = append(entries, Entry{Path: path, Digest: parts[1], StreamID: filepath.Base(filepath.Dir(path)), Sequence: sequence, Size: info.Size(), ModTime: info.ModTime()}) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].StreamID == entries[j].StreamID { + return entries[i].Sequence < entries[j].Sequence + } + return entries[i].StreamID < entries[j].StreamID + }) + return entries, nil +} + +func (s *Spool) Read(entry Entry) (model.Batch, error) { + batch, _, err := s.ReadWithCheckpoint(entry) + return batch, err +} + +func (s *Spool) ReadWithCheckpoint(entry Entry) (model.Batch, []byte, error) { + if !strings.HasPrefix(filepath.Clean(entry.Path), filepath.Join(s.root, "pending")+string(os.PathSeparator)) { + return model.Batch{}, nil, errors.New("spool entry escapes root") + } + info, err := os.Lstat(entry.Path) + if err != nil { + return model.Batch{}, nil, err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Size() < 1 || info.Size() > maxEncodedBatchBytes { + return model.Batch{}, nil, errors.New("spool batch is outside compressed size limit") + } + b, err := os.ReadFile(entry.Path) + if err != nil { + return model.Batch{}, nil, err + } + sum := sha256.Sum256(b) + if hex.EncodeToString(sum[:]) != entry.Digest { + return model.Batch{}, nil, errors.New("spool checksum mismatch") + } + decoder, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(maxEncodedBatchBytes), zstd.WithDecodeAllCapLimit(true)) + if err != nil { + return model.Batch{}, nil, err + } + raw, err := decoder.DecodeAll(b, make([]byte, 0, maxEncodedBatchBytes)) + decoder.Close() + if err != nil || len(raw) > maxEncodedBatchBytes { + return model.Batch{}, nil, errors.New("invalid compressed spool batch") + } + var payload envelope + jsonDecoder := json.NewDecoder(bytes.NewReader(raw)) + jsonDecoder.DisallowUnknownFields() + if err := jsonDecoder.Decode(&payload); err != nil { + return model.Batch{}, nil, err + } + if err := jsonDecoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return model.Batch{}, nil, errors.New("spool batch has trailing JSON") + } + if payload.Version != 1 || len(payload.Checkpoint) > 4096 { + return model.Batch{}, nil, errors.New("spool envelope is invalid") + } + batch := payload.Batch + if batch.StreamID != entry.StreamID || batch.Sequence != entry.Sequence { + return model.Batch{}, nil, errors.New("spool path does not match batch identity") + } + return batch, append([]byte(nil), payload.Checkpoint...), nil +} + +func (s *Spool) Acknowledge(entry Entry, digest string) error { + if digest != entry.Digest { + return errors.New("acknowledgement digest does not match spool entry") + } + if !strings.HasPrefix(filepath.Clean(entry.Path), filepath.Join(s.root, "pending")+string(os.PathSeparator)) { + return errors.New("spool acknowledgement path escapes root") + } + info, err := os.Lstat(entry.Path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("spool acknowledgement target is not a regular file") + } + if err := os.Remove(entry.Path); err != nil { + return err + } + return syncDir(filepath.Dir(entry.Path)) +} + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/internal/spool/spool_test.go b/internal/spool/spool_test.go new file mode 100644 index 0000000..add65b6 --- /dev/null +++ b/internal/spool/spool_test.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package spool + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/model" +) + +func TestSpoolRoundTripAndExactAcknowledgement(t *testing.T) { + root := filepath.Join(t.TempDir(), "spool") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + spool, err := Open(root, 1<<20, 72*time.Hour) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}} + checkpoint := []byte(`{"offset":42,"sequence":1}`) + entry, err := spool.PutWithCheckpoint(batch, checkpoint, now) + if err != nil { + t.Fatal(err) + } + got, err := spool.Read(entry) + if err != nil || got.Sequence != 1 { + t.Fatalf("batch=%+v err=%v", got, err) + } + _, gotCheckpoint, err := spool.ReadWithCheckpoint(entry) + if err != nil || string(gotCheckpoint) != string(checkpoint) { + t.Fatalf("checkpoint=%s err=%v", gotCheckpoint, err) + } + if err := spool.Acknowledge(entry, "wrong"); err == nil { + t.Fatal("expected digest mismatch") + } + outside := filepath.Join(t.TempDir(), "outside") + if err := os.WriteFile(outside, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := spool.Acknowledge(Entry{Path: outside, Digest: entry.Digest}, entry.Digest); err == nil { + t.Fatal("expected acknowledgement path rejection") + } + if err := spool.Acknowledge(entry, entry.Digest); err != nil { + t.Fatal(err) + } + if entries, err := spool.List(now); err != nil || len(entries) != 0 { + t.Fatalf("entries=%+v err=%v", entries, err) + } +} + +func TestSpoolRejectsQuotaAndSymlink(t *testing.T) { + root := filepath.Join(t.TempDir(), "spool") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + spool, err := Open(root, 1<<20, 72*time.Hour) + if err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "pending", "bad") + if err := os.Symlink(t.TempDir(), link); err != nil { + t.Skip(err) + } + if _, err := spool.List(time.Now().UTC()); err == nil { + t.Fatal("expected symlink rejection") + } +} + +func TestSpoolRejectsOversizedFileBeforeRead(t *testing.T) { + root := filepath.Join(t.TempDir(), "spool") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + queue, err := Open(root, 1<<20, 72*time.Hour) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, "pending", "access") + if err = os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "00000000000000000001-"+strings.Repeat("a", 64)+".zst") + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if err = file.Truncate(maxEncodedBatchBytes + 1); err != nil { + file.Close() + t.Fatal(err) + } + if err = file.Close(); err != nil { + t.Fatal(err) + } + if _, err = queue.List(time.Now().UTC()); err == nil || !strings.Contains(err.Error(), "compressed size limit") { + t.Fatalf("expected compressed-size rejection, got %v", err) + } +} diff --git a/internal/storage/activation.go b/internal/storage/activation.go new file mode 100644 index 0000000..9b29e0c --- /dev/null +++ b/internal/storage/activation.go @@ -0,0 +1,445 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" +) + +const maxProjectionVersion = 999999 +const indexedTimeFormat = "2006-01-02T15:04:05.000000000Z" + +type DescriptorActivation struct { + OrganizationID string `json:"organization_id"` + Signal model.Signal `json:"signal"` + Field string `json:"field"` + Previous int `json:"previous_version"` + Active int `json:"active_version"` + IndexedRows int64 `json:"indexed_rows"` + Descriptor schema.Descriptor `json:"descriptor"` +} + +func (s *Store) namedLock(name string) *sync.Mutex { + value, _ := s.locks.LoadOrStore(name, &sync.Mutex{}) + return value.(*sync.Mutex) +} + +func ensureProjectionMetadata(ctx context.Context, db sqlExecutor) error { + statements := []string{ + `CREATE TABLE IF NOT EXISTS projection_versions ( + version INTEGER PRIMARY KEY CHECK(version >= 1 AND version <= 999999), + created_at TEXT NOT NULL, + activated_at TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS projection_state ( + id INTEGER PRIMARY KEY CHECK(id=1), + active_version INTEGER NOT NULL REFERENCES projection_versions(version) + )`, + `CREATE TABLE IF NOT EXISTS projection_descriptors ( + version INTEGER NOT NULL REFERENCES projection_versions(version), + signal TEXT NOT NULL, + field TEXT NOT NULL, + descriptor_json TEXT NOT NULL, + PRIMARY KEY(version,signal,field) + )`, + `INSERT OR IGNORE INTO projection_versions(version,created_at,activated_at) VALUES(1,'1970-01-01T00:00:00Z','1970-01-01T00:00:00Z')`, + `INSERT OR IGNORE INTO projection_state(id,active_version) VALUES(1,1)`, + } + for _, statement := range statements { + if _, err := db.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("migrate projection metadata: %w", err) + } + } + return nil +} + +type sqlExecutor interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) +} + +func activeProjection(ctx context.Context, db interface { + QueryRowContext(context.Context, string, ...any) *sql.Row + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +}) (int, query.MapRegistry, []schema.Descriptor, error) { + var version int + if err := db.QueryRowContext(ctx, `SELECT active_version FROM projection_state WHERE id=1`).Scan(&version); err != nil { + return 0, nil, nil, errors.New("read active projection version") + } + if version < 1 || version > maxProjectionVersion { + return 0, nil, nil, errors.New("active projection version is invalid") + } + rows, err := db.QueryContext(ctx, `SELECT descriptor_json FROM projection_descriptors WHERE version=? ORDER BY signal,field`, version) + if err != nil { + return 0, nil, nil, errors.New("read active projection descriptors") + } + defer rows.Close() + registry := query.MapRegistry{} + var descriptors []schema.Descriptor + for rows.Next() { + if len(descriptors) >= model.MaxDistinctFields { + return 0, nil, nil, errors.New("active projection descriptor limit exceeded") + } + var encoded string + if err = rows.Scan(&encoded); err != nil { + return 0, nil, nil, errors.New("read active projection descriptor") + } + var descriptor schema.Descriptor + if err = json.Unmarshal([]byte(encoded), &descriptor); err != nil || descriptor.Validate() != nil || descriptor.ProjectionVersion != version { + return 0, nil, nil, errors.New("active projection descriptor is invalid") + } + key := string(descriptor.Signal) + ":" + query.CanonicalField(descriptor.Field) + if _, exists := registry[key]; exists { + return 0, nil, nil, errors.New("active projection descriptor is duplicated") + } + registry[key] = descriptor + descriptors = append(descriptors, descriptor) + } + if err = rows.Err(); err != nil { + return 0, nil, nil, errors.New("read active projection descriptors") + } + return version, registry, descriptors, nil +} + +func projectionIndexTable(version int) (string, error) { + if version < 2 || version > maxProjectionVersion { + return "", errors.New("indexed projection version is invalid") + } + return fmt.Sprintf("indexed_fields_v%06d", version), nil +} + +func (s *Store) ActiveDescriptors(ctx context.Context, organizationID string) (query.MapRegistry, int, error) { + if err := model.ValidateSourceID(organizationID); err != nil { + return nil, 0, errors.New("invalid organization identifier") + } + path := filepath.Join(s.root, "organizations", organizationID, "projection.sqlite") + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return query.MapRegistry{}, 1, nil + } + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return nil, 0, errors.New("organization projection is unavailable") + } + dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, 0, errors.New("open organization projection") + } + defer db.Close() + db.SetMaxOpenConns(1) + var metadataTables int + if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='projection_state'`).Scan(&metadataTables); err != nil { + return nil, 0, errors.New("inspect organization projection metadata") + } + if metadataTables == 0 { + return query.MapRegistry{}, 1, nil + } + version, registry, _, err := activeProjection(ctx, db) + return registry, version, err +} + +func (s *Store) ActivateDescriptor(ctx context.Context, organizationID string, reviewed schema.Descriptor, now time.Time) (DescriptorActivation, error) { + if err := model.ValidateSourceID(organizationID); err != nil || reviewed.Validate() != nil || reviewed.ProjectionVersion != 1 || now.IsZero() { + return DescriptorActivation{}, errors.New("descriptor activation input is invalid") + } + proposal, err := s.descriptorProposal(ctx, organizationID, reviewed.Signal, reviewed.Field) + if err != nil { + return DescriptorActivation{}, err + } + if proposal.Status == "rejected" { + return DescriptorActivation{}, errors.New("rejected descriptor proposal cannot be activated") + } + if proposal.Proposal.Descriptor.Signal != reviewed.Signal || proposal.Proposal.Descriptor.Field != reviewed.Field { + return DescriptorActivation{}, errors.New("reviewed descriptor does not match proposal") + } + + lock := s.namedLock("organization:" + organizationID) + lock.Lock() + defer lock.Unlock() + + path := filepath.Join(s.root, "organizations", organizationID, "projection.sqlite") + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return DescriptorActivation{}, errors.New("organization projection is unavailable") + } + db, err := sql.Open("sqlite", path) + if err != nil { + return DescriptorActivation{}, errors.New("open organization projection") + } + defer db.Close() + db.SetMaxOpenConns(1) + for _, statement := range []string{`PRAGMA journal_mode=WAL`, `PRAGMA synchronous=FULL`, `PRAGMA busy_timeout=5000`, `PRAGMA foreign_keys=ON`} { + if _, err = db.ExecContext(ctx, statement); err != nil { + return DescriptorActivation{}, errors.New("configure organization projection") + } + } + if err = ensureProjectionMetadata(ctx, db); err != nil { + return DescriptorActivation{}, err + } + currentVersion, _, current, err := activeProjection(ctx, db) + if err != nil { + return DescriptorActivation{}, err + } + for _, descriptor := range current { + if descriptor.Signal == reviewed.Signal && descriptor.Field == reviewed.Field { + if sameDescriptorIgnoringProjection(descriptor, reviewed) { + if err = s.markProposalActivated(ctx, organizationID, descriptor); err != nil { + return DescriptorActivation{}, err + } + return DescriptorActivation{OrganizationID: organizationID, Signal: descriptor.Signal, Field: descriptor.Field, Previous: currentVersion, Active: currentVersion, Descriptor: descriptor}, nil + } + } + } + if proposal.Status == "activated" { + return DescriptorActivation{}, errors.New("activated descriptor revision requires a new review proposal") + } + controlTx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return DescriptorActivation{}, errors.New("begin descriptor proposal claim") + } + defer controlTx.Rollback() + claim, err := controlTx.ExecContext(ctx, `UPDATE descriptor_proposals SET status=status WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, organizationID, reviewed.Signal, reviewed.Field) + if err != nil { + return DescriptorActivation{}, errors.New("claim descriptor proposal") + } + if changed, _ := claim.RowsAffected(); changed != 1 { + return DescriptorActivation{}, errors.New("descriptor proposal is no longer pending") + } + if currentVersion >= maxProjectionVersion { + return DescriptorActivation{}, errors.New("projection version space exhausted") + } + nextVersion := currentVersion + 1 + reviewed.ProjectionVersion = nextVersion + if err = reviewed.Validate(); err != nil { + return DescriptorActivation{}, errors.New("reviewed descriptor is invalid") + } + byKey := map[string]schema.Descriptor{} + for _, descriptor := range current { + descriptor.ProjectionVersion = nextVersion + byKey[string(descriptor.Signal)+":"+descriptor.Field] = descriptor + } + byKey[string(reviewed.Signal)+":"+reviewed.Field] = reviewed + if len(byKey) > model.MaxDistinctFields { + return DescriptorActivation{}, errors.New("active projection descriptor limit exceeded") + } + next := make([]schema.Descriptor, 0, len(byKey)) + for _, descriptor := range byKey { + next = append(next, descriptor) + } + sort.Slice(next, func(i, j int) bool { + if next[i].Signal == next[j].Signal { + return next[i].Field < next[j].Field + } + return next[i].Signal < next[j].Signal + }) + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return DescriptorActivation{}, errors.New("begin descriptor activation") + } + defer tx.Rollback() + timestamp := now.UTC().Format(time.RFC3339Nano) + if _, err = tx.ExecContext(ctx, `INSERT INTO projection_versions(version,created_at,activated_at) VALUES(?,?,?)`, nextVersion, timestamp, timestamp); err != nil { + return DescriptorActivation{}, errors.New("create projection version") + } + for _, descriptor := range next { + encoded, marshalErr := json.Marshal(descriptor) + if marshalErr != nil { + return DescriptorActivation{}, errors.New("encode active descriptor") + } + if _, err = tx.ExecContext(ctx, `INSERT INTO projection_descriptors(version,signal,field,descriptor_json) VALUES(?,?,?,?)`, nextVersion, descriptor.Signal, descriptor.Field, string(encoded)); err != nil { + return DescriptorActivation{}, errors.New("store active descriptor") + } + } + indexedRows, err := buildProjectionIndex(ctx, tx, nextVersion, next) + if err != nil { + return DescriptorActivation{}, err + } + result, err := tx.ExecContext(ctx, `UPDATE projection_state SET active_version=? WHERE id=1 AND active_version=?`, nextVersion, currentVersion) + if err != nil { + return DescriptorActivation{}, errors.New("activate projection version") + } + if changed, _ := result.RowsAffected(); changed != 1 { + return DescriptorActivation{}, errors.New("active projection changed during activation") + } + if err = tx.Commit(); err != nil { + return DescriptorActivation{}, errors.New("commit descriptor activation") + } + encoded, err := json.Marshal(reviewed) + if err != nil { + return DescriptorActivation{}, errors.New("encode activated descriptor") + } + result, err = controlTx.ExecContext(ctx, `UPDATE descriptor_proposals SET descriptor_json=?,status='activated' WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, string(encoded), organizationID, reviewed.Signal, reviewed.Field) + if err != nil { + return DescriptorActivation{}, errors.New("acknowledge activated descriptor") + } + if changed, _ := result.RowsAffected(); changed != 1 { + return DescriptorActivation{}, errors.New("descriptor proposal changed during activation") + } + if err = controlTx.Commit(); err != nil { + return DescriptorActivation{}, errors.New("commit activated descriptor acknowledgement") + } + return DescriptorActivation{OrganizationID: organizationID, Signal: reviewed.Signal, Field: reviewed.Field, Previous: currentVersion, Active: nextVersion, IndexedRows: indexedRows, Descriptor: reviewed}, nil +} + +func buildProjectionIndex(ctx context.Context, tx *sql.Tx, version int, descriptors []schema.Descriptor) (int64, error) { + table, err := projectionIndexTable(version) + if err != nil { + return 0, err + } + if _, err = tx.ExecContext(ctx, `CREATE TABLE `+table+` ( + signal TEXT NOT NULL, field TEXT NOT NULL, + source_id TEXT NOT NULL, stream_id TEXT NOT NULL, + sequence INTEGER NOT NULL, record_index INTEGER NOT NULL, + timestamp TEXT NOT NULL, value_text TEXT NOT NULL, value_number REAL, + PRIMARY KEY(signal,field,source_id,stream_id,sequence,record_index) + ) WITHOUT ROWID`); err != nil { + return 0, errors.New("create projection index") + } + indexed := map[string]schema.Descriptor{} + for _, descriptor := range descriptors { + if descriptor.Index != schema.IndexNone { + indexed[string(descriptor.Signal)+":"+descriptor.Field] = descriptor + } + } + rows, err := tx.QueryContext(ctx, `SELECT source_id,stream_id,sequence,record_index,signal,timestamp,attributes_json FROM observations ORDER BY source_id,stream_id,sequence,record_index`) + if err != nil { + return 0, errors.New("scan projection for index build") + } + statement, err := tx.PrepareContext(ctx, `INSERT INTO `+table+`(signal,field,source_id,stream_id,sequence,record_index,timestamp,value_text,value_number) VALUES(?,?,?,?,?,?,?,?,?)`) + if err != nil { + _ = rows.Close() + return 0, errors.New("prepare projection index build") + } + defer statement.Close() + var count int64 + for rows.Next() { + var sourceID, streamID, signalText, timestamp, attributesJSON string + var sequence uint64 + var recordIndex int + if err = rows.Scan(&sourceID, &streamID, &sequence, &recordIndex, &signalText, ×tamp, &attributesJSON); err != nil { + _ = rows.Close() + return 0, errors.New("read projection index source") + } + var attributes map[string]string + if err = json.Unmarshal([]byte(attributesJSON), &attributes); err != nil { + _ = rows.Close() + return 0, errors.New("decode projection index source") + } + for field, raw := range attributes { + descriptor, ok := indexed[signalText+":"+query.CanonicalField(field)] + if !ok { + continue + } + text, number, ok := indexValue(raw, descriptor.Type) + if !ok { + continue + } + if _, err = statement.ExecContext(ctx, signalText, descriptor.Field, sourceID, streamID, sequence, recordIndex, timestamp, text, number); err != nil { + _ = rows.Close() + return 0, errors.New("build projection index") + } + if count == math.MaxInt64 { + _ = rows.Close() + return 0, errors.New("projection index row count overflow") + } + count++ + } + } + if err = rows.Close(); err != nil { + return 0, errors.New("close projection index source") + } + if err = rows.Err(); err != nil { + return 0, errors.New("scan projection for index build") + } + for _, suffix := range []struct{ name, columns string }{ + {"exact", "signal,field,value_text,timestamp"}, + {"number", "signal,field,value_number,timestamp"}, + {"record", "source_id,stream_id,sequence,record_index,signal,field"}, + } { + if _, err = tx.ExecContext(ctx, `CREATE INDEX `+table+`_`+suffix.name+` ON `+table+`(`+suffix.columns+`)`); err != nil { + return 0, errors.New("index activated projection") + } + } + return count, nil +} + +func indexValue(raw string, valueType schema.Type) (string, any, bool) { + switch valueType { + case schema.TypeInteger: + value, err := strconv.ParseInt(raw, 10, 64) + return raw, value, err == nil + case schema.TypeFloat, schema.TypeDuration: + value, err := strconv.ParseFloat(raw, 64) + return raw, value, err == nil && !math.IsNaN(value) && !math.IsInf(value, 0) + case schema.TypeBoolean: + value, err := strconv.ParseBool(raw) + if err != nil { + return "", nil, false + } + return strconv.FormatBool(value), nil, true + case schema.TypeTime: + value, err := time.Parse(time.RFC3339Nano, raw) + if err != nil { + return "", nil, false + } + return value.UTC().Format(indexedTimeFormat), nil, true + case schema.TypeString: + return raw, nil, true + default: + return "", nil, false + } +} + +func indexProjectedObservation(ctx context.Context, tx *sql.Tx, version int, descriptors []schema.Descriptor, batch model.Batch, recordIndex int, observation model.Observation) error { + if version == 1 { + return nil + } + table, err := projectionIndexTable(version) + if err != nil { + return err + } + indexed := map[string]schema.Descriptor{} + for _, descriptor := range descriptors { + if descriptor.Signal == batch.Signal && descriptor.Index != schema.IndexNone { + indexed[descriptor.Field] = descriptor + } + } + for field, raw := range observation.Attributes { + descriptor, ok := indexed[query.CanonicalField(field)] + if !ok { + continue + } + text, number, ok := indexValue(raw, descriptor.Type) + if !ok { + continue + } + _, err = tx.ExecContext(ctx, `INSERT OR IGNORE INTO `+table+`(signal,field,source_id,stream_id,sequence,record_index,timestamp,value_text,value_number) VALUES(?,?,?,?,?,?,?,?,?)`, batch.Signal, descriptor.Field, batch.SourceID, batch.StreamID, batch.Sequence, recordIndex, observation.Timestamp.UTC().Format(time.RFC3339Nano), text, number) + if err != nil { + return errors.New("index projected observation") + } + } + return nil +} + +func sameDescriptorIgnoringProjection(left, right schema.Descriptor) bool { + left.ProjectionVersion = 1 + right.ProjectionVersion = 1 + leftJSON, _ := json.Marshal(left) + rightJSON, _ := json.Marshal(right) + return string(leftJSON) == string(rightJSON) +} diff --git a/internal/storage/activation_test.go b/internal/storage/activation_test.go new file mode 100644 index 0000000..610d845 --- /dev/null +++ b/internal/storage/activation_test.go @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" +) + +func TestIndexedTimeValuesUseFixedChronologicalUTCText(t *testing.T) { + earlier, _, ok := indexValue("2026-08-17T09:00:00Z", schema.TypeTime) + if !ok { + t.Fatal("earlier time rejected") + } + later, _, ok := indexValue("2026-08-17T09:00:00.1Z", schema.TypeTime) + if !ok { + t.Fatal("later time rejected") + } + if len(earlier) != len(later) || strings.Compare(earlier, later) >= 0 || earlier != "2026-08-17T09:00:00.000000000Z" { + t.Fatalf("earlier=%q later=%q", earlier, later) + } +} + +func TestDescriptorActivationAndIngestionSerializeAcrossStoreInstances(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + second, err := Open(store.root) + if err != nil { + t.Fatal(err) + } + defer second.Close() + now := time.Date(2026, 8, 17, 9, 0, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + first := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: now, Name: "queue.depth", Value: floatPointer(1), Attributes: map[string]string{"workshop.queue_depth": "1"}}}} + if _, err = store.Ingest(ctx, token, first, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + reviewed := schema.Descriptor{Version: schema.DescriptorVersion, Signal: model.SignalMetrics, Field: "workshop.queue_depth", Type: schema.TypeInteger, Meaning: "Number of work items waiting in the selected service queue.", Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, Index: schema.IndexRange, Retention: schema.RetentionRaw, ProjectionVersion: 1} + secondBatch := first + secondBatch.Sequence = 2 + secondBatch.ObservedAt = now.Add(time.Second) + secondBatch.Records = []model.Observation{{Timestamp: secondBatch.ObservedAt, Name: "queue.depth", Value: floatPointer(2), Attributes: map[string]string{"workshop.queue_depth": "2"}}} + start := make(chan struct{}) + errorsFound := make(chan error, 2) + go func() { + <-start + _, activateErr := store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(2*time.Second)) + errorsFound <- activateErr + }() + go func() { + <-start + _, ingestErr := second.Ingest(ctx, token, secondBatch, now.Add(2*time.Second)) + errorsFound <- ingestErr + }() + close(start) + for range 2 { + if runErr := <-errorsFound; runErr != nil { + t.Fatal(runErr) + } + } + projectAll(t, store) + path := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite") + db := openTestProjection(t, path) + defer db.Close() + var indexed int + if err = db.QueryRow(`SELECT COUNT(*) FROM indexed_fields_v000002`).Scan(&indexed); err != nil { + t.Fatal(err) + } + if indexed != 2 { + t.Fatalf("indexed=%d", indexed) + } +} + +func TestDescriptorActivationBuildsBesideCurrentAndFeedsQueriesAndIngestion(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 9, 0, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{ + {Timestamp: now, Name: "queue.depth", Value: floatPointer(10), Attributes: map[string]string{"workshop.queue_depth": "10", "workshop.mode": "fast"}}, + {Timestamp: now.Add(time.Second), Name: "queue.depth", Value: floatPointer(11), Attributes: map[string]string{"workshop.queue_depth": "not-an-integer", "workshop.mode": "slow"}}, + }} + if _, err = store.Ingest(ctx, token, batch, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + projectAll(t, store) + ast, err := query.Parse(`metrics | where workshop.queue_depth >= 9 | limit 50`, 100) + if err != nil { + t.Fatal(err) + } + if _, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now.Add(time.Minute)); !errors.Is(err, query.ErrSensitivePermissionRequired) { + t.Fatalf("unreviewed query err=%v", err) + } + reviewed := schema.Descriptor{ + Version: schema.DescriptorVersion, Signal: model.SignalMetrics, Field: "workshop.queue_depth", + Type: schema.TypeInteger, Meaning: "Number of work items waiting in the selected service queue.", + Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, + Index: schema.IndexRange, Retention: schema.RetentionRaw, ProjectionVersion: 1, + } + activation, err := store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(2*time.Minute)) + if err != nil { + t.Fatal(err) + } + if activation.Previous != 1 || activation.Active != 2 || activation.IndexedRows != 1 || activation.Descriptor.ProjectionVersion != 2 { + t.Fatalf("activation=%+v", activation) + } + registry, version, err := store.ActiveDescriptors(ctx, "organization-a") + if err != nil || version != 2 { + t.Fatalf("version=%d registry=%+v err=%v", version, registry, err) + } + descriptor, ok := registry.Lookup(model.SignalMetrics, "workshop.queue_depth") + if !ok || descriptor.ProjectionVersion != 2 || descriptor.Sensitivity != schema.SensitivityInternal || descriptor.Index != schema.IndexRange { + t.Fatalf("descriptor=%+v ok=%t", descriptor, ok) + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now.Add(3*time.Minute)) + if err != nil { + t.Fatal(err) + } + if len(result.Rows) != 1 || len(result.Explain.Fields) != 1 || !result.Explain.Fields[0].Indexed || result.Explain.Fields[0].Unknown { + t.Fatalf("result=%+v", result) + } + + batch.Sequence = 2 + batch.ObservedAt = now.Add(4 * time.Minute) + batch.Records = []model.Observation{{Timestamp: batch.ObservedAt, Name: "queue.depth", Value: floatPointer(12), Attributes: map[string]string{"workshop.queue_depth": "12", "workshop.mode": "fast"}}} + if _, err = store.Ingest(ctx, token, batch, batch.ObservedAt); err != nil { + t.Fatal(err) + } + projectAll(t, store) + result, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now.Add(5*time.Minute)) + if err != nil || len(result.Rows) != 2 { + t.Fatalf("rows=%d err=%v", len(result.Rows), err) + } + + path := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite") + db := openTestProjection(t, path) + defer db.Close() + var active, indexed int + if err = db.QueryRow(`SELECT active_version FROM projection_state WHERE id=1`).Scan(&active); err != nil { + t.Fatal(err) + } + if err = db.QueryRow(`SELECT COUNT(*) FROM indexed_fields_v000002`).Scan(&indexed); err != nil { + t.Fatal(err) + } + if active != 2 || indexed != 2 { + t.Fatalf("active=%d indexed=%d", active, indexed) + } + statement, arguments, err := projectionSelection(ast, query.Scope{OrganizationID: "organization-a"}, registry, version, now.Add(5*time.Minute)) + if err != nil { + t.Fatal(err) + } + planRows, err := db.Query(`EXPLAIN QUERY PLAN `+statement, arguments...) + if err != nil { + t.Fatal(err) + } + var plan strings.Builder + for planRows.Next() { + var id, parent, unused int + var detail string + if err = planRows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan.WriteString(detail) + plan.WriteByte('\n') + } + if err = planRows.Close(); err != nil { + t.Fatal(err) + } + if !strings.Contains(plan.String(), "indexed_fields_v000002_number") { + t.Fatalf("custom range index absent from query plan:\n%s", plan.String()) + } + + preActivationDescriptor, err := json.Marshal(reviewed) + if err != nil { + t.Fatal(err) + } + if _, err = store.control.Exec(`UPDATE descriptor_proposals SET descriptor_json=?,status='pending' WHERE organization_id='organization-a' AND signal='metrics' AND field='workshop.queue_depth'`, string(preActivationDescriptor)); err != nil { + t.Fatal(err) + } + retry, err := store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(6*time.Minute)) + if err != nil || retry.Active != 2 || retry.Previous != 2 || retry.IndexedRows != 0 { + t.Fatalf("retry=%+v err=%v", retry, err) + } + mode := schema.Descriptor{ + Version: schema.DescriptorVersion, Signal: model.SignalMetrics, Field: "workshop.mode", + Type: schema.TypeString, Meaning: "Reviewed operating mode label for the workshop queue.", + Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, + Index: schema.IndexExact, Retention: schema.RetentionRaw, ProjectionVersion: 1, + } + second, err := store.ActivateDescriptor(ctx, "organization-a", mode, now.Add(7*time.Minute)) + if err != nil || second.Previous != 2 || second.Active != 3 || second.IndexedRows != 5 { + t.Fatalf("second=%+v err=%v", second, err) + } + var retained, current int + if err = db.QueryRow(`SELECT COUNT(*) FROM indexed_fields_v000002`).Scan(&retained); err != nil { + t.Fatal(err) + } + if err = db.QueryRow(`SELECT COUNT(*) FROM indexed_fields_v000003`).Scan(¤t); err != nil { + t.Fatal(err) + } + if retained != 2 || current != 5 { + t.Fatalf("retained=%d current=%d", retained, current) + } + result, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now.Add(8*time.Minute)) + if err != nil || len(result.Rows) != 2 { + t.Fatalf("version-three rows=%d err=%v", len(result.Rows), err) + } + proposals, err := store.DescriptorProposals(ctx, "organization-a") + if err != nil || len(proposals) != 2 || proposals[0].Status != "activated" || proposals[1].Status != "activated" { + t.Fatalf("proposals=%+v err=%v", proposals, err) + } +} + +func TestDescriptorActivationFailureKeepsPriorVersion(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 9, 0, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request", Attributes: map[string]string{"workshop.label": "ready"}}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + path := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite") + db := openTestProjection(t, path) + if _, err = db.Exec(`DROP TABLE observations`); err != nil { + t.Fatal(err) + } + db.Close() + reviewed := schema.Descriptor{Version: schema.DescriptorVersion, Signal: model.SignalLogs, Field: "workshop.label", Type: schema.TypeString, Meaning: "Reviewed workshop state label.", Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, Index: schema.IndexExact, Retention: schema.RetentionRaw, ProjectionVersion: 1} + if _, err = store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(time.Minute)); err == nil { + t.Fatal("activation unexpectedly succeeded without the source projection") + } + db = openTestProjection(t, path) + defer db.Close() + var active, versions int + if err = db.QueryRow(`SELECT active_version FROM projection_state WHERE id=1`).Scan(&active); err != nil { + t.Fatal(err) + } + if err = db.QueryRow(`SELECT COUNT(*) FROM projection_versions`).Scan(&versions); err != nil { + t.Fatal(err) + } + if active != 1 || versions != 1 { + t.Fatalf("active=%d versions=%d", active, versions) + } + proposal, err := store.descriptorProposal(ctx, "organization-a", model.SignalLogs, "workshop.label") + if err != nil || proposal.Status != "pending" { + t.Fatalf("proposal=%+v err=%v", proposal, err) + } +} + +func TestDescriptorProposalRejectionIsIdempotentAndBlocksActivation(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 9, 0, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request", Attributes: map[string]string{"workshop.label": "ready"}}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + if err = store.RejectDescriptorProposal(ctx, "organization-a", model.SignalLogs, "workshop.label"); err != nil { + t.Fatal(err) + } + if err = store.RejectDescriptorProposal(ctx, "organization-a", model.SignalLogs, "workshop.label"); err != nil { + t.Fatal(err) + } + reviewed := schema.Descriptor{Version: schema.DescriptorVersion, Signal: model.SignalLogs, Field: "workshop.label", Type: schema.TypeString, Meaning: "Reviewed workshop state label.", Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, Index: schema.IndexExact, Retention: schema.RetentionRaw, ProjectionVersion: 1} + if _, err = store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(time.Minute)); err == nil { + t.Fatal("rejected proposal was activated") + } +} + +func openTestProjection(t *testing.T, path string) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + return db +} diff --git a/internal/storage/base_indexes.go b/internal/storage/base_indexes.go new file mode 100644 index 0000000..6020118 --- /dev/null +++ b/internal/storage/base_indexes.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" +) + +const baseIndexVersion = 1 + +var presenceIndexStatements = []string{ + `CREATE INDEX IF NOT EXISTS observations_value ON observations(signal,value,timestamp) WHERE value IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS observations_http_route ON observations(signal,json_extract(attributes_json,'$."http.route"'),timestamp) WHERE json_extract(attributes_json,'$."http.route"') IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS observations_http_status ON observations(signal,CAST(json_extract(attributes_json,'$."http.status_code"') AS INTEGER),timestamp) WHERE CAST(json_extract(attributes_json,'$."http.status_code"') AS INTEGER) IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS observations_duration ON observations(signal,CAST(json_extract(attributes_json,'$."duration_ns"') AS REAL),timestamp) WHERE CAST(json_extract(attributes_json,'$."duration_ns"') AS REAL) IS NOT NULL`, +} + +// ensureBaseIndexes migrates fields that are absent from most signal types to +// presence-only indexes. The DDL and migration marker share one SQLite +// transaction: an interrupted migration retains the complete previous index +// set and retries before the projection is served. +func ensureBaseIndexes(ctx context.Context, db *sql.DB) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin base index migration: %w", err) + } + defer tx.Rollback() + columns, err := sqliteColumns(tx, "storage_projection_state") + if err != nil { + return err + } + if !columns["base_index_version"] { + if _, err = tx.ExecContext(ctx, `ALTER TABLE storage_projection_state ADD COLUMN base_index_version INTEGER NOT NULL DEFAULT 0 CHECK(base_index_version BETWEEN 0 AND 1)`); err != nil { + return errors.New("add base index migration state") + } + } + var version int + if err = tx.QueryRowContext(ctx, `SELECT base_index_version FROM storage_projection_state WHERE id=1`).Scan(&version); err != nil { + return errors.New("read base index migration state") + } + if version < 0 || version > baseIndexVersion { + return errors.New("unsupported base index version") + } + if version == 0 { + for _, name := range []string{"observations_value", "observations_http_route", "observations_http_status", "observations_duration"} { + if _, err = tx.ExecContext(ctx, `DROP INDEX IF EXISTS `+name); err != nil { + return errors.New("remove superseded base index") + } + } + } + for _, statement := range presenceIndexStatements { + if _, err = tx.ExecContext(ctx, statement); err != nil { + return errors.New("create presence-only base index") + } + } + if version == 0 { + if _, err = tx.ExecContext(ctx, `UPDATE storage_projection_state SET base_index_version=? WHERE id=1 AND base_index_version=0`, baseIndexVersion); err != nil { + return errors.New("activate base index migration") + } + } + if err = tx.Commit(); err != nil { + return errors.New("commit base index migration") + } + return nil +} diff --git a/internal/storage/base_indexes_test.go b/internal/storage/base_indexes_test.go new file mode 100644 index 0000000..180df98 --- /dev/null +++ b/internal/storage/base_indexes_test.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +func TestBaseIndexMigrationReplacesLegacyFullIndexes(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + dir := filepath.Join(root, "organizations", "legacy") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "projection.sqlite") + legacy, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + statements := []string{ + `CREATE TABLE observations (organization_id TEXT NOT NULL,project_id TEXT NOT NULL,environment_id TEXT NOT NULL,service_id TEXT NOT NULL,source_id TEXT NOT NULL,stream_id TEXT NOT NULL,sequence INTEGER NOT NULL,record_index INTEGER NOT NULL,signal TEXT NOT NULL,timestamp TEXT NOT NULL,name TEXT NOT NULL,severity TEXT,body TEXT,value REAL,trace_id TEXT,span_id TEXT,correlation_id TEXT,attributes_json TEXT NOT NULL,segment_digest TEXT NOT NULL,PRIMARY KEY(source_id,stream_id,sequence,record_index))`, + `CREATE TABLE storage_projection_state (id INTEGER PRIMARY KEY CHECK(id=1),metric_rollup_version INTEGER NOT NULL CHECK(metric_rollup_version BETWEEN 0 AND 1))`, + `INSERT INTO storage_projection_state(id,metric_rollup_version) VALUES(1,1)`, + `CREATE INDEX observations_value ON observations(signal,value,timestamp)`, + `CREATE INDEX observations_http_route ON observations(signal,json_extract(attributes_json,'$."http.route"'),timestamp)`, + `CREATE INDEX observations_http_status ON observations(signal,CAST(json_extract(attributes_json,'$."http.status_code"') AS INTEGER),timestamp)`, + `CREATE INDEX observations_duration ON observations(signal,CAST(json_extract(attributes_json,'$."duration_ns"') AS REAL),timestamp)`, + } + for _, statement := range statements { + if _, err = legacy.Exec(statement); err != nil { + legacy.Close() + t.Fatal(err) + } + } + if err = legacy.Close(); err != nil { + t.Fatal(err) + } + db, err := openProjection(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var version int + if err = db.QueryRow(`SELECT base_index_version FROM storage_projection_state WHERE id=1`).Scan(&version); err != nil || version != baseIndexVersion { + t.Fatalf("base index version=%d err=%v", version, err) + } + for _, name := range []string{"observations_value", "observations_http_route", "observations_http_status", "observations_duration"} { + var definition string + if err = db.QueryRow(`SELECT sql FROM sqlite_schema WHERE type='index' AND name=?`, name).Scan(&definition); err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.ToUpper(definition), " WHERE ") || !strings.Contains(strings.ToUpper(definition), " IS NOT NULL") { + t.Fatalf("index %s was not migrated: %s", name, definition) + } + } +} + +func TestPresenceIndexesSupportSelectiveQueries(t *testing.T) { + ctx := t.Context() + store := testStore(t) + defer store.Close() + now := time.Now().UTC() + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + logToken, err := store.CreateSource(ctx, "source-logs", scope) + if err != nil { + t.Fatal(err) + } + logs := model.Batch{Version: model.BatchVersion, SourceID: "source-logs", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + {Timestamp: now, Name: "http.server.request", Severity: "error", Attributes: map[string]string{"http.route": "/items", "http.status_code": "503", "duration_ns": "500"}}, + {Timestamp: now.Add(-time.Second), Name: "application.event", Severity: "information", Attributes: map[string]string{}}, + }} + if _, err = store.Ingest(ctx, logToken, logs, now); err != nil { + t.Fatal(err) + } + metricToken, err := store.CreateSource(ctx, "source-metrics", scope) + if err != nil { + t.Fatal(err) + } + value := 42.0 + metrics := model.Batch{Version: model.BatchVersion, SourceID: "source-metrics", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: now, Name: "system.cpu.utilization", Value: &value}}} + if _, err = store.Ingest(ctx, metricToken, metrics, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + queries := []string{ + `logs | where route == "/items" | summarize count() by status | limit 10`, + `logs | where status >= 500 | summarize p95(duration) by route | limit 10`, + `logs | where duration >= 100 | summarize count() by route | limit 10`, + `metrics | where value >= 1 | summarize count() by name | limit 10`, + } + for _, text := range queries { + ast, parseErr := query.Parse(text, 10) + if parseErr != nil { + t.Fatalf("parse %q: %v", text, parseErr) + } + result, queryErr := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now) + if queryErr != nil || len(result.Rows) == 0 { + t.Fatalf("query %q rows=%d err=%v", text, len(result.Rows), queryErr) + } + } +} diff --git a/internal/storage/batch_envelope.go b/internal/storage/batch_envelope.go new file mode 100644 index 0000000..eba8fe0 --- /dev/null +++ b/internal/storage/batch_envelope.go @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math" + "time" + + "gamertan.com/observatory/internal/model" +) + +func migrateControlBatchEnvelopes(db *sql.DB) error { + tx, err := db.Begin() + if err != nil { + return errors.New("begin batch envelope migration") + } + defer tx.Rollback() + for _, statement := range []string{ + `ALTER TABLE streams ADD COLUMN last_batch_digest TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE streams ADD COLUMN last_wire_digest TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE streams ADD COLUMN last_signal TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE streams ADD COLUMN last_record_count INTEGER NOT NULL DEFAULT 0 CHECK(last_record_count BETWEEN 0 AND 5000)`, + `ALTER TABLE streams ADD COLUMN last_encoded_bytes INTEGER NOT NULL DEFAULT 0 CHECK(last_encoded_bytes >= 0)`, + `ALTER TABLE streams ADD COLUMN last_first_observed_at TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE streams ADD COLUMN last_last_observed_at TEXT NOT NULL DEFAULT ''`, + `UPDATE schema_version SET version=11 WHERE version=10`, + } { + if _, err = tx.Exec(statement); err != nil { + return fmt.Errorf("migrate batch envelope metadata: %w", err) + } + } + if err = tx.Commit(); err != nil { + return errors.New("commit batch envelope migration") + } + return nil +} + +type streamWatermark struct { + sequence uint64 + segmentDigest string + envelope model.BatchEnvelope + found bool + framed bool +} + +func (s *Store) streamWatermark(ctx context.Context, sourceID, streamID string) (streamWatermark, error) { + var watermark streamWatermark + var signal, first, last string + err := s.control.QueryRowContext(ctx, `SELECT last_sequence,last_digest,last_batch_digest,last_wire_digest,last_signal,last_record_count,last_encoded_bytes,last_first_observed_at,last_last_observed_at FROM streams WHERE source_id=? AND stream_id=?`, sourceID, streamID).Scan( + &watermark.sequence, &watermark.segmentDigest, &watermark.envelope.BatchDigest, + &watermark.envelope.WireDigest, &signal, &watermark.envelope.RecordCount, + &watermark.envelope.EncodedBytes, &first, &last, + ) + if errors.Is(err, sql.ErrNoRows) { + return streamWatermark{}, nil + } + if err != nil { + return streamWatermark{}, fmt.Errorf("read framed stream watermark: %w", err) + } + watermark.found = true + if watermark.envelope.BatchDigest == "" && watermark.envelope.WireDigest == "" { + return watermark, nil + } + firstObserved, firstErr := time.Parse(time.RFC3339Nano, first) + lastObserved, lastErr := time.Parse(time.RFC3339Nano, last) + if firstErr != nil || lastErr != nil { + return streamWatermark{}, errors.New("stored batch envelope time range is invalid") + } + watermark.envelope.Version = model.BatchEnvelopeVersion + watermark.envelope.StreamID = streamID + watermark.envelope.Sequence = watermark.sequence + watermark.envelope.Signal = model.Signal(signal) + watermark.envelope.FirstObservedAt = firstObserved.UTC() + watermark.envelope.LastObservedAt = lastObserved.UTC() + if err = watermark.envelope.Validate(math.MaxInt64); err != nil { + return streamWatermark{}, errors.New("stored batch envelope is invalid") + } + watermark.framed = true + return watermark, nil +} + +func (s *Store) checkEnvelope(ctx context.Context, source Source, envelope model.BatchEnvelope) (Ack, bool, error) { + watermark, err := s.streamWatermark(ctx, source.ID, envelope.StreamID) + if err != nil { + return Ack{}, false, err + } + if !watermark.found { + if envelope.Sequence != 1 { + return Ack{}, false, errors.New("sequence gap") + } + return Ack{}, false, nil + } + if envelope.Sequence < watermark.sequence { + return Ack{}, false, errors.New("sequence replay is older than acknowledged watermark") + } + if watermark.sequence != ^uint64(0) && envelope.Sequence > watermark.sequence+1 { + return Ack{}, false, errors.New("sequence gap") + } + if envelope.Sequence == watermark.sequence { + if !watermark.framed { + return Ack{}, false, nil + } + if envelope != watermark.envelope { + return Ack{}, false, errors.New("acknowledged sequence reused with different envelope") + } + return Ack{SourceID: source.ID, StreamID: envelope.StreamID, Sequence: envelope.Sequence, Digest: watermark.segmentDigest, BatchDigest: envelope.BatchDigest, Duplicate: true}, true, nil + } + return Ack{}, false, nil +} + +// CheckNativeReplay performs a cheap, read-only envelope lookup before the +// request body is decoded. Callers must still hash the complete bounded body +// and ConfirmNativeReplay before acknowledging it. +func (s *Store) CheckNativeReplay(ctx context.Context, token string, envelope model.BatchEnvelope) (Ack, bool, error) { + if err := envelope.Validate(math.MaxInt64); err != nil { + return Ack{}, false, err + } + source, err := s.Authenticate(ctx, token) + if err != nil { + return Ack{}, false, err + } + return s.checkEnvelope(ctx, source, envelope) +} + +func (s *Store) ConfirmNativeReplay(ctx context.Context, token string, envelope model.BatchEnvelope) (Ack, error) { + if err := envelope.Validate(math.MaxInt64); err != nil { + return Ack{}, err + } + source, err := s.Authenticate(ctx, token) + if err != nil { + return Ack{}, err + } + lock := s.sourceLock(source.ID) + lock.Lock() + defer lock.Unlock() + ack, exact, err := s.checkEnvelope(ctx, source, envelope) + if err != nil { + return Ack{}, err + } + if !exact { + return Ack{}, errors.New("batch is not an acknowledged exact replay") + } + return ack, nil +} + +func envelopeSQL(envelope *model.BatchEnvelope) (batchDigest, wireDigest, signal string, recordCount int, encodedBytes int64, first, last string) { + if envelope == nil { + return "", "", "", 0, 0, "", "" + } + return envelope.BatchDigest, envelope.WireDigest, string(envelope.Signal), envelope.RecordCount, envelope.EncodedBytes, envelope.FirstObservedAt.UTC().Format(time.RFC3339Nano), envelope.LastObservedAt.UTC().Format(time.RFC3339Nano) +} + +func (s *Store) backfillAcknowledgedEnvelope(ctx context.Context, batch model.Batch, segmentDigest string, envelope model.BatchEnvelope) error { + batchDigest, wireDigest, signal, recordCount, encodedBytes, first, last := envelopeSQL(&envelope) + result, err := s.control.ExecContext(ctx, `UPDATE streams SET last_batch_digest=?,last_wire_digest=?,last_signal=?,last_record_count=?,last_encoded_bytes=?,last_first_observed_at=?,last_last_observed_at=? WHERE source_id=? AND stream_id=? AND last_sequence=? AND last_digest=? AND last_batch_digest='' AND last_wire_digest=''`, batchDigest, wireDigest, signal, recordCount, encodedBytes, first, last, batch.SourceID, batch.StreamID, batch.Sequence, segmentDigest) + if err != nil { + return fmt.Errorf("backfill acknowledged batch envelope: %w", err) + } + n, _ := result.RowsAffected() + if n != 1 { + return errors.New("acknowledged batch envelope state changed") + } + return nil +} diff --git a/internal/storage/batch_metadata.go b/internal/storage/batch_metadata.go new file mode 100644 index 0000000..18dbf1f --- /dev/null +++ b/internal/storage/batch_metadata.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "database/sql" + "errors" + "fmt" +) + +func migrateControlBatchMetadata(db *sql.DB) error { + tx, err := db.Begin() + if err != nil { + return errors.New("begin batch metadata migration") + } + defer tx.Rollback() + for _, statement := range []string{ + `ALTER TABLE segments ADD COLUMN record_count INTEGER NOT NULL DEFAULT 0 CHECK(record_count BETWEEN 0 AND 5000)`, + `UPDATE schema_version SET version=10 WHERE version=9`, + } { + if _, err = tx.Exec(statement); err != nil { + return fmt.Errorf("migrate batch metadata: %w", err) + } + } + if err = tx.Commit(); err != nil { + return errors.New("commit batch metadata migration") + } + return nil +} diff --git a/internal/storage/batch_metadata_test.go b/internal/storage/batch_metadata_test.go new file mode 100644 index 0000000..77a7deb --- /dev/null +++ b/internal/storage/batch_metadata_test.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "testing" + "time" + + "gamertan.com/observatory/internal/model" +) + +func TestCommittedSegmentRecordsBoundedBatchMetadata(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := context.Background() + now := time.Date(2026, 8, 19, 0, 10, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now.Add(-time.Second), Name: "first"}, {Timestamp: now, Name: "second"}}} + ack, err := store.Ingest(ctx, token, batch, now) + if err != nil { + t.Fatal(err) + } + var count int + var first, last string + if err = store.control.QueryRowContext(ctx, `SELECT record_count,first_observed_at,last_observed_at FROM segments WHERE digest=?`, ack.Digest).Scan(&count, &first, &last); err != nil { + t.Fatal(err) + } + if count != 2 || first != now.Add(-time.Second).Format(time.RFC3339Nano) || last != now.Format(time.RFC3339Nano) { + t.Fatalf("count=%d first=%q last=%q", count, first, last) + } +} diff --git a/internal/storage/cold_query.go b/internal/storage/cold_query.go new file mode 100644 index 0000000..d6e2221 --- /dev/null +++ b/internal/storage/cold_query.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math" + "path/filepath" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +const maxRawQuerySegments = 1_000_000 + +type rawQuerySegment struct { + digest, path, sourceID, streamID string + projectID, environmentID, serviceID string + tier string + sequence uint64 + uncompressedBytes int64 + firstObservedAt, lastObservedAt time.Time +} + +func (s *Store) coldSegmentsForQuery(ctx context.Context, ast query.AST, scope query.Scope, now time.Time) ([]rawQuerySegment, int64, error) { + return s.rawSegmentsForQuery(ctx, ast, scope, now, false) +} + +func (s *Store) allRawSegmentsForQuery(ctx context.Context, ast query.AST, scope query.Scope, now time.Time) ([]rawQuerySegment, int64, error) { + return s.rawSegmentsForQuery(ctx, ast, scope, now, true) +} + +func (s *Store) rawSegmentsForQuery(ctx context.Context, ast query.AST, scope query.Scope, now time.Time, includeHot bool) ([]rawQuerySegment, int64, error) { + statement := `SELECT segment.digest,segment.path,segment.source_id,segment.stream_id,segment.sequence,segment.uncompressed_bytes,segment.first_observed_at,segment.last_observed_at,segment.tier,segment.archiving_at,segment.retiring_at,source.project_id,source.environment_id,source.service_id FROM segments segment JOIN sources source ON source.id=segment.source_id WHERE segment.organization_id=? AND source.organization_id=segment.organization_id AND segment.signal=?` + if includeHot { + statement += ` AND segment.tier IN ('hot','cold')` + } else { + statement += ` AND segment.tier='cold' AND segment.retiring_at IS NULL` + } + arguments := []any{scope.OrganizationID, ast.Signal} + for _, selected := range []struct{ column, value string }{{"source.project_id", scope.ProjectID}, {"source.environment_id", scope.EnvironmentID}, {"source.service_id", scope.ServiceID}} { + if selected.value != "" { + statement += " AND " + selected.column + "=?" + arguments = append(arguments, selected.value) + } + } + if ast.Window > 0 { + statement += " AND segment.last_observed_at>=?" + arguments = append(arguments, now.UTC().Add(-ast.Window).Format(time.RFC3339Nano)) + } + statement += ` ORDER BY segment.last_observed_at DESC,segment.source_id,segment.stream_id,segment.sequence DESC` + rows, err := s.control.QueryContext(ctx, statement, arguments...) + if err != nil { + return nil, 0, errors.New("list raw query segments") + } + defer rows.Close() + segments := make([]rawQuerySegment, 0) + var estimated int64 + for rows.Next() { + if len(segments) >= maxRawQuerySegments { + return nil, 0, errors.New("raw query segment limit exceeded") + } + var segment rawQuerySegment + var firstText, lastText string + var archivingAt, retiringAt sql.NullString + if err = rows.Scan(&segment.digest, &segment.path, &segment.sourceID, &segment.streamID, &segment.sequence, &segment.uncompressedBytes, &firstText, &lastText, &segment.tier, &archivingAt, &retiringAt, &segment.projectID, &segment.environmentID, &segment.serviceID); err != nil { + return nil, 0, errors.New("read raw query segment") + } + if includeHot && (archivingAt.Valid || retiringAt.Valid) { + return nil, 0, errors.New("raw query segment transition is incomplete") + } + segment.firstObservedAt, err = time.Parse(time.RFC3339Nano, firstText) + if err != nil { + return nil, 0, errors.New("raw query segment range is invalid") + } + segment.lastObservedAt, err = time.Parse(time.RFC3339Nano, lastText) + if err != nil || segment.lastObservedAt.Before(segment.firstObservedAt) || segment.uncompressedBytes < 1 { + return nil, 0, errors.New("raw query segment range is invalid") + } + var expected string + var pathErr error + if segment.tier == "cold" { + expected, pathErr = s.coldArchivePath(scope.OrganizationID, archivingSegment{digest: segment.digest, path: segment.path, sourceID: segment.sourceID, streamID: segment.streamID, signal: ast.Signal}) + } else if segment.tier == "hot" && model.ValidateSourceID(scope.OrganizationID) == nil && model.ValidateSourceID(segment.sourceID) == nil && model.ValidateStreamID(segment.streamID) == nil { + expected = filepath.Join(s.root, "raw", scope.OrganizationID, segment.sourceID, segment.streamID, fmt.Sprintf("%020d-%s.zst", segment.sequence, segment.digest)) + } else { + pathErr = errors.New("unsupported raw segment tier") + } + if pathErr != nil || expected != segment.path { + return nil, 0, errors.New("raw query segment path is invalid") + } + if segment.uncompressedBytes > math.MaxInt64-estimated { + return nil, 0, errors.New("raw query estimate overflow") + } + estimated += segment.uncompressedBytes + segments = append(segments, segment) + } + if err = rows.Err(); err != nil { + return nil, 0, errors.New("list raw query segments") + } + return segments, estimated, nil +} + +func rawRecord(segment rawQuerySegment, batch model.Batch, index int) projectedRecord { + observation := batch.Records[index] + return projectedRecord{ + projectID: segment.projectID, environmentID: segment.environmentID, serviceID: segment.serviceID, + sourceID: segment.sourceID, streamID: segment.streamID, sequence: batch.Sequence, recordIndex: index, + signal: batch.Signal, timestamp: observation.Timestamp.UTC(), name: observation.Name, severity: observation.Severity, + body: observation.Body, value: observation.Value, traceID: observation.TraceID, spanID: observation.SpanID, + correlationID: observation.CorrelationID, attributes: observation.Attributes, + } +} + +func rawRecordMemory(record projectedRecord) int64 { + total := int64(256 + len(record.projectID) + len(record.environmentID) + len(record.serviceID) + len(record.sourceID) + len(record.streamID) + len(record.name) + len(record.severity) + len(record.body) + len(record.traceID) + len(record.spanID) + len(record.correlationID)) + for key, value := range record.attributes { + addition := int64(len(key) + len(value) + 32) + if addition > math.MaxInt64-total { + return math.MaxInt64 + } + total += addition + } + return total +} diff --git a/internal/storage/dashboards.go b/internal/storage/dashboards.go new file mode 100644 index 0000000..77b14d9 --- /dev/null +++ b/internal/storage/dashboards.go @@ -0,0 +1,598 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + "unicode/utf8" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +const ( + SavedQueryVersion = 1 + DashboardVersion = 1 + MaxDashboardPanels = 16 +) + +var ErrDashboardRevisionConflict = errors.New("dashboard revision conflict") + +type ResourceScope struct { + ProjectID string `json:"project_id,omitempty"` + EnvironmentID string `json:"environment_id,omitempty"` + ServiceID string `json:"service_id,omitempty"` +} + +type SavedQuery struct { + Version int `json:"version"` + Revision int `json:"revision"` + OrganizationID string `json:"organization_id"` + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Query string `json:"query"` + AST query.AST `json:"ast"` + Scope ResourceScope `json:"scope"` + CreatedBy string `json:"created_by"` + UpdatedBy string `json:"updated_by"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type SavedQueryInput struct { + ID string + ExpectedRevision int + OrganizationID string + Name string + Description string + Query string + Scope ResourceScope + ActorUserID string + MaxRows int +} + +type DashboardPanel struct { + ID string `json:"id"` + Position int `json:"position"` + Title string `json:"title"` + Visualization string `json:"visualization"` + SavedQueryID string `json:"saved_query_id"` +} + +type Dashboard struct { + Version int `json:"version"` + Revision int `json:"revision"` + OrganizationID string `json:"organization_id"` + ID string `json:"id"` + Slug string `json:"slug"` + Name string `json:"name"` + Description string `json:"description"` + Panels []DashboardPanel `json:"panels"` + CreatedBy string `json:"created_by"` + UpdatedBy string `json:"updated_by"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type DashboardInput struct { + ID string + ExpectedRevision int + OrganizationID string + Slug string + Name string + Description string + Panels []DashboardPanel + ActorUserID string +} + +type DashboardExport struct { + Version int `json:"version"` + Dashboard DashboardDefinition `json:"dashboard"` + SavedQueries []SavedQueryDefinition `json:"saved_queries"` +} + +type DashboardDefinition struct { + Slug string `json:"slug"` + Name string `json:"name"` + Description string `json:"description"` + Panels []DashboardPanel `json:"panels"` +} + +type SavedQueryDefinition struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Query string `json:"query"` + Scope ResourceScope `json:"scope"` +} + +type DashboardImportInput struct { + OrganizationID string + ActorUserID string + MaxRows int + Bundle DashboardExport +} + +func (s *Store) SaveQuery(ctx context.Context, input SavedQueryInput, now time.Time) (SavedQuery, error) { + ast, err := validateSavedQueryInput(input, now) + if err != nil { + return SavedQuery{}, err + } + astJSON, err := json.Marshal(ast) + if err != nil { + return SavedQuery{}, errors.New("encode saved query AST") + } + timestamp := now.UTC().Format(time.RFC3339Nano) + if input.ID == "" { + input.ID, err = storageID("query") + if err != nil { + return SavedQuery{}, err + } + _, err = s.control.ExecContext(ctx, `INSERT INTO saved_queries(organization_id,id,version,revision,name,description,query_text,ast_json,project_id,environment_id,service_id,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?,?,?,?,?)`, input.OrganizationID, input.ID, input.Name, input.Description, input.Query, string(astJSON), input.Scope.ProjectID, input.Scope.EnvironmentID, input.Scope.ServiceID, input.ActorUserID, input.ActorUserID, timestamp, timestamp) + if err != nil { + return SavedQuery{}, errors.New("create saved query") + } + } else { + if input.ExpectedRevision < 1 || model.ValidateSourceID(input.ID) != nil { + return SavedQuery{}, errors.New("saved query revision input is invalid") + } + result, updateErr := s.control.ExecContext(ctx, `UPDATE saved_queries SET revision=revision+1,name=?,description=?,query_text=?,ast_json=?,project_id=?,environment_id=?,service_id=?,updated_by=?,updated_at=? WHERE organization_id=? AND id=? AND revision=?`, input.Name, input.Description, input.Query, string(astJSON), input.Scope.ProjectID, input.Scope.EnvironmentID, input.Scope.ServiceID, input.ActorUserID, timestamp, input.OrganizationID, input.ID, input.ExpectedRevision) + if updateErr != nil { + return SavedQuery{}, errors.New("update saved query") + } + if changed, _ := result.RowsAffected(); changed != 1 { + return SavedQuery{}, errors.New("saved query revision conflict") + } + } + return s.SavedQuery(ctx, input.OrganizationID, input.ID) +} + +func (s *Store) SavedQuery(ctx context.Context, organizationID, id string) (SavedQuery, error) { + if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(id) != nil { + return SavedQuery{}, errors.New("saved query identity is invalid") + } + row := s.control.QueryRowContext(ctx, `SELECT version,revision,name,description,query_text,ast_json,project_id,environment_id,service_id,created_by,updated_by,created_at,updated_at FROM saved_queries WHERE organization_id=? AND id=?`, organizationID, id) + return scanSavedQuery(row, organizationID, id) +} + +func (s *Store) SavedQueries(ctx context.Context, organizationID string) ([]SavedQuery, error) { + if model.ValidateSourceID(organizationID) != nil { + return nil, errors.New("invalid organization identifier") + } + rows, err := s.control.QueryContext(ctx, `SELECT id,version,revision,name,description,query_text,ast_json,project_id,environment_id,service_id,created_by,updated_by,created_at,updated_at FROM saved_queries WHERE organization_id=? ORDER BY name,id`, organizationID) + if err != nil { + return nil, errors.New("list saved queries") + } + defer rows.Close() + var result []SavedQuery + for rows.Next() { + var id string + var value SavedQuery + var astJSON, createdAt, updatedAt string + value.OrganizationID = organizationID + if err = rows.Scan(&id, &value.Version, &value.Revision, &value.Name, &value.Description, &value.Query, &astJSON, &value.Scope.ProjectID, &value.Scope.EnvironmentID, &value.Scope.ServiceID, &value.CreatedBy, &value.UpdatedBy, &createdAt, &updatedAt); err != nil { + return nil, errors.New("read saved query") + } + value.ID = id + if err = decodeSavedQuery(&value, astJSON, createdAt, updatedAt); err != nil { + return nil, err + } + result = append(result, value) + } + if err = rows.Err(); err != nil { + return nil, errors.New("list saved queries") + } + return result, nil +} + +type rowScanner interface{ Scan(...any) error } + +func scanSavedQuery(row rowScanner, organizationID, id string) (SavedQuery, error) { + value := SavedQuery{OrganizationID: organizationID, ID: id} + var astJSON, createdAt, updatedAt string + if err := row.Scan(&value.Version, &value.Revision, &value.Name, &value.Description, &value.Query, &astJSON, &value.Scope.ProjectID, &value.Scope.EnvironmentID, &value.Scope.ServiceID, &value.CreatedBy, &value.UpdatedBy, &createdAt, &updatedAt); errors.Is(err, sql.ErrNoRows) { + return SavedQuery{}, errors.New("saved query not found") + } else if err != nil { + return SavedQuery{}, errors.New("read saved query") + } + if err := decodeSavedQuery(&value, astJSON, createdAt, updatedAt); err != nil { + return SavedQuery{}, err + } + return value, nil +} + +func decodeSavedQuery(value *SavedQuery, astJSON, createdAt, updatedAt string) error { + if value.Version != SavedQueryVersion || value.Revision < 1 || model.ValidateSourceID(value.OrganizationID) != nil || model.ValidateSourceID(value.ID) != nil || model.ValidateSourceID(value.CreatedBy) != nil || model.ValidateSourceID(value.UpdatedBy) != nil || !validResourceScope(value.Scope) || !boundedText(value.Name, 128, false) || !boundedText(value.Description, 1024, true) { + return errors.New("stored saved query is invalid") + } + parsed, err := query.Parse(value.Query, 100_000) + if err != nil || json.Unmarshal([]byte(astJSON), &value.AST) != nil || hydrateSavedAST(&value.AST) != nil || query.Validate(value.AST, 100_000) != nil { + return errors.New("stored saved query AST is invalid") + } + parsedJSON, _ := json.Marshal(parsed) + storedJSON, _ := json.Marshal(value.AST) + if string(parsedJSON) != string(storedJSON) { + return errors.New("stored saved query AST does not match query text") + } + value.CreatedAt, err = time.Parse(time.RFC3339Nano, createdAt) + if err != nil { + return errors.New("stored saved query created time is invalid") + } + value.UpdatedAt, err = time.Parse(time.RFC3339Nano, updatedAt) + if err != nil || value.UpdatedAt.Before(value.CreatedAt) { + return errors.New("stored saved query updated time is invalid") + } + return nil +} + +func hydrateSavedAST(ast *query.AST) error { + var err error + if ast.WindowText != "" { + ast.Window, err = time.ParseDuration(ast.WindowText) + } + if err == nil && ast.BucketText != "" { + ast.Bucket, err = time.ParseDuration(ast.BucketText) + } + if err != nil { + return errors.New("stored saved query duration is invalid") + } + return nil +} + +func (s *Store) SaveDashboard(ctx context.Context, input DashboardInput, now time.Time) (Dashboard, error) { + if err := validateDashboardInput(input, now); err != nil { + return Dashboard{}, err + } + if input.ExpectedRevision == 0 && input.ID != "" { + return Dashboard{}, errors.New("new dashboard identity is server-generated") + } + if input.ExpectedRevision > 0 && model.ValidateSourceID(input.ID) != nil { + return Dashboard{}, errors.New("dashboard revision input is invalid") + } + var err error + if input.ID == "" { + input.ID, err = storageID("dashboard") + if err != nil { + return Dashboard{}, err + } + } + panels := append([]DashboardPanel(nil), input.Panels...) + for index := range panels { + if panels[index].ID == "" { + panels[index].ID, err = storageID("panel") + if err != nil { + return Dashboard{}, err + } + } + } + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return Dashboard{}, errors.New("begin dashboard update") + } + defer tx.Rollback() + timestamp := now.UTC().Format(time.RFC3339Nano) + if input.ExpectedRevision == 0 { + _, err = tx.ExecContext(ctx, `INSERT INTO dashboards(organization_id,id,version,revision,slug,name,description,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?)`, input.OrganizationID, input.ID, input.Slug, input.Name, input.Description, input.ActorUserID, input.ActorUserID, timestamp, timestamp) + if err != nil { + return Dashboard{}, errors.New("create dashboard") + } + } else { + if model.ValidateSourceID(input.ID) != nil || input.ExpectedRevision < 1 { + return Dashboard{}, errors.New("dashboard revision input is invalid") + } + result, updateErr := tx.ExecContext(ctx, `UPDATE dashboards SET revision=revision+1,slug=?,name=?,description=?,updated_by=?,updated_at=? WHERE organization_id=? AND id=? AND revision=?`, input.Slug, input.Name, input.Description, input.ActorUserID, timestamp, input.OrganizationID, input.ID, input.ExpectedRevision) + if updateErr != nil { + return Dashboard{}, errors.New("update dashboard") + } + if changed, _ := result.RowsAffected(); changed != 1 { + return Dashboard{}, ErrDashboardRevisionConflict + } + if _, err = tx.ExecContext(ctx, `DELETE FROM dashboard_panels WHERE organization_id=? AND dashboard_id=?`, input.OrganizationID, input.ID); err != nil { + return Dashboard{}, errors.New("replace dashboard panels") + } + } + for _, panel := range panels { + if _, err = tx.ExecContext(ctx, `INSERT INTO dashboard_panels(organization_id,dashboard_id,id,position,title,visualization,saved_query_id) VALUES(?,?,?,?,?,?,?)`, input.OrganizationID, input.ID, panel.ID, panel.Position, panel.Title, panel.Visualization, panel.SavedQueryID); err != nil { + return Dashboard{}, errors.New("store dashboard panel") + } + } + if err = tx.Commit(); err != nil { + return Dashboard{}, errors.New("commit dashboard update") + } + return s.Dashboard(ctx, input.OrganizationID, input.Slug) +} + +func (s *Store) Dashboard(ctx context.Context, organizationID, slug string) (Dashboard, error) { + if model.ValidateSourceID(organizationID) != nil || !validSlug(slug) { + return Dashboard{}, errors.New("dashboard identity is invalid") + } + value := Dashboard{OrganizationID: organizationID} + var createdAt, updatedAt string + err := s.control.QueryRowContext(ctx, `SELECT id,version,revision,name,description,created_by,updated_by,created_at,updated_at FROM dashboards WHERE organization_id=? AND slug=?`, organizationID, slug).Scan(&value.ID, &value.Version, &value.Revision, &value.Name, &value.Description, &value.CreatedBy, &value.UpdatedBy, &createdAt, &updatedAt) + if errors.Is(err, sql.ErrNoRows) { + return Dashboard{}, errors.New("dashboard not found") + } + if err != nil { + return Dashboard{}, errors.New("read dashboard") + } + value.Slug = slug + value.CreatedAt, err = time.Parse(time.RFC3339Nano, createdAt) + if err != nil { + return Dashboard{}, errors.New("stored dashboard created time is invalid") + } + value.UpdatedAt, err = time.Parse(time.RFC3339Nano, updatedAt) + if err != nil || value.UpdatedAt.Before(value.CreatedAt) { + return Dashboard{}, errors.New("stored dashboard updated time is invalid") + } + rows, err := s.control.QueryContext(ctx, `SELECT id,position,title,visualization,saved_query_id FROM dashboard_panels WHERE organization_id=? AND dashboard_id=? ORDER BY position,id`, organizationID, value.ID) + if err != nil { + return Dashboard{}, errors.New("read dashboard panels") + } + defer rows.Close() + for rows.Next() { + var panel DashboardPanel + if err = rows.Scan(&panel.ID, &panel.Position, &panel.Title, &panel.Visualization, &panel.SavedQueryID); err != nil || validatePanel(panel) != nil { + return Dashboard{}, errors.New("stored dashboard panel is invalid") + } + value.Panels = append(value.Panels, panel) + } + if err = rows.Err(); err != nil || validateDashboard(value) != nil { + return Dashboard{}, errors.New("stored dashboard is invalid") + } + return value, nil +} + +func (s *Store) Dashboards(ctx context.Context, organizationID string) ([]Dashboard, error) { + if model.ValidateSourceID(organizationID) != nil { + return nil, errors.New("invalid organization identifier") + } + rows, err := s.control.QueryContext(ctx, `SELECT slug FROM dashboards WHERE organization_id=? ORDER BY name,slug`, organizationID) + if err != nil { + return nil, errors.New("list dashboards") + } + var slugs []string + for rows.Next() { + var slug string + if err = rows.Scan(&slug); err != nil { + _ = rows.Close() + return nil, errors.New("list dashboards") + } + slugs = append(slugs, slug) + } + if err = rows.Close(); err != nil || rows.Err() != nil { + return nil, errors.New("list dashboards") + } + result := make([]Dashboard, 0, len(slugs)) + for _, slug := range slugs { + value, loadErr := s.Dashboard(ctx, organizationID, slug) + if loadErr != nil { + return nil, loadErr + } + result = append(result, value) + } + return result, nil +} + +func (s *Store) ExportDashboard(ctx context.Context, organizationID, slug string) (DashboardExport, error) { + dashboard, err := s.Dashboard(ctx, organizationID, slug) + if err != nil { + return DashboardExport{}, err + } + queries := make([]SavedQueryDefinition, 0, len(dashboard.Panels)) + seen := map[string]bool{} + for _, panel := range dashboard.Panels { + if seen[panel.SavedQueryID] { + continue + } + value, loadErr := s.SavedQuery(ctx, organizationID, panel.SavedQueryID) + if loadErr != nil { + return DashboardExport{}, loadErr + } + seen[value.ID] = true + queries = append(queries, SavedQueryDefinition{ID: value.ID, Name: value.Name, Description: value.Description, Query: value.Query, Scope: value.Scope}) + } + sort.Slice(queries, func(i, j int) bool { return queries[i].ID < queries[j].ID }) + definition := DashboardDefinition{Slug: dashboard.Slug, Name: dashboard.Name, Description: dashboard.Description, Panels: dashboard.Panels} + return DashboardExport{Version: DashboardVersion, Dashboard: definition, SavedQueries: queries}, nil +} + +// ImportDashboard validates one source-control-safe export and creates its +// queries, dashboard, and panels atomically with new server-owned identities. +// Tenant and actor metadata are supplied independently of the bundle. +func (s *Store) ImportDashboard(ctx context.Context, input DashboardImportInput, now time.Time) (Dashboard, error) { + if input.Bundle.Version != DashboardVersion || model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.ActorUserID) != nil || input.MaxRows < 1 || input.MaxRows > 100_000 || now.IsZero() || len(input.Bundle.SavedQueries) > MaxDashboardPanels { + return Dashboard{}, errors.New("dashboard import is invalid") + } + queryIDs := make(map[string]string, len(input.Bundle.SavedQueries)) + type importedQuery struct { + definition SavedQueryDefinition + id string + astJSON string + ast query.AST + } + queries := make([]importedQuery, 0, len(input.Bundle.SavedQueries)) + for _, definition := range input.Bundle.SavedQueries { + if model.ValidateSourceID(definition.ID) != nil || queryIDs[definition.ID] != "" { + return Dashboard{}, errors.New("dashboard import query identity is invalid or duplicated") + } + ast, err := validateSavedQueryInput(SavedQueryInput{OrganizationID: input.OrganizationID, ActorUserID: input.ActorUserID, MaxRows: input.MaxRows, Name: definition.Name, Description: definition.Description, Query: definition.Query, Scope: definition.Scope}, now) + if err != nil { + return Dashboard{}, err + } + encoded, err := json.Marshal(ast) + if err != nil { + return Dashboard{}, errors.New("encode imported saved query AST") + } + id, err := storageID("query") + if err != nil { + return Dashboard{}, err + } + queryIDs[definition.ID] = id + queries = append(queries, importedQuery{definition: definition, id: id, astJSON: string(encoded), ast: ast}) + } + queryASTs := make(map[string]query.AST, len(queries)) + for _, imported := range queries { + queryASTs[imported.definition.ID] = imported.ast + } + panels := make([]DashboardPanel, len(input.Bundle.Dashboard.Panels)) + referenced := make(map[string]bool, len(queries)) + for index, panel := range input.Bundle.Dashboard.Panels { + mapped := queryIDs[panel.SavedQueryID] + if mapped == "" { + return Dashboard{}, errors.New("dashboard import panel references an unknown saved query") + } + ast := queryASTs[panel.SavedQueryID] + if panel.Visualization == "stat" && ast.Summary == nil || panel.Visualization == "timeseries" && (ast.Summary == nil || ast.Bucket <= 0) { + return Dashboard{}, errors.New("dashboard import presentation does not match its saved query") + } + panelID, err := storageID("panel") + if err != nil { + return Dashboard{}, err + } + panel.ID, panel.SavedQueryID = panelID, mapped + panels[index] = panel + referenced[panel.SavedQueryID] = true + } + if len(referenced) != len(queries) { + return Dashboard{}, errors.New("dashboard import contains an unreferenced saved query") + } + dashboardID, err := storageID("dashboard") + if err != nil { + return Dashboard{}, err + } + dashboardInput := DashboardInput{ID: dashboardID, ExpectedRevision: 1, OrganizationID: input.OrganizationID, ActorUserID: input.ActorUserID, Slug: input.Bundle.Dashboard.Slug, Name: input.Bundle.Dashboard.Name, Description: input.Bundle.Dashboard.Description, Panels: panels} + if err = validateDashboardInput(dashboardInput, now); err != nil { + return Dashboard{}, err + } + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return Dashboard{}, errors.New("begin dashboard import") + } + defer tx.Rollback() + timestamp := now.UTC().Format(time.RFC3339Nano) + for _, imported := range queries { + definition := imported.definition + _, err = tx.ExecContext(ctx, `INSERT INTO saved_queries(organization_id,id,version,revision,name,description,query_text,ast_json,project_id,environment_id,service_id,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?,?,?,?,?)`, input.OrganizationID, imported.id, definition.Name, definition.Description, strings.TrimSpace(definition.Query), imported.astJSON, definition.Scope.ProjectID, definition.Scope.EnvironmentID, definition.Scope.ServiceID, input.ActorUserID, input.ActorUserID, timestamp, timestamp) + if err != nil { + return Dashboard{}, errors.New("import saved query") + } + } + _, err = tx.ExecContext(ctx, `INSERT INTO dashboards(organization_id,id,version,revision,slug,name,description,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?)`, input.OrganizationID, dashboardID, input.Bundle.Dashboard.Slug, input.Bundle.Dashboard.Name, input.Bundle.Dashboard.Description, input.ActorUserID, input.ActorUserID, timestamp, timestamp) + if err != nil { + return Dashboard{}, errors.New("import dashboard") + } + for _, panel := range panels { + if _, err = tx.ExecContext(ctx, `INSERT INTO dashboard_panels(organization_id,dashboard_id,id,position,title,visualization,saved_query_id) VALUES(?,?,?,?,?,?,?)`, input.OrganizationID, dashboardID, panel.ID, panel.Position, panel.Title, panel.Visualization, panel.SavedQueryID); err != nil { + return Dashboard{}, errors.New("import dashboard panel") + } + } + if err = tx.Commit(); err != nil { + return Dashboard{}, errors.New("commit dashboard import") + } + return s.Dashboard(ctx, input.OrganizationID, input.Bundle.Dashboard.Slug) +} + +func validateSavedQueryInput(input SavedQueryInput, now time.Time) (query.AST, error) { + if model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.ActorUserID) != nil || !boundedText(input.Name, 128, false) || !boundedText(input.Description, 1024, true) || !validResourceScope(input.Scope) || input.MaxRows < 1 || input.MaxRows > 100_000 || now.IsZero() { + return query.AST{}, errors.New("saved query input is invalid") + } + ast, err := query.Parse(strings.TrimSpace(input.Query), input.MaxRows) + if err != nil { + return query.AST{}, fmt.Errorf("saved query: %w", err) + } + return ast, nil +} + +func validateDashboardInput(input DashboardInput, now time.Time) error { + if model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.ActorUserID) != nil || !validSlug(input.Slug) || !boundedText(input.Name, 128, false) || !boundedText(input.Description, 1024, true) || len(input.Panels) > MaxDashboardPanels || now.IsZero() { + return errors.New("dashboard input is invalid") + } + positions := map[int]bool{} + ids := map[string]bool{} + for _, panel := range input.Panels { + if err := validatePanelInput(panel); err != nil || positions[panel.Position] || panel.ID != "" && ids[panel.ID] { + return errors.New("dashboard panel input is invalid") + } + positions[panel.Position] = true + if panel.ID != "" { + ids[panel.ID] = true + } + } + return nil +} + +func validateDashboard(value Dashboard) error { + if value.Version != DashboardVersion || value.Revision < 1 || model.ValidateSourceID(value.OrganizationID) != nil || model.ValidateSourceID(value.ID) != nil || model.ValidateSourceID(value.CreatedBy) != nil || model.ValidateSourceID(value.UpdatedBy) != nil || !validSlug(value.Slug) || !boundedText(value.Name, 128, false) || !boundedText(value.Description, 1024, true) || len(value.Panels) > MaxDashboardPanels { + return errors.New("dashboard is invalid") + } + return nil +} + +func validatePanel(panel DashboardPanel) error { + if model.ValidateSourceID(panel.ID) != nil { + return errors.New("dashboard panel is invalid") + } + return validatePanelInput(panel) +} + +func validatePanelInput(panel DashboardPanel) error { + if panel.ID != "" && model.ValidateSourceID(panel.ID) != nil || panel.Position < 0 || panel.Position >= 64 || !boundedText(panel.Title, 128, false) || model.ValidateSourceID(panel.SavedQueryID) != nil { + return errors.New("dashboard panel is invalid") + } + switch panel.Visualization { + case "table", "stat", "timeseries": + return nil + default: + return errors.New("dashboard panel visualization is invalid") + } +} + +func validResourceScope(scope ResourceScope) bool { + values := []string{scope.ProjectID, scope.EnvironmentID, scope.ServiceID} + for _, value := range values { + if value != "" && model.ValidateSourceID(value) != nil { + return false + } + } + if scope.EnvironmentID != "" && scope.ProjectID == "" { + return false + } + if scope.ServiceID != "" && (scope.ProjectID == "" || scope.EnvironmentID == "") { + return false + } + return true +} + +func boundedText(value string, maximum int, empty bool) bool { + return utf8.ValidString(value) && !strings.ContainsAny(value, "\x00\r\n") && len(value) <= maximum && (empty || value != "") +} + +func validSlug(value string) bool { + if len(value) < 2 || len(value) > 63 || value[0] < 'a' || value[0] > 'z' { + return false + } + for _, character := range value { + if character != '-' && (character < 'a' || character > 'z') && (character < '0' || character > '9') { + return false + } + } + return true +} + +func storageID(prefix string) (string, error) { + random := make([]byte, 18) + if _, err := rand.Read(random); err != nil { + return "", errors.New("cryptographic randomness unavailable") + } + return prefix + "_" + base64.RawURLEncoding.EncodeToString(random), nil +} diff --git a/internal/storage/dashboards_test.go b/internal/storage/dashboards_test.go new file mode 100644 index 0000000..27d1a06 --- /dev/null +++ b/internal/storage/dashboards_test.go @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" +) + +func TestSavedQueriesAreTypedVersionedAndOptimisticallyUpdated(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 10, 0, 0, 0, time.UTC) + input := SavedQueryInput{ + OrganizationID: "organization-a", Name: "Recent failures", + Description: "Recent failed application requests grouped by route.", + Query: `logs | where status >= 500 | window 1h | summarize count() by route | sort count desc | limit 50`, + Scope: ResourceScope{ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}, + ActorUserID: "operator-a", MaxRows: 1000, + } + created, err := store.SaveQuery(ctx, input, now) + if err != nil { + t.Fatal(err) + } + if created.Version != SavedQueryVersion || created.Revision != 1 || created.ID == "" || created.AST.Signal != "logs" || created.AST.Window != time.Hour || created.CreatedAt != now || created.UpdatedAt != now { + t.Fatalf("created=%+v", created) + } + input.ID, input.ExpectedRevision = created.ID, created.Revision + input.Description = "Reviewed application failures grouped by normalized route." + updated, err := store.SaveQuery(ctx, input, now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if updated.Revision != 2 || updated.Description != input.Description || updated.UpdatedBy != "operator-a" || !updated.UpdatedAt.Equal(now.Add(time.Minute)) { + t.Fatalf("updated=%+v", updated) + } + if _, err = store.SaveQuery(ctx, input, now.Add(2*time.Minute)); err == nil { + t.Fatal("stale saved-query revision was accepted") + } + queries, err := store.SavedQueries(ctx, "organization-a") + if err != nil || len(queries) != 1 || queries[0].Revision != 2 { + t.Fatalf("queries=%+v err=%v", queries, err) + } + if _, err = store.control.Exec(`UPDATE saved_queries SET ast_json='{}' WHERE organization_id='organization-a' AND id=?`, created.ID); err != nil { + t.Fatal(err) + } + if _, err = store.SavedQuery(ctx, "organization-a", created.ID); err == nil { + t.Fatal("query text and stored AST disagreement was accepted") + } +} + +func TestDashboardPanelsRemainOrganizationScopedAndExportSafeDefinitions(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 10, 0, 0, 0, time.UTC) + queryA, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", Name: "Request rate", Description: "Five-minute request counts.", Query: `logs | window 1h | summarize count() by window(5m) | limit 50`, ActorUserID: "operator-a", MaxRows: 1000}, now) + if err != nil { + t.Fatal(err) + } + queryB, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-b", Name: "Other organization", Description: "Must not cross the tenant boundary.", Query: `logs | limit 10`, ActorUserID: "operator-b", MaxRows: 1000}, now) + if err != nil { + t.Fatal(err) + } + input := DashboardInput{ + OrganizationID: "organization-a", Slug: "operations", Name: "Operations", + Description: "Recent service activity and evidence.", ActorUserID: "operator-a", + Panels: []DashboardPanel{{Position: 0, Title: "Request rate", Visualization: "timeseries", SavedQueryID: queryA.ID}}, + } + created, err := store.SaveDashboard(ctx, input, now) + if err != nil { + t.Fatal(err) + } + if created.Version != DashboardVersion || created.Revision != 1 || len(created.Panels) != 1 || created.Panels[0].ID == "" { + t.Fatalf("created=%+v", created) + } + unsafe := input + unsafe.Slug = "cross-tenant" + unsafe.Panels = []DashboardPanel{{Position: 0, Title: "Other", Visualization: "table", SavedQueryID: queryB.ID}} + if _, err = store.SaveDashboard(ctx, unsafe, now); err == nil { + t.Fatal("cross-organization saved query entered dashboard") + } + input.ID, input.ExpectedRevision = created.ID, created.Revision + input.Name = "Service operations" + input.Panels[0].ID = created.Panels[0].ID + updated, err := store.SaveDashboard(ctx, input, now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if updated.Revision != 2 || updated.Name != input.Name { + t.Fatalf("updated=%+v", updated) + } + if _, err = store.SaveDashboard(ctx, input, now.Add(2*time.Minute)); !errors.Is(err, ErrDashboardRevisionConflict) { + t.Fatalf("stale dashboard revision err=%v", err) + } + exported, err := store.ExportDashboard(ctx, "organization-a", "operations") + if err != nil { + t.Fatal(err) + } + body, err := json.Marshal(exported) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"organization-a", "operator-a", "created_at", "updated_at", "ast"} { + if strings.Contains(string(body), forbidden) { + t.Fatalf("private runtime metadata %q entered export: %s", forbidden, body) + } + } + if !strings.Contains(string(body), `"version":1`) || !strings.Contains(string(body), `"query":"logs | window 1h`) { + t.Fatalf("export=%s", body) + } +} + +func TestDashboardValidationBoundsScopePanelsAndServerGeneratedIdentity(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 10, 0, 0, 0, time.UTC) + if _, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", Name: "Invalid scope", Query: `logs | limit 10`, Scope: ResourceScope{ServiceID: "service-a"}, ActorUserID: "operator-a", MaxRows: 1000}, now); err == nil { + t.Fatal("service-only query scope was accepted") + } + queryValue, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", Name: "Valid", Query: `logs | limit 10`, ActorUserID: "operator-a", MaxRows: 1000}, now) + if err != nil { + t.Fatal(err) + } + if _, err = store.SaveDashboard(ctx, DashboardInput{ID: "caller-selected", OrganizationID: "organization-a", Slug: "invalid-id", Name: "Invalid", ActorUserID: "operator-a"}, now); err == nil { + t.Fatal("caller-selected new dashboard identity was accepted") + } + duplicate := DashboardInput{OrganizationID: "organization-a", Slug: "duplicate-panels", Name: "Duplicate panels", ActorUserID: "operator-a", Panels: []DashboardPanel{ + {Position: 0, Title: "First", Visualization: "table", SavedQueryID: queryValue.ID}, + {Position: 0, Title: "Second", Visualization: "stat", SavedQueryID: queryValue.ID}, + }} + if _, err = store.SaveDashboard(ctx, duplicate, now); err == nil { + t.Fatal("duplicate dashboard panel positions were accepted") + } +} + +func TestDashboardImportIsAtomicTenantIndependentAndRevalidated(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 11, 0, 0, 0, time.UTC) + bundle := DashboardExport{ + Version: DashboardVersion, + Dashboard: DashboardDefinition{Slug: "operations", Name: "Operations", Description: "A portable service view.", Panels: []DashboardPanel{ + {ID: "old-panel-1", Position: 0, Title: "Recent failures", Visualization: "table", SavedQueryID: "portable-query-1"}, + }}, + SavedQueries: []SavedQueryDefinition{{ID: "portable-query-1", Name: "Recent failures", Description: "Bounded failures.", Query: `logs | where status >= 500 | window 1h | limit 50`}}, + } + imported, err := store.ImportDashboard(ctx, DashboardImportInput{OrganizationID: "organization-b", ActorUserID: "operator-b", MaxRows: 1_000, Bundle: bundle}, now) + if err != nil { + t.Fatal(err) + } + if imported.OrganizationID != "organization-b" || imported.CreatedBy != "operator-b" || imported.ID == "" || len(imported.Panels) != 1 || imported.Panels[0].ID == "old-panel-1" || imported.Panels[0].SavedQueryID == "portable-query-1" { + t.Fatalf("imported=%+v", imported) + } + queries, err := store.SavedQueries(ctx, "organization-b") + if err != nil || len(queries) != 1 || queries[0].OrganizationID != "organization-b" || queries[0].CreatedBy != "operator-b" || queries[0].ID != imported.Panels[0].SavedQueryID { + t.Fatalf("queries=%+v err=%v", queries, err) + } + invalid := bundle + invalid.Dashboard.Slug = "partial-import" + invalid.SavedQueries = append(invalid.SavedQueries, SavedQueryDefinition{ID: "unused-query-2", Name: "Unused", Query: `logs | limit 10`}) + if _, err = store.ImportDashboard(ctx, DashboardImportInput{OrganizationID: "organization-b", ActorUserID: "operator-b", MaxRows: 1_000, Bundle: invalid}, now); err == nil { + t.Fatal("unreferenced imported query was accepted") + } + if dashboards, listErr := store.Dashboards(ctx, "organization-b"); listErr != nil || len(dashboards) != 1 { + t.Fatalf("failed import left partial dashboard: dashboards=%+v err=%v", dashboards, listErr) + } + if queries, listErr := store.SavedQueries(ctx, "organization-b"); listErr != nil || len(queries) != 1 { + t.Fatalf("failed import left partial queries: queries=%+v err=%v", queries, listErr) + } + incompatible := bundle + incompatible.Dashboard.Slug = "invalid-presentation" + incompatible.Dashboard.Panels[0].Visualization = "timeseries" + if _, err = store.ImportDashboard(ctx, DashboardImportInput{OrganizationID: "organization-b", ActorUserID: "operator-b", MaxRows: 1_000, Bundle: incompatible}, now); err == nil { + t.Fatal("timeseries dashboard without a bucketed summary was accepted") + } +} diff --git a/internal/storage/descriptors.go b/internal/storage/descriptors.go new file mode 100644 index 0000000..ded95f0 --- /dev/null +++ b/internal/storage/descriptors.go @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" +) + +type StoredProposal struct { + Proposal schema.Proposal `json:"proposal"` + Status string `json:"status"` + FirstSeenAt time.Time `json:"first_seen_at"` + LastSeenAt time.Time `json:"last_seen_at"` +} + +type fieldEvidence struct { + descriptor schema.Descriptor + count int64 + bytes int64 +} + +func (s *Store) descriptorProposal(ctx context.Context, organizationID string, signal model.Signal, field string) (StoredProposal, error) { + if err := model.ValidateSourceID(organizationID); err != nil { + return StoredProposal{}, errors.New("invalid organization identifier") + } + var descriptorJSON, examplesJSON, firstSeen, lastSeen string + var proposal StoredProposal + err := s.control.QueryRowContext(ctx, `SELECT descriptor_json,observed_values,estimated_bytes,example_queries_json,status,first_seen_at,last_seen_at FROM descriptor_proposals WHERE organization_id=? AND signal=? AND field=?`, organizationID, signal, query.CanonicalField(field)).Scan(&descriptorJSON, &proposal.Proposal.ObservedValues, &proposal.Proposal.EstimatedBytes, &examplesJSON, &proposal.Status, &firstSeen, &lastSeen) + if errors.Is(err, sql.ErrNoRows) { + return StoredProposal{}, errors.New("descriptor proposal not found") + } + if err != nil { + return StoredProposal{}, errors.New("load descriptor proposal") + } + if err = json.Unmarshal([]byte(descriptorJSON), &proposal.Proposal.Descriptor); err != nil { + return StoredProposal{}, errors.New("decode descriptor proposal") + } + if err = json.Unmarshal([]byte(examplesJSON), &proposal.Proposal.ExampleQueries); err != nil { + return StoredProposal{}, errors.New("decode descriptor examples") + } + proposal.FirstSeenAt, err = time.Parse(time.RFC3339Nano, firstSeen) + if err != nil { + return StoredProposal{}, errors.New("descriptor proposal first-seen time is invalid") + } + proposal.LastSeenAt, err = time.Parse(time.RFC3339Nano, lastSeen) + if err != nil || proposal.Proposal.Validate() != nil || proposal.Status != "pending" && proposal.Status != "activated" && proposal.Status != "rejected" { + return StoredProposal{}, errors.New("stored descriptor proposal is invalid") + } + return proposal, nil +} + +func (s *Store) markProposalActivated(ctx context.Context, organizationID string, descriptor schema.Descriptor) error { + encoded, err := json.Marshal(descriptor) + if err != nil { + return errors.New("encode activated descriptor") + } + result, err := s.control.ExecContext(ctx, `UPDATE descriptor_proposals SET descriptor_json=?,status='activated' WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, string(encoded), organizationID, descriptor.Signal, descriptor.Field) + if err != nil { + return errors.New("acknowledge activated descriptor") + } + if changed, _ := result.RowsAffected(); changed == 1 { + return nil + } + proposal, err := s.descriptorProposal(ctx, organizationID, descriptor.Signal, descriptor.Field) + if err != nil || proposal.Status != "activated" || !sameDescriptorIgnoringProjection(proposal.Proposal.Descriptor, descriptor) || proposal.Proposal.Descriptor.ProjectionVersion != descriptor.ProjectionVersion { + return errors.New("activated descriptor acknowledgement is inconsistent") + } + return nil +} + +func (s *Store) RejectDescriptorProposal(ctx context.Context, organizationID string, signal model.Signal, field string) error { + if err := model.ValidateSourceID(organizationID); err != nil { + return errors.New("invalid organization identifier") + } + lock := s.namedLock("organization:" + organizationID) + lock.Lock() + defer lock.Unlock() + registry, _, err := s.ActiveDescriptors(ctx, organizationID) + if err != nil { + return err + } + if _, active := registry.Lookup(signal, field); active { + return errors.New("active descriptor proposal cannot be rejected") + } + result, err := s.control.ExecContext(ctx, `UPDATE descriptor_proposals SET status='rejected' WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, organizationID, signal, query.CanonicalField(field)) + if err != nil { + return errors.New("reject descriptor proposal") + } + if changed, _ := result.RowsAffected(); changed == 1 { + return nil + } + proposal, err := s.descriptorProposal(ctx, organizationID, signal, field) + if err == nil && proposal.Status == "rejected" { + return nil + } + return errors.New("pending descriptor proposal not found") +} + +func (s *Store) recordDescriptorProposals(ctx context.Context, organizationID string, batch model.Batch, segmentDigest string, now time.Time) error { + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if err = recordDescriptorProposalsTx(ctx, tx, organizationID, batch, segmentDigest, now); err != nil { + return err + } + if err = tx.Commit(); err != nil { + return errors.New("commit descriptor proposals") + } + return nil +} + +func recordDescriptorProposalsTx(ctx context.Context, tx *sql.Tx, organizationID string, batch model.Batch, segmentDigest string, now time.Time) error { + if err := model.ValidateSourceID(organizationID); err != nil || !validDigest(segmentDigest) || now.IsZero() { + return errors.New("descriptor proposal identity is invalid") + } + evidence := map[string]fieldEvidence{} + for _, observation := range batch.Records { + for field, value := range observation.Attributes { + canonical := query.CanonicalField(field) + if _, known := query.BuiltinDescriptor(batch.Signal, canonical); known { + continue + } + current, exists := evidence[canonical] + if !exists && len(evidence) >= model.MaxDistinctFields { + return errors.New("descriptor proposal field limit exceeded") + } + descriptor := proposedDescriptor(batch.Signal, canonical, inferType(value)) + if err := descriptor.Validate(); err != nil { + continue + } + if current.count > 0 { + descriptor.Type = mergeType(current.descriptor.Type, descriptor.Type) + } + current.descriptor = descriptor + current.count++ + current.bytes += int64(len(canonical) + len(value)) + evidence[canonical] = current + } + } + if len(evidence) == 0 { + return nil + } + fields := make([]string, 0, len(evidence)) + for field := range evidence { + fields = append(fields, field) + } + sort.Strings(fields) + for _, field := range fields { + observed := evidence[field] + result, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO descriptor_proposal_segments(segment_digest,organization_id,signal,field,observed_values,estimated_bytes) VALUES(?,?,?,?,?,?)`, segmentDigest, organizationID, batch.Signal, field, observed.count, observed.bytes) + if err != nil { + return errors.New("record descriptor proposal segment") + } + inserted, err := result.RowsAffected() + if err != nil { + return errors.New("inspect descriptor proposal segment") + } + if inserted == 0 { + continue + } + if err = upsertProposal(ctx, tx, organizationID, batch.Signal, observed, now); err != nil { + return err + } + } + return nil +} + +func upsertProposal(ctx context.Context, tx *sql.Tx, organizationID string, signal model.Signal, observed fieldEvidence, now time.Time) error { + field := observed.descriptor.Field + var descriptorJSON, examplesJSON, status string + var count, estimated int64 + err := tx.QueryRowContext(ctx, `SELECT descriptor_json,observed_values,estimated_bytes,example_queries_json,status FROM descriptor_proposals WHERE organization_id=? AND signal=? AND field=?`, organizationID, signal, field).Scan(&descriptorJSON, &count, &estimated, &examplesJSON, &status) + if errors.Is(err, sql.ErrNoRows) { + descriptorBody, marshalErr := json.Marshal(observed.descriptor) + if marshalErr != nil { + return errors.New("encode descriptor proposal") + } + examples := []string{fmt.Sprintf(`%s | where %s == "value" | limit 50`, signal, field)} + exampleBody, marshalErr := json.Marshal(examples) + if marshalErr != nil { + return errors.New("encode descriptor examples") + } + _, err = tx.ExecContext(ctx, `INSERT INTO descriptor_proposals(organization_id,signal,field,descriptor_json,observed_values,estimated_bytes,example_queries_json,status,first_seen_at,last_seen_at) VALUES(?,?,?,?,?,?,?,'pending',?,?)`, organizationID, signal, field, string(descriptorBody), observed.count, observed.bytes, string(exampleBody), now.UTC().Format(time.RFC3339Nano), now.UTC().Format(time.RFC3339Nano)) + if err != nil { + return errors.New("create descriptor proposal") + } + return nil + } + if err != nil { + return errors.New("load descriptor proposal") + } + if status != "pending" { + return nil + } + var descriptor schema.Descriptor + if err = json.Unmarshal([]byte(descriptorJSON), &descriptor); err != nil || descriptor.Validate() != nil { + return errors.New("stored descriptor proposal is invalid") + } + descriptor.Type = mergeType(descriptor.Type, observed.descriptor.Type) + if count > math.MaxInt64-observed.count || estimated > math.MaxInt64-observed.bytes { + return errors.New("descriptor proposal evidence overflow") + } + descriptorBody, err := json.Marshal(descriptor) + if err != nil { + return errors.New("encode descriptor proposal") + } + _, err = tx.ExecContext(ctx, `UPDATE descriptor_proposals SET descriptor_json=?,observed_values=?,estimated_bytes=?,last_seen_at=? WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, string(descriptorBody), count+observed.count, estimated+observed.bytes, now.UTC().Format(time.RFC3339Nano), organizationID, signal, field) + if err != nil { + return errors.New("update descriptor proposal") + } + return nil +} + +func (s *Store) DescriptorProposals(ctx context.Context, organizationID string) ([]StoredProposal, error) { + if err := model.ValidateSourceID(organizationID); err != nil { + return nil, errors.New("invalid organization identifier") + } + rows, err := s.control.QueryContext(ctx, `SELECT descriptor_json,observed_values,estimated_bytes,example_queries_json,status,first_seen_at,last_seen_at FROM descriptor_proposals WHERE organization_id=? ORDER BY signal,field`, organizationID) + if err != nil { + return nil, errors.New("list descriptor proposals") + } + defer rows.Close() + var proposals []StoredProposal + for rows.Next() { + var descriptorJSON, examplesJSON, firstSeen, lastSeen string + var proposal StoredProposal + if err = rows.Scan(&descriptorJSON, &proposal.Proposal.ObservedValues, &proposal.Proposal.EstimatedBytes, &examplesJSON, &proposal.Status, &firstSeen, &lastSeen); err != nil { + return nil, errors.New("read descriptor proposal") + } + if err = json.Unmarshal([]byte(descriptorJSON), &proposal.Proposal.Descriptor); err != nil { + return nil, errors.New("decode descriptor proposal") + } + if err = json.Unmarshal([]byte(examplesJSON), &proposal.Proposal.ExampleQueries); err != nil { + return nil, errors.New("decode descriptor examples") + } + proposal.FirstSeenAt, err = time.Parse(time.RFC3339Nano, firstSeen) + if err != nil { + return nil, errors.New("descriptor proposal first-seen time is invalid") + } + proposal.LastSeenAt, err = time.Parse(time.RFC3339Nano, lastSeen) + if err != nil || proposal.Proposal.Validate() != nil || proposal.Status != "pending" && proposal.Status != "activated" && proposal.Status != "rejected" { + return nil, errors.New("stored descriptor proposal is invalid") + } + proposals = append(proposals, proposal) + } + if err = rows.Err(); err != nil { + return nil, errors.New("list descriptor proposals") + } + return proposals, nil +} + +func proposedDescriptor(signal model.Signal, field string, valueType schema.Type) schema.Descriptor { + return schema.Descriptor{ + Version: schema.DescriptorVersion, Signal: signal, Field: field, + Type: valueType, Meaning: "Observed unreviewed field awaiting administrator classification.", + Sensitivity: schema.SensitivitySensitive, Cardinality: schema.CardinalityHigh, + Index: schema.IndexNone, Retention: schema.RetentionRaw, ProjectionVersion: 1, + } +} + +func inferType(value string) schema.Type { + if _, err := strconv.ParseInt(value, 10, 64); err == nil { + return schema.TypeInteger + } + if number, err := strconv.ParseFloat(value, 64); err == nil && !math.IsNaN(number) && !math.IsInf(number, 0) { + return schema.TypeFloat + } + if value == "true" || value == "false" { + return schema.TypeBoolean + } + if _, err := time.Parse(time.RFC3339Nano, value); err == nil { + return schema.TypeTime + } + return schema.TypeString +} + +func mergeType(left, right schema.Type) schema.Type { + if left == right { + return left + } + if left == schema.TypeInteger && right == schema.TypeFloat || left == schema.TypeFloat && right == schema.TypeInteger { + return schema.TypeFloat + } + return schema.TypeString +} + +func validDigest(value string) bool { + if len(value) != 64 { + return false + } + for _, character := range value { + if !strings.ContainsRune("0123456789abcdef", character) { + return false + } + } + return true +} diff --git a/internal/storage/descriptors_test.go b/internal/storage/descriptors_test.go new file mode 100644 index 0000000..3573184 --- /dev/null +++ b/internal/storage/descriptors_test.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/schema" +) + +func TestDescriptorProposalsAreSegmentIdempotentAndDefaultSensitive(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 7, 30, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{ + {Timestamp: now, Name: "custom.metric", Value: floatPointer(1), Attributes: map[string]string{"workshop.queue_depth": "9238471928374", "http.route": "/known", "invalid-field": "ignored"}}, + {Timestamp: now, Name: "custom.metric", Value: floatPointer(2), Attributes: map[string]string{"workshop.queue_depth": "9238471928375"}}, + }} + ack, err := store.Ingest(ctx, token, batch, now) + if err != nil { + t.Fatal(err) + } + projectAll(t, store) + proposals, err := store.DescriptorProposals(ctx, "organization-a") + if err != nil { + t.Fatal(err) + } + if len(proposals) != 1 { + t.Fatalf("proposals=%+v", proposals) + } + proposal := proposals[0] + if proposal.Proposal.Descriptor.Field != "workshop.queue_depth" || proposal.Proposal.Descriptor.Type != schema.TypeInteger || proposal.Proposal.Descriptor.Sensitivity != schema.SensitivitySensitive || proposal.Proposal.Descriptor.Index != schema.IndexNone || proposal.Proposal.ObservedValues != 2 || proposal.Status != "pending" { + t.Fatalf("proposal=%+v", proposal) + } + if len(proposal.Proposal.ExampleQueries) != 1 || proposal.Proposal.ExampleQueries[0] != `metrics | where workshop.queue_depth == "value" | limit 50` { + t.Fatalf("examples=%v", proposal.Proposal.ExampleQueries) + } + encoded, err := json.Marshal(proposals) + if err != nil || strings.Contains(string(encoded), "9238471928374") || strings.Contains(string(encoded), "9238471928375") { + t.Fatalf("observed values entered proposal metadata: %s err=%v", encoded, err) + } + if err = store.recordDescriptorProposals(ctx, "organization-a", batch, ack.Digest, now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + proposals, err = store.DescriptorProposals(ctx, "organization-a") + if err != nil || proposals[0].Proposal.ObservedValues != 2 || !proposals[0].LastSeenAt.Equal(now) { + t.Fatalf("idempotent proposals=%+v err=%v", proposals, err) + } +} + +func TestDescriptorProposalTypesWidenWithoutMixingOrganizations(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 7, 30, 0, 0, time.UTC) + for index, organization := range []string{"organization-a", "organization-b"} { + source := "source-" + string(rune('a'+index)) + token, err := store.CreateSource(ctx, source, model.Scope{OrganizationID: organization, ProjectID: "project", EnvironmentID: "production", ServiceID: "service"}) + if err != nil { + t.Fatal(err) + } + value := "10" + if organization == "organization-b" { + value = "label" + } + batch := model.Batch{Version: model.BatchVersion, SourceID: source, StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: now, Name: "custom.metric", Value: floatPointer(1), Attributes: map[string]string{"workshop.value": value}}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + if organization == "organization-a" { + batch.Sequence = 2 + batch.Records[0].Attributes["workshop.value"] = "10.5" + if _, err = store.Ingest(ctx, token, batch, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + } + } + projectAll(t, store) + proposalA, err := store.DescriptorProposals(ctx, "organization-a") + if err != nil { + t.Fatal(err) + } + proposalB, err := store.DescriptorProposals(ctx, "organization-b") + if err != nil { + t.Fatal(err) + } + if proposalA[0].Proposal.Descriptor.Type != schema.TypeFloat || proposalB[0].Proposal.Descriptor.Type != schema.TypeString { + t.Fatalf("organization-a=%+v organization-b=%+v", proposalA, proposalB) + } +} + +func floatPointer(value float64) *float64 { return &value } diff --git a/internal/storage/incidents.go b/internal/storage/incidents.go new file mode 100644 index 0000000..c6ec49e --- /dev/null +++ b/internal/storage/incidents.go @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +const ( + AlertRuleVersion = 1 + IncidentVersion = 1 + MaxDueRules = 64 +) + +type AlertRule struct { + Version int `json:"version"` + Revision int `json:"revision"` + OrganizationID string `json:"organization_id"` + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + SavedQueryID string `json:"saved_query_id"` + Severity string `json:"severity"` + MinimumMatches int `json:"minimum_matches"` + RequiredConsecutive int `json:"required_consecutive"` + EvaluationInterval time.Duration `json:"evaluation_interval"` + Enabled bool `json:"enabled"` + LastEvaluatedAt *time.Time `json:"last_evaluated_at,omitempty"` + NextEvaluationAt time.Time `json:"next_evaluation_at"` + LastResult *int `json:"last_result,omitempty"` + LastError string `json:"last_error,omitempty"` + CreatedBy string `json:"created_by"` + UpdatedBy string `json:"updated_by"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type AlertRuleInput struct { + ID string + ExpectedRevision int + OrganizationID string + Name string + Description string + SavedQueryID string + Severity string + MinimumMatches int + RequiredConsecutive int + EvaluationInterval time.Duration + Enabled bool + ActorUserID string +} + +type Incident struct { + Version int `json:"version"` + OrganizationID string `json:"organization_id"` + ID string `json:"id"` + RuleID string `json:"rule_id"` + State string `json:"state"` + Severity string `json:"severity"` + Title string `json:"title"` + ConsecutiveMatches int `json:"consecutive_matches"` + StartedAt time.Time `json:"started_at"` + LastObservedAt time.Time `json:"last_observed_at"` + AcknowledgedBy string `json:"acknowledged_by,omitempty"` + AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"` + SilencedBy string `json:"silenced_by,omitempty"` + SilencedUntil *time.Time `json:"silenced_until,omitempty"` + ResolvedAt *time.Time `json:"resolved_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +type IncidentEvent struct { + Sequence int `json:"sequence"` + Event string `json:"event"` + Actor string `json:"actor"` + CreatedAt time.Time `json:"created_at"` +} + +type AlertEvaluation struct { + RuleID string `json:"rule_id"` + OrganizationID string `json:"organization_id"` + Matched bool `json:"matched"` + Rows int `json:"rows"` + IncidentID string `json:"incident_id,omitempty"` + IncidentState string `json:"incident_state,omitempty"` + IncidentChanged bool `json:"incident_changed"` + Error string `json:"error,omitempty"` +} + +func (s *Store) SaveAlertRule(ctx context.Context, input AlertRuleInput, now time.Time) (AlertRule, error) { + if err := validateAlertRuleInput(input, now); err != nil { + return AlertRule{}, err + } + if _, err := s.SavedQuery(ctx, input.OrganizationID, input.SavedQueryID); err != nil { + return AlertRule{}, errors.New("alert rule saved query is unavailable") + } + var err error + if input.ID == "" { + input.ID, err = storageID("rule") + if err != nil { + return AlertRule{}, err + } + } + timestamp := now.UTC().Format(time.RFC3339Nano) + if input.ExpectedRevision == 0 { + _, err = s.control.ExecContext(ctx, `INSERT INTO alert_rules(organization_id,id,version,revision,name,description,saved_query_id,severity,minimum_matches,required_consecutive,evaluation_interval_seconds,enabled,next_evaluation_at,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?,?,?,?,?,?,?)`, input.OrganizationID, input.ID, input.Name, input.Description, input.SavedQueryID, input.Severity, input.MinimumMatches, input.RequiredConsecutive, int(input.EvaluationInterval/time.Second), boolInt(input.Enabled), timestamp, input.ActorUserID, input.ActorUserID, timestamp, timestamp) + } else { + if model.ValidateSourceID(input.ID) != nil || input.ExpectedRevision < 1 { + return AlertRule{}, errors.New("alert rule revision input is invalid") + } + var result sql.Result + result, err = s.control.ExecContext(ctx, `UPDATE alert_rules SET revision=revision+1,name=?,description=?,saved_query_id=?,severity=?,minimum_matches=?,required_consecutive=?,evaluation_interval_seconds=?,enabled=?,next_evaluation_at=?,updated_by=?,updated_at=? WHERE organization_id=? AND id=? AND revision=?`, input.Name, input.Description, input.SavedQueryID, input.Severity, input.MinimumMatches, input.RequiredConsecutive, int(input.EvaluationInterval/time.Second), boolInt(input.Enabled), timestamp, input.ActorUserID, timestamp, input.OrganizationID, input.ID, input.ExpectedRevision) + if err == nil { + if changed, _ := result.RowsAffected(); changed != 1 { + return AlertRule{}, errors.New("alert rule revision conflict") + } + } + } + if err != nil { + return AlertRule{}, errors.New("save alert rule") + } + return s.AlertRule(ctx, input.OrganizationID, input.ID) +} + +func (s *Store) AlertRule(ctx context.Context, organizationID, id string) (AlertRule, error) { + if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(id) != nil { + return AlertRule{}, errors.New("alert rule identity is invalid") + } + row := s.control.QueryRowContext(ctx, `SELECT version,revision,name,description,saved_query_id,severity,minimum_matches,required_consecutive,evaluation_interval_seconds,enabled,last_evaluated_at,next_evaluation_at,last_result,last_error,created_by,updated_by,created_at,updated_at FROM alert_rules WHERE organization_id=? AND id=?`, organizationID, id) + return scanAlertRule(row, organizationID, id) +} + +func (s *Store) AlertRules(ctx context.Context, organizationID string) ([]AlertRule, error) { + if model.ValidateSourceID(organizationID) != nil { + return nil, errors.New("invalid organization identifier") + } + rows, err := s.control.QueryContext(ctx, `SELECT id FROM alert_rules WHERE organization_id=? ORDER BY name,id`, organizationID) + if err != nil { + return nil, errors.New("list alert rules") + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err = rows.Scan(&id); err != nil { + return nil, errors.New("list alert rules") + } + ids = append(ids, id) + } + if err = rows.Err(); err != nil { + return nil, errors.New("list alert rules") + } + result := make([]AlertRule, 0, len(ids)) + for _, id := range ids { + value, loadErr := s.AlertRule(ctx, organizationID, id) + if loadErr != nil { + return nil, loadErr + } + result = append(result, value) + } + return result, nil +} + +func (s *Store) EvaluateDueAlertRules(ctx context.Context, budget query.Budget, now time.Time) ([]AlertEvaluation, error) { + if now.IsZero() { + return nil, errors.New("alert evaluation time is required") + } + rows, err := s.control.QueryContext(ctx, `SELECT organization_id,id FROM alert_rules WHERE enabled=1 AND next_evaluation_at<=? ORDER BY next_evaluation_at,organization_id,id LIMIT ?`, now.UTC().Format(time.RFC3339Nano), MaxDueRules) + if err != nil { + return nil, errors.New("list due alert rules") + } + type identity struct{ organizationID, ruleID string } + var identities []identity + for rows.Next() { + var item identity + if err = rows.Scan(&item.organizationID, &item.ruleID); err != nil { + _ = rows.Close() + return nil, errors.New("list due alert rules") + } + identities = append(identities, item) + } + if err = rows.Close(); err != nil || rows.Err() != nil { + return nil, errors.New("list due alert rules") + } + result := make([]AlertEvaluation, 0, len(identities)) + for _, item := range identities { + evaluation, evaluated, evaluationErr := s.evaluateAlertRule(ctx, item.organizationID, item.ruleID, budget, now.UTC()) + if evaluationErr != nil { + return result, evaluationErr + } + if evaluated { + result = append(result, evaluation) + } + } + return result, nil +} + +func (s *Store) evaluateAlertRule(ctx context.Context, organizationID, ruleID string, budget query.Budget, now time.Time) (AlertEvaluation, bool, error) { + rule, err := s.AlertRule(ctx, organizationID, ruleID) + if err != nil { + return AlertEvaluation{}, false, err + } + claim, err := s.control.ExecContext(ctx, `UPDATE alert_rules SET next_evaluation_at=? WHERE organization_id=? AND id=? AND enabled=1 AND next_evaluation_at<=?`, now.Add(rule.EvaluationInterval).Format(time.RFC3339Nano), organizationID, ruleID, now.Format(time.RFC3339Nano)) + if err != nil { + return AlertEvaluation{}, false, errors.New("claim alert rule evaluation") + } + if changed, _ := claim.RowsAffected(); changed != 1 { + return AlertEvaluation{}, false, nil + } + saved, err := s.SavedQuery(ctx, organizationID, rule.SavedQueryID) + if err != nil { + return AlertEvaluation{RuleID: rule.ID, OrganizationID: organizationID, Error: "query_unavailable"}, true, s.recordRuleError(ctx, rule, now) + } + result, err := s.Query(ctx, saved.AST, query.Scope{OrganizationID: organizationID, ProjectID: saved.Scope.ProjectID, EnvironmentID: saved.Scope.EnvironmentID, ServiceID: saved.Scope.ServiceID}, budget, now) + if err != nil { + return AlertEvaluation{RuleID: rule.ID, OrganizationID: organizationID, Error: "query_unavailable"}, true, s.recordRuleError(ctx, rule, now) + } + matched := len(result.Rows) >= rule.MinimumMatches + incident, changed, err := s.recordAlertResult(ctx, rule, len(result.Rows), matched, now) + if err != nil { + return AlertEvaluation{}, true, err + } + evaluation := AlertEvaluation{RuleID: rule.ID, OrganizationID: organizationID, Matched: matched, Rows: len(result.Rows)} + if incident.ID != "" { + evaluation.IncidentID, evaluation.IncidentState = incident.ID, incident.State + } + evaluation.IncidentChanged = changed + return evaluation, true, nil +} + +func (s *Store) recordRuleError(ctx context.Context, rule AlertRule, now time.Time) error { + _, err := s.control.ExecContext(ctx, `UPDATE alert_rules SET last_evaluated_at=?,last_result=NULL,last_error='query_unavailable' WHERE organization_id=? AND id=?`, now.Format(time.RFC3339Nano), rule.OrganizationID, rule.ID) + if err != nil { + return errors.New("record alert rule failure") + } + return nil +} + +func (s *Store) recordAlertResult(ctx context.Context, rule AlertRule, rows int, matched bool, now time.Time) (Incident, bool, error) { + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return Incident{}, false, errors.New("begin alert evaluation") + } + defer tx.Rollback() + if _, err = tx.ExecContext(ctx, `UPDATE alert_rules SET last_evaluated_at=?,last_result=?,last_error='' WHERE organization_id=? AND id=?`, now.Format(time.RFC3339Nano), rows, rule.OrganizationID, rule.ID); err != nil { + return Incident{}, false, errors.New("record alert evaluation") + } + incident, found, err := openIncidentTx(ctx, tx, rule.OrganizationID, rule.ID) + if err != nil { + return Incident{}, false, err + } + changed := false + if !matched { + if !found { + if err = tx.Commit(); err != nil { + return Incident{}, false, errors.New("commit alert evaluation") + } + return Incident{}, false, nil + } + if err = resolveIncidentTx(ctx, tx, &incident, "system", now); err != nil { + return Incident{}, false, err + } + changed = true + } else if !found { + incidentID, idErr := storageID("incident") + if idErr != nil { + return Incident{}, false, idErr + } + state := "pending" + if rule.RequiredConsecutive == 1 { + state = "firing" + } + stamp := now.Format(time.RFC3339Nano) + _, err = tx.ExecContext(ctx, `INSERT INTO incidents(organization_id,id,version,rule_id,state,severity,title,consecutive_matches,started_at,last_observed_at,updated_at) VALUES(?,?,1,?,?,?,?,1,?,?,?)`, rule.OrganizationID, incidentID, rule.ID, state, rule.Severity, rule.Name, stamp, stamp, stamp) + if err != nil { + return Incident{}, false, errors.New("open incident") + } + incident = Incident{Version: IncidentVersion, OrganizationID: rule.OrganizationID, ID: incidentID, RuleID: rule.ID, State: state, Severity: rule.Severity, Title: rule.Name, ConsecutiveMatches: 1, StartedAt: now, LastObservedAt: now, UpdatedAt: now} + if err = appendIncidentEventTx(ctx, tx, incident, "opened", "system", now); err != nil { + return Incident{}, false, err + } + changed = true + } else { + incident.ConsecutiveMatches++ + incident.LastObservedAt, incident.UpdatedAt = now, now + event := "" + if incident.State == "pending" && incident.ConsecutiveMatches >= rule.RequiredConsecutive { + incident.State, event = "firing", "promoted" + } else if incident.State == "silenced" && incident.SilencedUntil != nil && !now.Before(*incident.SilencedUntil) { + incident.State, incident.SilencedBy, incident.SilencedUntil, event = "firing", "", nil, "unsilenced" + } + _, err = tx.ExecContext(ctx, `UPDATE incidents SET state=?,consecutive_matches=?,last_observed_at=?,silenced_by=?,silenced_until=?,updated_at=? WHERE organization_id=? AND id=?`, incident.State, incident.ConsecutiveMatches, now.Format(time.RFC3339Nano), nullableText(incident.SilencedBy), nullableTime(incident.SilencedUntil), now.Format(time.RFC3339Nano), incident.OrganizationID, incident.ID) + if err != nil { + return Incident{}, false, errors.New("update incident observation") + } + if event != "" { + if err = appendIncidentEventTx(ctx, tx, incident, event, "system", now); err != nil { + return Incident{}, false, err + } + changed = true + } + } + if err = tx.Commit(); err != nil { + return Incident{}, false, errors.New("commit alert evaluation") + } + return incident, changed, nil +} + +func (s *Store) Incidents(ctx context.Context, organizationID string, includeResolved bool, limit int) ([]Incident, error) { + if model.ValidateSourceID(organizationID) != nil || limit < 1 || limit > 1000 { + return nil, errors.New("incident list input is invalid") + } + statement := `SELECT id,version,rule_id,state,severity,title,consecutive_matches,started_at,last_observed_at,acknowledged_by,acknowledged_at,silenced_by,silenced_until,resolved_at,updated_at FROM incidents WHERE organization_id=?` + if !includeResolved { + statement += ` AND state!='resolved'` + } + statement += ` ORDER BY CASE state WHEN 'firing' THEN 0 WHEN 'pending' THEN 1 WHEN 'acknowledged' THEN 2 WHEN 'silenced' THEN 3 ELSE 4 END,updated_at DESC,id LIMIT ?` + rows, err := s.control.QueryContext(ctx, statement, organizationID, limit) + if err != nil { + return nil, errors.New("list incidents") + } + defer rows.Close() + var result []Incident + for rows.Next() { + value, scanErr := scanIncident(rows, organizationID) + if scanErr != nil { + return nil, scanErr + } + result = append(result, value) + } + if err = rows.Err(); err != nil { + return nil, errors.New("list incidents") + } + return result, nil +} + +func (s *Store) TransitionIncident(ctx context.Context, organizationID, incidentID, action, actor string, silenceUntil *time.Time, now time.Time) (Incident, error) { + if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(incidentID) != nil || model.ValidateSourceID(actor) != nil || now.IsZero() { + return Incident{}, errors.New("incident transition input is invalid") + } + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return Incident{}, errors.New("begin incident transition") + } + defer tx.Rollback() + incident, err := incidentByIDTx(ctx, tx, organizationID, incidentID) + if err != nil { + return Incident{}, err + } + if incident.State == "resolved" { + return Incident{}, errors.New("resolved incident cannot transition") + } + switch action { + case "acknowledge": + incident.State, incident.AcknowledgedBy = "acknowledged", actor + incident.SilencedBy, incident.SilencedUntil = "", nil + stamp := now.UTC() + incident.AcknowledgedAt = &stamp + if err = appendIncidentEventTx(ctx, tx, incident, "acknowledged", actor, stamp); err != nil { + return Incident{}, err + } + case "silence": + if silenceUntil == nil || !silenceUntil.After(now) || silenceUntil.After(now.Add(30*24*time.Hour)) { + return Incident{}, errors.New("incident silence expiry is invalid") + } + stamp := silenceUntil.UTC() + incident.State, incident.SilencedBy, incident.SilencedUntil = "silenced", actor, &stamp + if err = appendIncidentEventTx(ctx, tx, incident, "silenced", actor, now.UTC()); err != nil { + return Incident{}, err + } + case "resolve": + if err = resolveIncidentTx(ctx, tx, &incident, actor, now.UTC()); err != nil { + return Incident{}, err + } + default: + return Incident{}, errors.New("incident action is invalid") + } + incident.UpdatedAt = now.UTC() + if action != "resolve" { + _, err = tx.ExecContext(ctx, `UPDATE incidents SET state=?,acknowledged_by=?,acknowledged_at=?,silenced_by=?,silenced_until=?,updated_at=? WHERE organization_id=? AND id=?`, incident.State, nullableText(incident.AcknowledgedBy), nullableTime(incident.AcknowledgedAt), nullableText(incident.SilencedBy), nullableTime(incident.SilencedUntil), incident.UpdatedAt.Format(time.RFC3339Nano), organizationID, incidentID) + if err != nil { + return Incident{}, errors.New("update incident") + } + } + if err = tx.Commit(); err != nil { + return Incident{}, errors.New("commit incident transition") + } + return incident, nil +} + +func (s *Store) IncidentEvents(ctx context.Context, organizationID, incidentID string) ([]IncidentEvent, error) { + if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(incidentID) != nil { + return nil, errors.New("incident event identity is invalid") + } + rows, err := s.control.QueryContext(ctx, `SELECT sequence,event,actor,created_at FROM incident_events WHERE organization_id=? AND incident_id=? ORDER BY sequence`, organizationID, incidentID) + if err != nil { + return nil, errors.New("list incident events") + } + defer rows.Close() + var result []IncidentEvent + for rows.Next() { + var value IncidentEvent + var created string + if err = rows.Scan(&value.Sequence, &value.Event, &value.Actor, &created); err != nil { + return nil, errors.New("read incident event") + } + value.CreatedAt, err = time.Parse(time.RFC3339Nano, created) + if err != nil { + return nil, errors.New("stored incident event is invalid") + } + result = append(result, value) + } + return result, rows.Err() +} + +type alertRuleScanner interface{ Scan(...any) error } + +func scanAlertRule(row alertRuleScanner, organizationID, id string) (AlertRule, error) { + value := AlertRule{OrganizationID: organizationID, ID: id} + var enabled int + var interval int64 + var lastEvaluated, nextEvaluation, created, updated sql.NullString + var lastResult sql.NullInt64 + if err := row.Scan(&value.Version, &value.Revision, &value.Name, &value.Description, &value.SavedQueryID, &value.Severity, &value.MinimumMatches, &value.RequiredConsecutive, &interval, &enabled, &lastEvaluated, &nextEvaluation, &lastResult, &value.LastError, &value.CreatedBy, &value.UpdatedBy, &created, &updated); errors.Is(err, sql.ErrNoRows) { + return AlertRule{}, errors.New("alert rule not found") + } else if err != nil { + return AlertRule{}, errors.New("read alert rule") + } + value.Enabled = enabled == 1 + value.EvaluationInterval = time.Duration(interval) * time.Second + if lastResult.Valid { + result := int(lastResult.Int64) + value.LastResult = &result + } + var err error + if lastEvaluated.Valid { + parsed, parseErr := time.Parse(time.RFC3339Nano, lastEvaluated.String) + if parseErr != nil { + return AlertRule{}, errors.New("stored alert rule evaluation time is invalid") + } + value.LastEvaluatedAt = &parsed + } + value.NextEvaluationAt, err = time.Parse(time.RFC3339Nano, nextEvaluation.String) + if err == nil { + value.CreatedAt, err = time.Parse(time.RFC3339Nano, created.String) + } + if err == nil { + value.UpdatedAt, err = time.Parse(time.RFC3339Nano, updated.String) + } + if err != nil || validateAlertRule(value) != nil { + return AlertRule{}, errors.New("stored alert rule is invalid") + } + return value, nil +} + +type incidentScanner interface{ Scan(...any) error } + +func scanIncident(row incidentScanner, organizationID string) (Incident, error) { + value := Incident{OrganizationID: organizationID} + var started, observed, updated string + var acknowledgedBy, acknowledgedAt, silencedBy, silencedUntil, resolvedAt sql.NullString + if err := row.Scan(&value.ID, &value.Version, &value.RuleID, &value.State, &value.Severity, &value.Title, &value.ConsecutiveMatches, &started, &observed, &acknowledgedBy, &acknowledgedAt, &silencedBy, &silencedUntil, &resolvedAt, &updated); errors.Is(err, sql.ErrNoRows) { + return Incident{}, sql.ErrNoRows + } else if err != nil { + return Incident{}, errors.New("read incident") + } + value.AcknowledgedBy, value.SilencedBy = acknowledgedBy.String, silencedBy.String + var err error + value.StartedAt, err = time.Parse(time.RFC3339Nano, started) + if err == nil { + value.LastObservedAt, err = time.Parse(time.RFC3339Nano, observed) + } + if err == nil { + value.UpdatedAt, err = time.Parse(time.RFC3339Nano, updated) + } + if err == nil { + value.AcknowledgedAt, err = parseOptionalTime(acknowledgedAt) + } + if err == nil { + value.SilencedUntil, err = parseOptionalTime(silencedUntil) + } + if err == nil { + value.ResolvedAt, err = parseOptionalTime(resolvedAt) + } + if err != nil || validateIncident(value) != nil { + return Incident{}, errors.New("stored incident is invalid") + } + return value, nil +} + +func openIncidentTx(ctx context.Context, tx *sql.Tx, organizationID, ruleID string) (Incident, bool, error) { + row := tx.QueryRowContext(ctx, `SELECT id,version,rule_id,state,severity,title,consecutive_matches,started_at,last_observed_at,acknowledged_by,acknowledged_at,silenced_by,silenced_until,resolved_at,updated_at FROM incidents WHERE organization_id=? AND rule_id=? AND state!='resolved'`, organizationID, ruleID) + value, err := scanIncident(row, organizationID) + if errors.Is(err, sql.ErrNoRows) { + return Incident{}, false, nil + } + if err != nil { + return Incident{}, false, err + } + return value, true, nil +} + +func incidentByIDTx(ctx context.Context, tx *sql.Tx, organizationID, incidentID string) (Incident, error) { + row := tx.QueryRowContext(ctx, `SELECT id,version,rule_id,state,severity,title,consecutive_matches,started_at,last_observed_at,acknowledged_by,acknowledged_at,silenced_by,silenced_until,resolved_at,updated_at FROM incidents WHERE organization_id=? AND id=?`, organizationID, incidentID) + value, err := scanIncident(row, organizationID) + if err != nil { + return Incident{}, errors.New("incident not found") + } + return value, nil +} + +func resolveIncidentTx(ctx context.Context, tx *sql.Tx, incident *Incident, actor string, now time.Time) error { + incident.State, incident.ResolvedAt, incident.UpdatedAt = "resolved", &now, now + _, err := tx.ExecContext(ctx, `UPDATE incidents SET state='resolved',resolved_at=?,updated_at=? WHERE organization_id=? AND id=?`, now.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano), incident.OrganizationID, incident.ID) + if err != nil { + return errors.New("resolve incident") + } + return appendIncidentEventTx(ctx, tx, *incident, "resolved", actor, now) +} + +func appendIncidentEventTx(ctx context.Context, tx *sql.Tx, incident Incident, event, actor string, now time.Time) error { + var sequence int + if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(sequence),0)+1 FROM incident_events WHERE organization_id=? AND incident_id=?`, incident.OrganizationID, incident.ID).Scan(&sequence); err != nil { + return errors.New("sequence incident event") + } + if _, err := tx.ExecContext(ctx, `INSERT INTO incident_events(organization_id,incident_id,sequence,event,actor,created_at) VALUES(?,?,?,?,?,?)`, incident.OrganizationID, incident.ID, sequence, event, actor, now.UTC().Format(time.RFC3339Nano)); err != nil { + return errors.New("append incident event") + } + return nil +} + +func validateAlertRuleInput(input AlertRuleInput, now time.Time) error { + if model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.ActorUserID) != nil || model.ValidateSourceID(input.SavedQueryID) != nil || !boundedText(input.Name, 128, false) || !boundedText(input.Description, 1024, true) || input.MinimumMatches < 1 || input.MinimumMatches > 100_000 || input.RequiredConsecutive < 1 || input.RequiredConsecutive > 10 || input.EvaluationInterval < 15*time.Second || input.EvaluationInterval > 24*time.Hour || input.EvaluationInterval%time.Second != 0 || now.IsZero() { + return errors.New("alert rule input is invalid") + } + if !validSeverity(input.Severity) || input.ExpectedRevision == 0 && input.ID != "" { + return errors.New("alert rule input is invalid") + } + return nil +} + +func validateAlertRule(value AlertRule) error { + if value.Version != AlertRuleVersion || value.Revision < 1 || model.ValidateSourceID(value.OrganizationID) != nil || model.ValidateSourceID(value.ID) != nil || model.ValidateSourceID(value.SavedQueryID) != nil || model.ValidateSourceID(value.CreatedBy) != nil || model.ValidateSourceID(value.UpdatedBy) != nil || !boundedText(value.Name, 128, false) || !boundedText(value.Description, 1024, true) || !validSeverity(value.Severity) || value.MinimumMatches < 1 || value.MinimumMatches > 100_000 || value.RequiredConsecutive < 1 || value.RequiredConsecutive > 10 || value.EvaluationInterval < 15*time.Second || value.EvaluationInterval > 24*time.Hour || value.NextEvaluationAt.IsZero() || value.CreatedAt.IsZero() || value.UpdatedAt.Before(value.CreatedAt) || value.LastError != "" && value.LastError != "query_unavailable" { + return errors.New("alert rule is invalid") + } + return nil +} + +func validateIncident(value Incident) error { + states := map[string]bool{"pending": true, "firing": true, "acknowledged": true, "silenced": true, "resolved": true} + if value.Version != IncidentVersion || model.ValidateSourceID(value.OrganizationID) != nil || model.ValidateSourceID(value.ID) != nil || model.ValidateSourceID(value.RuleID) != nil || !states[value.State] || !validSeverity(value.Severity) || !boundedText(value.Title, 128, false) || value.ConsecutiveMatches < 0 || value.StartedAt.IsZero() || value.LastObservedAt.Before(value.StartedAt) || value.UpdatedAt.Before(value.StartedAt) { + return errors.New("incident is invalid") + } + if value.State == "acknowledged" && (value.AcknowledgedBy == "" || value.AcknowledgedAt == nil) || value.State == "silenced" && (value.SilencedBy == "" || value.SilencedUntil == nil) || value.State == "resolved" && value.ResolvedAt == nil { + return errors.New("incident state metadata is invalid") + } + if value.AcknowledgedBy != "" && (model.ValidateSourceID(value.AcknowledgedBy) != nil || value.AcknowledgedAt == nil) || value.AcknowledgedAt != nil && (value.AcknowledgedBy == "" || value.AcknowledgedAt.Before(value.StartedAt)) || value.SilencedBy != "" && (model.ValidateSourceID(value.SilencedBy) != nil || value.SilencedUntil == nil) || value.SilencedUntil != nil && (value.SilencedBy == "" || value.SilencedUntil.Before(value.StartedAt)) || value.ResolvedAt != nil && value.ResolvedAt.Before(value.StartedAt) { + return errors.New("incident actor metadata is invalid") + } + return nil +} + +func parseOptionalTime(value sql.NullString) (*time.Time, error) { + if !value.Valid { + return nil, nil + } + parsed, err := time.Parse(time.RFC3339Nano, value.String) + if err != nil { + return nil, err + } + return &parsed, nil +} + +func validSeverity(value string) bool { + return value == "information" || value == "warning" || value == "critical" +} +func boolInt(value bool) int { + if value { + return 1 + } + return 0 +} +func nullableText(value string) any { + if value == "" { + return nil + } + return value +} +func nullableTime(value *time.Time) any { + if value == nil { + return nil + } + return value.UTC().Format(time.RFC3339Nano) +} diff --git a/internal/storage/incidents_test.go b/internal/storage/incidents_test.go new file mode 100644 index 0000000..a6a68e0 --- /dev/null +++ b/internal/storage/incidents_test.go @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "sync" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +func TestAlertRuleEvaluationAndIncidentLifecycle(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + organizationID := "organization-a" + actor := "operator-a" + now := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC) + saved, err := store.SaveQuery(ctx, SavedQueryInput{ + OrganizationID: organizationID, ActorUserID: actor, MaxRows: 100, + Name: "Recent failures", Description: "Recent HTTP failures for an alert.", + Query: "logs | where status >= 500 | window 1h | limit 50", + }, now) + if err != nil { + t.Fatal(err) + } + rule, err := store.SaveAlertRule(ctx, AlertRuleInput{ + OrganizationID: organizationID, ActorUserID: actor, SavedQueryID: saved.ID, + Name: "HTTP failures", Description: "Open after two consecutive matching evaluations.", + Severity: "critical", MinimumMatches: 1, RequiredConsecutive: 2, + EvaluationInterval: 15 * time.Second, Enabled: true, + }, now) + if err != nil { + t.Fatal(err) + } + budget := query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 16 << 20, MaxMemoryBytes: 8 << 20} + + empty, err := store.EvaluateDueAlertRules(ctx, budget, now) + if err != nil || len(empty) != 1 || empty[0].Matched || empty[0].Rows != 0 || empty[0].IncidentID != "" { + t.Fatalf("empty evaluation=%+v err=%v", empty, err) + } + if incidents, listErr := store.Incidents(ctx, organizationID, false, 100); listErr != nil || len(incidents) != 0 { + t.Fatalf("empty incidents=%+v err=%v", incidents, listErr) + } + + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: organizationID, ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + observed := now.Add(10 * time.Second) + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: observed, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: observed, Name: "http.request", Attributes: map[string]string{"http.status_code": "503", "http.route": "/failed"}}}} + if _, err = store.Ingest(ctx, token, batch, observed); err != nil { + t.Fatal(err) + } + projectAll(t, store) + + first, err := store.EvaluateDueAlertRules(ctx, budget, now.Add(15*time.Second)) + if err != nil || len(first) != 1 || !first[0].Matched || first[0].IncidentState != "pending" { + t.Fatalf("first evaluation=%+v err=%v", first, err) + } + second, err := store.EvaluateDueAlertRules(ctx, budget, now.Add(30*time.Second)) + if err != nil || len(second) != 1 || second[0].IncidentID != first[0].IncidentID || second[0].IncidentState != "firing" { + t.Fatalf("second evaluation=%+v err=%v", second, err) + } + incidentID := second[0].IncidentID + third, err := store.EvaluateDueAlertRules(ctx, budget, now.Add(45*time.Second)) + if err != nil || len(third) != 1 || third[0].IncidentChanged || third[0].IncidentState != "firing" { + t.Fatalf("steady evaluation=%+v err=%v", third, err) + } + steadyEvents, err := store.IncidentEvents(ctx, organizationID, incidentID) + if err != nil || len(steadyEvents) != 2 { + t.Fatalf("steady events=%+v err=%v", steadyEvents, err) + } + + acknowledged, err := store.TransitionIncident(ctx, organizationID, incidentID, "acknowledge", actor, nil, now.Add(46*time.Second)) + if err != nil || acknowledged.State != "acknowledged" || acknowledged.AcknowledgedBy != actor { + t.Fatalf("acknowledged=%+v err=%v", acknowledged, err) + } + silenceUntil := now.Add(time.Hour) + silenced, err := store.TransitionIncident(ctx, organizationID, incidentID, "silence", actor, &silenceUntil, now.Add(47*time.Second)) + if err != nil || silenced.State != "silenced" || silenced.SilencedUntil == nil || !silenced.SilencedUntil.Equal(silenceUntil) { + t.Fatalf("silenced=%+v err=%v", silenced, err) + } + resolved, err := store.TransitionIncident(ctx, organizationID, incidentID, "resolve", actor, nil, now.Add(48*time.Second)) + if err != nil || resolved.State != "resolved" || resolved.ResolvedAt == nil { + t.Fatalf("resolved=%+v err=%v", resolved, err) + } + if _, err = store.TransitionIncident(ctx, organizationID, incidentID, "acknowledge", actor, nil, now.Add(49*time.Second)); err == nil { + t.Fatal("resolved incident accepted another transition") + } + + events, err := store.IncidentEvents(ctx, organizationID, incidentID) + if err != nil { + t.Fatal(err) + } + want := []string{"opened", "promoted", "acknowledged", "silenced", "resolved"} + if len(events) != len(want) { + t.Fatalf("events=%+v", events) + } + for index, event := range events { + if event.Sequence != index+1 || event.Event != want[index] { + t.Fatalf("events=%+v", events) + } + } + all, err := store.Incidents(ctx, organizationID, true, 100) + if err != nil || len(all) != 1 || all[0].State != "resolved" { + t.Fatalf("all incidents=%+v err=%v", all, err) + } + loadedRule, err := store.AlertRule(ctx, organizationID, rule.ID) + if err != nil || loadedRule.LastEvaluatedAt == nil || loadedRule.LastResult == nil || *loadedRule.LastResult != 1 || loadedRule.LastError != "" { + t.Fatalf("rule=%+v err=%v", loadedRule, err) + } +} + +func TestAlertQueryFailureDoesNotOpenOrResolveAnIncident(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 14, 0, 0, 0, time.UTC) + saved, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", ActorUserID: "operator-a", MaxRows: 100, Name: "Unknown field", Description: "Requires reviewed sensitive-field access.", Query: "logs | where private.value == secret | limit 10"}, now) + if err != nil { + t.Fatal(err) + } + rule, err := store.SaveAlertRule(ctx, AlertRuleInput{OrganizationID: "organization-a", ActorUserID: "operator-a", SavedQueryID: saved.ID, Name: "Fail closed", Description: "A query failure is not a healthy result.", Severity: "critical", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true}, now) + if err != nil { + t.Fatal(err) + } + budget := query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 16 << 20, MaxMemoryBytes: 8 << 20} + evaluations, err := store.EvaluateDueAlertRules(ctx, budget, now) + if err != nil || len(evaluations) != 1 || evaluations[0].Error != "query_unavailable" || evaluations[0].IncidentID != "" || evaluations[0].IncidentChanged { + t.Fatalf("evaluations=%+v err=%v", evaluations, err) + } + loaded, err := store.AlertRule(ctx, "organization-a", rule.ID) + if err != nil || loaded.LastError != "query_unavailable" || loaded.LastResult != nil { + t.Fatalf("rule=%+v err=%v", loaded, err) + } + incidents, err := store.Incidents(ctx, "organization-a", true, 10) + if err != nil || len(incidents) != 0 { + t.Fatalf("incidents=%+v err=%v", incidents, err) + } +} + +func TestDueAlertRuleClaimPreventsConcurrentDuplicateEvaluation(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 13, 0, 0, 0, time.UTC) + saved, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", ActorUserID: "operator-a", MaxRows: 100, Name: "Any logs", Description: "Any recent log.", Query: "logs | window 1h | limit 10"}, now) + if err != nil { + t.Fatal(err) + } + if _, err = store.SaveAlertRule(ctx, AlertRuleInput{OrganizationID: "organization-a", ActorUserID: "operator-a", SavedQueryID: saved.ID, Name: "Any log", Description: "Concurrency claim test.", Severity: "warning", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true}, now); err != nil { + t.Fatal(err) + } + budget := query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 16 << 20, MaxMemoryBytes: 8 << 20} + var group sync.WaitGroup + results := make(chan []AlertEvaluation, 2) + errors := make(chan error, 2) + for range 2 { + group.Add(1) + go func() { + defer group.Done() + value, evaluationErr := store.EvaluateDueAlertRules(ctx, budget, now) + results <- value + errors <- evaluationErr + }() + } + group.Wait() + close(results) + close(errors) + total := 0 + for err = range errors { + if err != nil { + t.Fatal(err) + } + } + for result := range results { + total += len(result) + } + if total != 1 { + t.Fatalf("evaluations=%d, want 1", total) + } +} + +func TestAlertRulesRejectCrossOrganizationAndUnboundedInputs(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Now().UTC() + saved, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", ActorUserID: "operator-a", MaxRows: 100, Name: "Query", Description: "Scoped query.", Query: "logs | limit 10"}, now) + if err != nil { + t.Fatal(err) + } + base := AlertRuleInput{OrganizationID: "organization-b", ActorUserID: "operator-b", SavedQueryID: saved.ID, Name: "Cross scope", Description: "Must fail.", Severity: "warning", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true} + if _, err = store.SaveAlertRule(ctx, base, now); err == nil { + t.Fatal("cross-organization saved query was accepted") + } + base.OrganizationID = "organization-a" + base.EvaluationInterval = 14 * time.Second + if _, err = store.SaveAlertRule(ctx, base, now); err == nil { + t.Fatal("too-frequent evaluation was accepted") + } + base.EvaluationInterval = 15 * time.Second + base.RequiredConsecutive = 11 + if _, err = store.SaveAlertRule(ctx, base, now); err == nil { + t.Fatal("unbounded confirmation count was accepted") + } +} diff --git a/internal/storage/log_rollup.go b/internal/storage/log_rollup.go new file mode 100644 index 0000000..c61d840 --- /dev/null +++ b/internal/storage/log_rollup.go @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math" + "strconv" + "time" + + "gamertan.com/observatory/internal/model" +) + +const ( + logRollupVersion = 1 + logRollupWindow = 5 * time.Minute + maxLogRollupGroupsBatch = model.MaxRecords +) + +const logObservationBytesSQL = `64+LENGTH(project_id)+LENGTH(environment_id)+LENGTH(service_id)+LENGTH(source_id)+LENGTH(stream_id)+LENGTH(signal)+LENGTH(timestamp)+LENGTH(name)+COALESCE(LENGTH(severity),0)+COALESCE(LENGTH(body),0)+COALESCE(LENGTH(trace_id),0)+COALESCE(LENGTH(span_id),0)+COALESCE(LENGTH(correlation_id),0)+LENGTH(attributes_json)` + +type logRollup struct { + projectID, environmentID, serviceID, route string + bucket int64 + status, routePresent int + count, scannedBytes int64 +} + +func ensureLogRollups(ctx context.Context, db *sql.DB) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin log rollup migration: %w", err) + } + defer tx.Rollback() + for _, statement := range []string{ + `CREATE TABLE IF NOT EXISTS log_rollup_state (id INTEGER PRIMARY KEY CHECK(id=1),version INTEGER NOT NULL CHECK(version BETWEEN 0 AND 1))`, + `INSERT OR IGNORE INTO log_rollup_state(id,version) VALUES(1,0)`, + `CREATE TABLE IF NOT EXISTS log_status_route_rollups_5m ( + organization_id TEXT NOT NULL, + project_id TEXT NOT NULL, + environment_id TEXT NOT NULL, + service_id TEXT NOT NULL, + bucket_start INTEGER NOT NULL, + status INTEGER NOT NULL CHECK(status BETWEEN 100 AND 999), + route TEXT NOT NULL, + route_present INTEGER NOT NULL CHECK(route_present IN (0,1)), + observation_count INTEGER NOT NULL CHECK(typeof(observation_count)='integer' AND observation_count > 0), + scanned_bytes INTEGER NOT NULL CHECK(typeof(scanned_bytes)='integer' AND scanned_bytes >= 0), + PRIMARY KEY(organization_id,project_id,environment_id,service_id,bucket_start,status,route,route_present) + )`, + `CREATE TABLE IF NOT EXISTS log_rollup_segments (segment_digest TEXT PRIMARY KEY)`, + `CREATE INDEX IF NOT EXISTS log_rollups_status_time ON log_status_route_rollups_5m(organization_id,status,bucket_start)`, + `CREATE INDEX IF NOT EXISTS log_rollups_scope_time ON log_status_route_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start,status)`, + } { + if _, err = tx.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("migrate log rollups: %w", err) + } + } + var version int + if err = tx.QueryRowContext(ctx, `SELECT version FROM log_rollup_state WHERE id=1`).Scan(&version); err != nil { + return errors.New("read log rollup migration state") + } + if version == 0 { + if _, err = tx.ExecContext(ctx, `DELETE FROM log_status_route_rollups_5m`); err != nil { + return errors.New("clear incomplete log rollup migration") + } + if _, err = tx.ExecContext(ctx, `DELETE FROM log_rollup_segments`); err != nil { + return errors.New("clear incomplete log rollup ledger") + } + status := `json_extract(attributes_json,'$."http.status_code"')` + route := `json_extract(attributes_json,'$."http.route"')` + statement := `INSERT INTO log_status_route_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start,status,route,route_present,observation_count,scanned_bytes) + SELECT organization_id,project_id,environment_id,service_id,CAST(unixepoch(timestamp)/300 AS INTEGER)*300,CAST(` + status + ` AS INTEGER),COALESCE(CAST(` + route + ` AS TEXT),''),CASE WHEN json_type(attributes_json,'$."http.route"') IS NULL THEN 0 ELSE 1 END,COUNT(*),SUM(` + logObservationBytesSQL + `) + FROM observations WHERE signal=? AND printf('%d',CAST(` + status + ` AS INTEGER))=` + status + ` AND CAST(` + status + ` AS INTEGER) BETWEEN 100 AND 999 + GROUP BY organization_id,project_id,environment_id,service_id,CAST(unixepoch(timestamp)/300 AS INTEGER)*300,CAST(` + status + ` AS INTEGER),COALESCE(CAST(` + route + ` AS TEXT),''),CASE WHEN json_type(attributes_json,'$."http.route"') IS NULL THEN 0 ELSE 1 END` + if _, err = tx.ExecContext(ctx, statement, model.SignalLogs); err != nil { + return errors.New("backfill log rollups") + } + if _, err = tx.ExecContext(ctx, `INSERT INTO log_rollup_segments(segment_digest) SELECT DISTINCT segment_digest FROM observations WHERE signal=?`, model.SignalLogs); err != nil { + return errors.New("record backfilled log rollup segments") + } + if _, err = tx.ExecContext(ctx, `UPDATE log_rollup_state SET version=? WHERE id=1 AND version=0`, logRollupVersion); err != nil { + return errors.New("activate log rollup migration") + } + } + if version != 0 && version != logRollupVersion { + return errors.New("unsupported log rollup version") + } + if err = tx.Commit(); err != nil { + return errors.New("commit log rollup migration") + } + return nil +} + +func projectLogRollups(ctx context.Context, tx *sql.Tx, scope model.Scope, batch model.Batch, segmentDigest string, observationBytes []int64) error { + if batch.Signal != model.SignalLogs { + return nil + } + if len(observationBytes) != len(batch.Records) { + return errors.New("log projection byte evidence is incomplete") + } + ledger, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO log_rollup_segments(segment_digest) VALUES(?)`, segmentDigest) + if err != nil { + return errors.New("record log rollup segment") + } + inserted, err := ledger.RowsAffected() + if err != nil { + return errors.New("inspect log rollup segment") + } + if inserted == 0 { + return nil + } + groups := map[string]*logRollup{} + for index, observation := range batch.Records { + statusText, ok := observation.Attributes["http.status_code"] + if !ok { + continue + } + status, statusErr := strconv.Atoi(statusText) + if statusErr != nil || strconv.Itoa(status) != statusText || status < 100 || status > 999 { + continue + } + route, routeOK := observation.Attributes["http.route"] + routePresent := 0 + if routeOK { + routePresent = 1 + } + bucket := observation.Timestamp.UTC().Truncate(logRollupWindow).Unix() + key := scope.ProjectID + "\x00" + scope.EnvironmentID + "\x00" + scope.ServiceID + "\x00" + strconv.FormatInt(bucket, 10) + "\x00" + statusText + "\x00" + strconv.Itoa(routePresent) + "\x00" + route + group := groups[key] + if group == nil { + group = &logRollup{projectID: scope.ProjectID, environmentID: scope.EnvironmentID, serviceID: scope.ServiceID, bucket: bucket, status: status, route: route, routePresent: routePresent} + groups[key] = group + if len(groups) > maxLogRollupGroupsBatch { + return fmt.Errorf("log batch exceeds %d rollup groups", maxLogRollupGroupsBatch) + } + } + bytes := observationBytes[index] + if bytes < 0 { + return errors.New("log projection byte evidence is invalid") + } + if group.count == math.MaxInt64 || bytes > math.MaxInt64-group.scannedBytes { + return errors.New("log rollup count exceeds integer range") + } + group.count++ + group.scannedBytes += bytes + } + for _, group := range groups { + _, err = tx.ExecContext(ctx, `INSERT INTO log_status_route_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start,status,route,route_present,observation_count,scanned_bytes) VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(organization_id,project_id,environment_id,service_id,bucket_start,status,route,route_present) DO UPDATE SET observation_count=log_status_route_rollups_5m.observation_count+excluded.observation_count,scanned_bytes=log_status_route_rollups_5m.scanned_bytes+excluded.scanned_bytes`, scope.OrganizationID, group.projectID, group.environmentID, group.serviceID, group.bucket, group.status, group.route, group.routePresent, group.count, group.scannedBytes) + if err != nil { + return errors.New("merge log rollup") + } + } + return nil +} + +func projectedObservationBytes(scope model.Scope, batch model.Batch, observation model.Observation, attributesBytes int) int64 { + return int64(64 + len(scope.ProjectID) + len(scope.EnvironmentID) + len(scope.ServiceID) + len(batch.SourceID) + len(batch.StreamID) + len(batch.Signal) + len(observation.Timestamp.UTC().Format(time.RFC3339Nano)) + len(observation.Name) + len(observation.Severity) + len(observation.Body) + len(observation.TraceID) + len(observation.SpanID) + len(observation.CorrelationID) + attributesBytes) +} diff --git a/internal/storage/log_summary_query.go b/internal/storage/log_summary_query.go new file mode 100644 index 0000000..55f484e --- /dev/null +++ b/internal/storage/log_summary_query.go @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "math" + "net/url" + "os" + "sort" + "strconv" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" +) + +// indexedLogCountSummaryEligible identifies the typed status/route count +// summary maintained as an exact five-minute projection. Larger bucket sizes +// can combine those rows; a partial lower window boundary scans at most one +// five-minute fragment from the primary projection. +func indexedLogCountSummaryEligible(ast query.AST) bool { + if ast.Signal != model.SignalLogs || ast.Summary == nil || len(ast.Summary.Aggregates) != 1 || len(ast.Summary.GroupBy) != 1 { + return false + } + aggregate := ast.Summary.Aggregates[0] + if aggregate.Function != "count" || aggregate.Field != "" || query.CanonicalField(ast.Summary.GroupBy[0]) != "http.route" { + return false + } + if ast.Bucket != 0 && (ast.Bucket < logRollupWindow || ast.Bucket%logRollupWindow != 0) { + return false + } + if len(ast.Filters) != 1 || query.CanonicalField(ast.Filters[0].Field) != "http.status_code" { + return false + } + threshold, err := strconv.Atoi(ast.Filters[0].Value) + return err == nil && threshold >= 100 && threshold <= 999 && ast.Filters[0].Op == ">=" +} + +// estimateLogRollupBytes accounts for the compact rows the optimized query +// reads, plus at most one raw five-minute fragment when the requested lower +// boundary does not align with a rollup bucket. It deliberately does not use +// a client-supplied estimate or treat logical observation bytes as physical +// scan cost. +func (s *Store) estimateLogRollupBytes(ctx context.Context, scope query.Scope, ast query.AST, now time.Time) (int64, error) { + path := s.organizationProjectionPath(scope.OrganizationID) + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return 0, nil + } + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return 0, errors.New("organization projection is unavailable") + } + dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) + if err != nil { + return 0, errors.New("open log rollup estimate") + } + defer db.Close() + db.SetMaxOpenConns(1) + status, err := typedFilterValue(ast.Filters[0].Value, schema.TypeInteger) + if err != nil { + return 0, err + } + cutoff, rollupStart := logRollupWindowStart(ast, now) + statement := `SELECT COALESCE(SUM(96+LENGTH(project_id)+LENGTH(environment_id)+LENGTH(service_id)+LENGTH(route)),0) FROM log_status_route_rollups_5m WHERE organization_id=? AND status>=?` + arguments := []any{scope.OrganizationID, status} + for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} { + if selected.value != "" { + statement += " AND " + selected.column + "=?" + arguments = append(arguments, selected.value) + } + } + if ast.Window > 0 { + statement += ` AND bucket_start>=?` + arguments = append(arguments, rollupStart) + } + var estimated int64 + if err = db.QueryRowContext(ctx, statement, arguments...).Scan(&estimated); err != nil || estimated < 0 { + return 0, errors.New("estimate log rollup scan") + } + if ast.Window == 0 || !cutoff.Before(time.Unix(rollupStart, 0).UTC()) { + return estimated, nil + } + raw := `SELECT COALESCE(SUM(` + logObservationBytesSQL + `),0) FROM observations o INDEXED BY observations_http_status WHERE o.organization_id=? AND o.signal=?` + rawArguments := []any{scope.OrganizationID, string(ast.Signal)} + for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} { + if selected.value != "" { + raw += " AND o." + selected.column + "=?" + rawArguments = append(rawArguments, selected.value) + } + } + raw += ` AND o.timestamp>=? AND o.timestamp=?` + rawArguments = append(rawArguments, status) + var partial int64 + if err = db.QueryRowContext(ctx, raw, rawArguments...).Scan(&partial); err != nil || partial < 0 || partial > math.MaxInt64-estimated { + return 0, errors.New("estimate partial log rollup scan") + } + return estimated + partial, nil +} + +func logRollupWindowStart(ast query.AST, now time.Time) (time.Time, int64) { + if ast.Window == 0 { + return time.Time{}, math.MinInt64 + } + cutoff := now.UTC().Add(-ast.Window) + start := cutoff.Truncate(logRollupWindow) + if !start.Equal(cutoff) { + start = start.Add(logRollupWindow) + } + return cutoff, start.Unix() +} + +type logSummaryKey struct { + bucket int64 + route string + routePresent bool +} + +type logSummaryValue struct { + count, scannedBytes int64 +} + +func (s *Store) queryIndexedLogCountSummary(ctx context.Context, path string, ast query.AST, scope query.Scope, budget query.Budget, now time.Time, result query.Result) (query.Result, error) { + runContext, cancel := context.WithTimeout(ctx, budget.MaxDuration) + defer cancel() + started := time.Now() + dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) + if err != nil { + return query.Result{}, errors.New("open indexed log summary") + } + defer db.Close() + db.SetMaxOpenConns(1) + + status, statusErr := typedFilterValue(ast.Filters[0].Value, schema.TypeInteger) + if statusErr != nil { + return query.Result{}, statusErr + } + groups := map[logSummaryKey]logSummaryValue{} + cutoff, rollupStart := logRollupWindowStart(ast, now) + + selects := []string{} + arguments := []any{} + groupColumns := "route,route_present" + if ast.Bucket > 0 { + seconds := int64(ast.Bucket / time.Second) + selects = append(selects, `CAST(bucket_start/? AS INTEGER)*?`) + arguments = append(arguments, seconds, seconds) + groupColumns = "1,route,route_present" + } + selects = append(selects, `route`, `route_present`, `SUM(observation_count)`, `SUM(scanned_bytes)`) + statement := `SELECT ` + joinSQL(selects) + ` FROM log_status_route_rollups_5m WHERE organization_id=?` + arguments = append(arguments, scope.OrganizationID) + for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} { + if selected.value != "" { + statement += " AND " + selected.column + "=?" + arguments = append(arguments, selected.value) + } + } + statement += ` AND status>=?` + arguments = append(arguments, status) + if ast.Window > 0 { + statement += ` AND bucket_start>=?` + arguments = append(arguments, rollupStart) + } + statement += ` GROUP BY ` + groupColumns + rows, err := db.QueryContext(runContext, statement, arguments...) + if err != nil { + return query.Result{}, queryExecutionError(runContext, err) + } + if err = readLogSummaryRows(rows, ast.Bucket > 0, groups); err != nil { + return query.Result{}, queryExecutionError(runContext, err) + } + + if ast.Window > 0 && cutoff.Unix() < rollupStart { + rawSelects := []string{} + rawArguments := []any{} + rawGroupColumns := `json_extract(o.attributes_json,'$."http.route"'),CASE WHEN json_type(o.attributes_json,'$."http.route"') IS NULL THEN 0 ELSE 1 END` + if ast.Bucket > 0 { + seconds := int64(ast.Bucket / time.Second) + rawSelects = append(rawSelects, `CAST(unixepoch(o.timestamp)/? AS INTEGER)*?`) + rawArguments = append(rawArguments, seconds, seconds) + rawGroupColumns = `1,` + rawGroupColumns + } + rawSelects = append(rawSelects, `json_extract(o.attributes_json,'$."http.route"')`, `CASE WHEN json_type(o.attributes_json,'$."http.route"') IS NULL THEN 0 ELSE 1 END`, `COUNT(*)`, `COALESCE(SUM(`+logObservationBytesSQL+`),0)`) + raw := `SELECT ` + joinSQL(rawSelects) + ` FROM observations o INDEXED BY observations_http_status WHERE o.organization_id=? AND o.signal=?` + rawArguments = append(rawArguments, scope.OrganizationID, string(ast.Signal)) + for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} { + if selected.value != "" { + raw += " AND o." + selected.column + "=?" + rawArguments = append(rawArguments, selected.value) + } + } + raw += ` AND o.timestamp>=? AND o.timestamp=? GROUP BY ` + rawGroupColumns + rawArguments = append(rawArguments, status) + partial, queryErr := db.QueryContext(runContext, raw, rawArguments...) + if queryErr != nil { + return query.Result{}, queryExecutionError(runContext, queryErr) + } + if err = readLogSummaryRows(partial, ast.Bucket > 0, groups); err != nil { + return query.Result{}, queryExecutionError(runContext, err) + } + } + + keys := make([]logSummaryKey, 0, len(groups)) + for key := range groups { + keys = append(keys, key) + } + sort.Slice(keys, func(left, right int) bool { + if keys[left].bucket != keys[right].bucket { + return keys[left].bucket < keys[right].bucket + } + if keys[left].routePresent != keys[right].routePresent { + return !keys[left].routePresent + } + return keys[left].route < keys[right].route + }) + var memoryBytes int64 + maximumInt := int64(^uint(0) >> 1) + for _, key := range keys { + value := groups[key] + if value.count < 1 || value.scannedBytes < 0 || value.count > maximumInt-int64(result.Stats.ScannedRows) || value.scannedBytes > budget.MaxScannedBytes-result.Stats.ScannedBytes { + return query.Result{}, query.ErrBudgetExceeded + } + addition := int64(256 + len(key.route)) + if addition > budget.MaxMemoryBytes-memoryBytes { + return query.Result{}, query.ErrBudgetExceeded + } + memoryBytes += addition + result.Stats.ScannedRows += int(value.count) + result.Stats.MatchedRows += int(value.count) + result.Stats.ScannedBytes += value.scannedBytes + values := make([]*string, 0, len(result.Columns)) + if ast.Bucket > 0 { + bucket := time.Unix(key.bucket, 0).UTC().Format(time.RFC3339Nano) + values = append(values, stringPointer(bucket)) + } + if key.routePresent { + values = append(values, stringPointer(key.route)) + } else { + values = append(values, nil) + } + values = append(values, stringPointer(strconv.FormatInt(value.count, 10))) + result.Rows = append(result.Rows, query.Row{Values: values}) + } + if ast.Sort != nil { + if err = sortRows(result.Rows, result.Columns, ast.Sort.Field, ast.Sort.Descending); err != nil { + return query.Result{}, err + } + } + if len(result.Rows) > ast.Limit { + result.Rows = result.Rows[:ast.Limit] + result.Stats.Truncated = true + } + result.Stats.DurationNS = time.Since(started).Nanoseconds() + return result, nil +} + +func readLogSummaryRows(rows *sql.Rows, bucketed bool, groups map[logSummaryKey]logSummaryValue) error { + defer rows.Close() + for rows.Next() { + var bucket sql.NullInt64 + var route sql.NullString + var routePresent int + var count, scannedBytes int64 + var err error + if bucketed { + err = rows.Scan(&bucket, &route, &routePresent, &count, &scannedBytes) + } else { + err = rows.Scan(&route, &routePresent, &count, &scannedBytes) + } + if err != nil || bucketed && !bucket.Valid || routePresent < 0 || routePresent > 1 || routePresent == 1 && !route.Valid || count < 1 || scannedBytes < 0 { + return errors.New("read indexed log summary") + } + key := logSummaryKey{routePresent: routePresent == 1} + if bucketed { + key.bucket = bucket.Int64 + } + if route.Valid { + key.route = route.String + } + current := groups[key] + if count > math.MaxInt64-current.count || scannedBytes > math.MaxInt64-current.scannedBytes { + return query.ErrBudgetExceeded + } + current.count += count + current.scannedBytes += scannedBytes + groups[key] = current + } + return rows.Err() +} + +func joinSQL(parts []string) string { + if len(parts) == 0 { + return "" + } + joined := parts[0] + for _, part := range parts[1:] { + joined += "," + part + } + return joined +} diff --git a/internal/storage/log_summary_query_test.go b/internal/storage/log_summary_query_test.go new file mode 100644 index 0000000..8940a1f --- /dev/null +++ b/internal/storage/log_summary_query_test.go @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +func TestIndexedLogCountSummaryPreservesScopeBucketsAndStatistics(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 18, 12, 4, 0, 0, time.UTC) + primaryScope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "web"} + primaryToken, err := store.CreateSource(ctx, "source-a", primaryScope) + if err != nil { + t.Fatal(err) + } + primary := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + requestObservation(now.Add(-time.Minute), "/items", 503, 300), + requestObservation(now.Add(-2*time.Minute), "/items", 500, 200), + requestObservation(now.Add(-3*time.Minute), "/ignored", 200, 100), + requestObservation(now.Add(-10*time.Minute), "/about", 503, 400), + {Timestamp: now.Add(-time.Minute), Name: "application.http.request", Attributes: map[string]string{"http.route": "/invalid", "http.status_code": "500suffix", "duration_ns": "100"}}, + }} + if _, err = store.Ingest(ctx, primaryToken, primary, now); err != nil { + t.Fatal(err) + } + secondaryScope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-b", EnvironmentID: "production", ServiceID: "worker"} + secondaryToken, err := store.CreateSource(ctx, "source-b", secondaryScope) + if err != nil { + t.Fatal(err) + } + secondary := model.Batch{Version: model.BatchVersion, SourceID: "source-b", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + requestObservation(now.Add(-time.Minute), "/other", 500, 500), + }} + if _, err = store.Ingest(ctx, secondaryToken, secondary, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + + ast, err := query.Parse(`logs | where status >= 500 | window 1h | summarize count() by route, window(5m) | sort count desc | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now) + if err != nil { + t.Fatal(err) + } + if len(result.Explain.ProjectedSources) != 1 || !strings.HasSuffix(result.Explain.ProjectedSources[0], "/rollup:http-status-route:5m") { + t.Fatalf("sources=%v", result.Explain.ProjectedSources) + } + if len(result.Rows) != 3 || columnValue(t, result, 0, "http.route") != "/items" || columnValue(t, result, 0, "count") != "2" { + t.Fatalf("result=%+v", result) + } + if result.Stats.ScannedRows != 4 || result.Stats.MatchedRows != 4 || result.Stats.ScannedBytes < 1 || result.Stats.Truncated { + t.Fatalf("statistics=%+v", result.Stats) + } + if result.Explain.EstimatedScanBytes < 1 || result.Explain.EstimatedScanBytes >= result.Stats.ScannedBytes { + t.Fatalf("estimate=%d logical_scan=%d", result.Explain.EstimatedScanBytes, result.Stats.ScannedBytes) + } + if bucket := columnValue(t, result, 0, "window_start"); bucket != now.Add(-time.Minute).Truncate(5*time.Minute).Format(time.RFC3339Nano) { + t.Fatalf("bucket=%q", bucket) + } + + scoped, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a", ProjectID: "project-a"}, testQueryBudget(), now) + if err != nil { + t.Fatal(err) + } + if len(scoped.Rows) != 2 || scoped.Stats.ScannedRows != 3 || scoped.Stats.MatchedRows != 3 { + t.Fatalf("scoped=%+v", scoped) + } +} + +func TestIndexedLogCountSummaryUsesRawOnlyForPartialLowerBucket(t *testing.T) { + ctx := t.Context() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 18, 12, 7, 30, 0, time.UTC) + scope := model.Scope{OrganizationID: "organization-partial", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-partial", scope) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-partial", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + requestObservation(time.Date(2026, 8, 18, 12, 1, 0, 0, time.UTC), "/before", 500, 1), + requestObservation(time.Date(2026, 8, 18, 12, 2, 0, 0, time.UTC), "/partial", 500, 2), + requestObservation(time.Date(2026, 8, 18, 12, 4, 59, 0, time.UTC), "/partial", 503, 3), + requestObservation(time.Date(2026, 8, 18, 12, 5, 0, 0, time.UTC), "/full", 500, 4), + requestObservation(time.Date(2026, 8, 18, 12, 6, 0, 0, time.UTC), "/ignored", 200, 5), + {Timestamp: time.Date(2026, 8, 18, 12, 5, 1, 0, time.UTC), Name: "application.http.request", Attributes: map[string]string{"http.status_code": "500"}}, + {Timestamp: time.Date(2026, 8, 18, 12, 5, 2, 0, time.UTC), Name: "application.http.request", Attributes: map[string]string{"http.route": "", "http.status_code": "500"}}, + }} + ack, err := store.Ingest(ctx, token, batch, now) + if err != nil { + t.Fatal(err) + } + projectAll(t, store) + // Replaying the already projected segment must not increment the additive + // rollup. The segment ledger makes projection recovery idempotent. + if err = projectAt(ctx, store.organizationProjectionPath(scope.OrganizationID), scope, batch, ack.Digest); err != nil { + t.Fatal(err) + } + + ast, err := query.Parse(`logs | where status >= 500 | window 6m | summarize count() by route, window(5m) | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now) + if err != nil { + t.Fatal(err) + } + if result.Stats.ScannedRows != 5 || result.Stats.MatchedRows != 5 || len(result.Rows) != 4 { + t.Fatalf("result=%+v", result) + } + if bucket := columnValue(t, result, 0, "window_start"); bucket != "2026-08-18T12:00:00Z" { + t.Fatalf("partial bucket=%q", bucket) + } + if route := columnValue(t, result, 0, "http.route"); route != "/partial" || columnValue(t, result, 0, "count") != "2" { + t.Fatalf("partial row=%+v", result.Rows[0]) + } + // Missing and explicitly empty routes remain distinct typed values. + if result.Rows[1].Values[1] != nil || result.Rows[2].Values[1] == nil || *result.Rows[2].Values[1] != "" { + t.Fatalf("route presence was collapsed: %+v", result.Rows) + } + if columnValue(t, result, 3, "http.route") != "/full" || columnValue(t, result, 3, "count") != "1" { + t.Fatalf("full row=%+v", result.Rows[3]) + } +} + +func TestLogRollupMigrationBackfillsExistingProjection(t *testing.T) { + ctx := t.Context() + root := filepath.Join(t.TempDir(), "data") + if err := os.MkdirAll(filepath.Join(root, "organizations", "legacy"), 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "organizations", "legacy", "projection.sqlite") + legacy, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + if _, err = legacy.Exec(`CREATE TABLE observations ( + organization_id TEXT NOT NULL, project_id TEXT NOT NULL, environment_id TEXT NOT NULL, service_id TEXT NOT NULL, + source_id TEXT NOT NULL, stream_id TEXT NOT NULL, sequence INTEGER NOT NULL, record_index INTEGER NOT NULL, + signal TEXT NOT NULL, timestamp TEXT NOT NULL, name TEXT NOT NULL, severity TEXT, body TEXT, value REAL, + trace_id TEXT, span_id TEXT, correlation_id TEXT, attributes_json TEXT NOT NULL, segment_digest TEXT NOT NULL, + PRIMARY KEY(source_id,stream_id,sequence,record_index)); + INSERT INTO observations VALUES + ('legacy','project','prod','web','source','logs',1,0,'logs','2026-08-18T12:01:00Z','request',NULL,NULL,NULL,NULL,NULL,NULL,'{"http.route":"/one","http.status_code":"503"}','digest'), + ('legacy','project','prod','web','source','logs',1,1,'logs','2026-08-18T12:02:00Z','request',NULL,NULL,NULL,NULL,NULL,NULL,'{"http.route":"/one","http.status_code":"503"}','digest'), + ('legacy','project','prod','web','source','logs',1,2,'logs','2026-08-18T12:02:01Z','request',NULL,NULL,NULL,NULL,NULL,NULL,'{"http.route":"/invalid","http.status_code":"503suffix"}','digest')`); err != nil { + t.Fatal(err) + } + if err = legacy.Close(); err != nil { + t.Fatal(err) + } + projection, err := openProjection(ctx, path) + if err != nil { + t.Fatal(err) + } + defer projection.Close() + var count, segments, version int + if err = projection.QueryRow(`SELECT observation_count FROM log_status_route_rollups_5m`).Scan(&count); err != nil || count != 2 { + t.Fatalf("count=%d err=%v", count, err) + } + if err = projection.QueryRow(`SELECT COUNT(*) FROM log_rollup_segments`).Scan(&segments); err != nil || segments != 1 { + t.Fatalf("segments=%d err=%v", segments, err) + } + if err = projection.QueryRow(`SELECT version FROM log_rollup_state WHERE id=1`).Scan(&version); err != nil || version != logRollupVersion { + t.Fatalf("version=%d err=%v", version, err) + } +} + +func TestIndexedLogCountSummaryEligibilityIsNarrow(t *testing.T) { + tests := []struct { + text string + want bool + }{ + {`logs | where status >= 500 | summarize count() by route, window(5m) | limit 10`, true}, + {`logs | where status >= 400 | summarize count() by route | limit 10`, true}, + {`logs | where status == 500 | summarize count() by route | limit 10`, false}, + {`logs | where status =~ "5.." | summarize count() by route, window(5m) | limit 10`, false}, + {`logs | where route == "/items" | summarize count() by route, window(5m) | limit 10`, false}, + {`logs | where status >= 500 | summarize count() by status, window(5m) | limit 10`, false}, + {`logs | where status >= 500 | summarize p95(duration) by route, window(5m) | limit 10`, false}, + {`logs | where status >= 500 | summarize count() by route, window(1500ms) | limit 10`, false}, + {`traces | where status >= 500 | summarize count() by route, window(5m) | limit 10`, false}, + } + for _, test := range tests { + ast, err := query.Parse(test.text, 100) + if err != nil { + t.Fatalf("parse %q: %v", test.text, err) + } + if got := indexedLogCountSummaryEligible(ast); got != test.want { + t.Fatalf("eligible(%q)=%t want %t", test.text, got, test.want) + } + } +} diff --git a/internal/storage/migration_test.go b/internal/storage/migration_test.go new file mode 100644 index 0000000..ebb16ac --- /dev/null +++ b/internal/storage/migration_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "database/sql" + "path/filepath" + "testing" +) + +func TestControlSchemaFourMigratesThroughRetentionSchema(t *testing.T) { + db := openSchemaDatabase(t, 4) + defer db.Close() + var err error + if err = migrateControl(db); err != nil { + t.Fatal(err) + } + var version int + if err = db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != controlSchema { + t.Fatalf("version=%d err=%v", version, err) + } + for _, table := range []string{"alert_rules", "incidents", "incident_events", "organization_retention_policies", "retention_policy_events", "source_alert_transitions"} { + var count int + if err = db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&count); err != nil || count != 1 { + t.Fatalf("table=%s count=%d err=%v", table, count, err) + } + } + columns, err := sqliteColumns(db, "segments") + if err != nil { + t.Fatal(err) + } + for _, column := range []string{"signal", "first_observed_at", "last_observed_at", "record_count", "tier", "archiving_at", "archive_path", "cold_at", "retiring_at"} { + if !columns[column] { + t.Fatalf("missing segment column %s", column) + } + } + policyColumns, err := sqliteColumns(db, "organization_retention_policies") + if err != nil || !policyColumns["cold_raw_days"] || !policyColumns["delete_cold_raw"] { + t.Fatalf("retention policy cold=%t delete=%t err=%v", policyColumns["cold_raw_days"], policyColumns["delete_cold_raw"], err) + } + streamColumns, err := sqliteColumns(db, "streams") + if err != nil { + t.Fatal(err) + } + for _, column := range []string{"last_batch_digest", "last_wire_digest", "last_signal", "last_record_count", "last_encoded_bytes", "last_first_observed_at", "last_last_observed_at"} { + if !streamColumns[column] { + t.Fatalf("missing stream envelope column %s", column) + } + } +} + +func TestControlSchemaSevenMigratesToForensicPreservation(t *testing.T) { + db := openSchemaDatabase(t, 6) + defer db.Close() + if err := migrateControlRetention(db); err != nil { + t.Fatal(err) + } + var version int + if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != 7 { + t.Fatalf("pre-migration version=%d err=%v", version, err) + } + if err := migrateControl(db); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO organization_retention_policies(organization_id,raw_logs_days,raw_traces_days,raw_metrics_days,cold_raw_days,metric_rollups_days,evidence_days,updated_by,updated_at) VALUES('org',30,30,14,400,400,400,'owner','2026-08-17T00:00:00Z')`); err != nil { + t.Fatal(err) + } + var deleteColdRaw bool + if err := db.QueryRow(`SELECT delete_cold_raw FROM organization_retention_policies WHERE organization_id='org'`).Scan(&deleteColdRaw); err != nil || deleteColdRaw { + t.Fatalf("delete_cold_raw=%t err=%v", deleteColdRaw, err) + } +} + +func TestControlSchemaTenPreservesExistingStreamWatermark(t *testing.T) { + db := openSchemaDatabase(t, 10) + defer db.Close() + if _, err := db.Exec(` + CREATE TABLE sources ( + id TEXT PRIMARY KEY, organization_id TEXT NOT NULL, project_id TEXT NOT NULL, + environment_id TEXT NOT NULL, service_id TEXT NOT NULL, + credential_digest BLOB NOT NULL UNIQUE, active INTEGER NOT NULL, + created_at TEXT NOT NULL, rotated_at TEXT + ); + CREATE TABLE streams ( + source_id TEXT NOT NULL REFERENCES sources(id), stream_id TEXT NOT NULL, + last_sequence INTEGER NOT NULL, last_digest TEXT NOT NULL, + PRIMARY KEY(source_id,stream_id) + ); + INSERT INTO sources VALUES('source','org','project','prod','service',X'01',1,'2026-08-18T20:00:00Z',NULL); + INSERT INTO streams VALUES('source','logs',42,'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); + `); err != nil { + t.Fatal(err) + } + if err := migrateControl(db); err != nil { + t.Fatal(err) + } + var sequence int + var segmentDigest, batchDigest, wireDigest string + if err := db.QueryRow(`SELECT last_sequence,last_digest,last_batch_digest,last_wire_digest FROM streams WHERE source_id='source' AND stream_id='logs'`).Scan(&sequence, &segmentDigest, &batchDigest, &wireDigest); err != nil { + t.Fatal(err) + } + if sequence != 42 || segmentDigest != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || batchDigest != "" || wireDigest != "" { + t.Fatalf("sequence=%d segment=%q batch=%q wire=%q", sequence, segmentDigest, batchDigest, wireDigest) + } +} + +func openSchemaDatabase(t *testing.T, version int) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "control.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL); INSERT INTO schema_version(version) VALUES(?)`, version); err != nil { + db.Close() + t.Fatal(err) + } + return db +} diff --git a/internal/storage/processlock.go b/internal/storage/processlock.go new file mode 100644 index 0000000..ff78f43 --- /dev/null +++ b/internal/storage/processlock.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "errors" + "os" + "path/filepath" + "syscall" +) + +type ProcessLock struct { + file *os.File +} + +// AcquireProcessLock coordinates the long-running server and offline migration +// commands. Servers hold a shared lock; projection rebuilds require exclusive +// ownership and therefore cannot overlap a live server process. +func AcquireProcessLock(root string, exclusive bool) (*ProcessLock, error) { + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return nil, errors.New("process lock root must be absolute and clean") + } + if err := os.MkdirAll(root, 0o700); err != nil { + return nil, errors.New("create process lock root") + } + rootInfo, err := os.Lstat(root) + if err != nil || !rootInfo.IsDir() || rootInfo.Mode()&os.ModeSymlink != 0 || rootInfo.Mode().Perm()&0o077 != 0 { + return nil, errors.New("process lock root must be a private non-symlink directory") + } + path := filepath.Join(root, "process.lock") + fd, err := syscall.Open(path, syscall.O_RDWR|syscall.O_CREAT|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return nil, errors.New("open process lock") + } + file := os.NewFile(uintptr(fd), path) + if file == nil { + _ = syscall.Close(fd) + return nil, errors.New("open process lock") + } + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + _ = file.Close() + return nil, errors.New("process lock must be a regular non-symlink file") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Nlink != 1 { + _ = file.Close() + return nil, errors.New("process lock must not have additional hard links") + } + operation := syscall.LOCK_SH | syscall.LOCK_NB + if exclusive { + operation = syscall.LOCK_EX | syscall.LOCK_NB + } + if err = syscall.Flock(fd, operation); err != nil { + _ = file.Close() + return nil, errors.New("Observatory data directory is active in another process") + } + if err = file.Chmod(0o600); err != nil { + _ = syscall.Flock(fd, syscall.LOCK_UN) + _ = file.Close() + return nil, errors.New("secure process lock") + } + return &ProcessLock{file: file}, nil +} + +func (lock *ProcessLock) Close() error { + if lock == nil || lock.file == nil { + return nil + } + err := syscall.Flock(int(lock.file.Fd()), syscall.LOCK_UN) + closeErr := lock.file.Close() + lock.file = nil + if err != nil { + return err + } + return closeErr +} diff --git a/internal/storage/processlock_test.go b/internal/storage/processlock_test.go new file mode 100644 index 0000000..be3e54b --- /dev/null +++ b/internal/storage/processlock_test.go @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "os" + "path/filepath" + "testing" +) + +func TestProcessLockSeparatesLiveServerFromOfflineMigration(t *testing.T) { + root := filepath.Join(t.TempDir(), "data") + first, err := AcquireProcessLock(root, false) + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := AcquireProcessLock(root, false) + if err != nil { + t.Fatal(err) + } + if _, err = AcquireProcessLock(root, true); err == nil { + t.Fatal("exclusive migration lock overlapped live server locks") + } + if err = second.Close(); err != nil { + t.Fatal(err) + } + if err = first.Close(); err != nil { + t.Fatal(err) + } + exclusive, err := AcquireProcessLock(root, true) + if err != nil { + t.Fatal(err) + } + defer exclusive.Close() + if _, err = AcquireProcessLock(root, false); err == nil { + t.Fatal("server lock overlapped exclusive migration lock") + } +} + +func TestProcessLockRejectsUnsafeFilesystemObjects(t *testing.T) { + base := t.TempDir() + private := filepath.Join(base, "private") + if err := os.Mkdir(private, 0o700); err != nil { + t.Fatal(err) + } + symlinkRoot := filepath.Join(base, "symlink-root") + if err := os.Symlink(private, symlinkRoot); err != nil { + t.Fatal(err) + } + if _, err := AcquireProcessLock(symlinkRoot, false); err == nil { + t.Fatal("symlink process-lock root was accepted") + } + public := filepath.Join(base, "public") + if err := os.Mkdir(public, 0o755); err != nil { + t.Fatal(err) + } + if _, err := AcquireProcessLock(public, false); err == nil { + t.Fatal("public process-lock root was accepted") + } + lockPath := filepath.Join(private, "process.lock") + target := filepath.Join(private, "target") + if err := os.WriteFile(target, []byte("target"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, lockPath); err != nil { + t.Fatal(err) + } + if _, err := AcquireProcessLock(private, false); err == nil { + t.Fatal("symlink process-lock file was accepted") + } + if err := os.Remove(lockPath); err != nil { + t.Fatal(err) + } + if err := os.Link(target, lockPath); err != nil { + t.Fatal(err) + } + if _, err := AcquireProcessLock(private, false); err == nil { + t.Fatal("hard-linked process-lock file was accepted") + } +} diff --git a/internal/storage/projector.go b/internal/storage/projector.go new file mode 100644 index 0000000..b5f9b39 --- /dev/null +++ b/internal/storage/projector.go @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sync" + "time" + + "gamertan.com/observatory/internal/model" +) + +const ( + projectionGroupMaxSegments = 16 + projectionGroupMaxBytes = 32 << 20 + projectionWorkerLimit = 4 + defaultProjectorInterval = time.Second +) + +// ProjectionReport describes one bounded projector pass. Accepted raw +// segments remain durable and replayable independently of this report. +type ProjectionReport struct { + ProjectedSegments int `json:"projected_segments"` + ProjectedRecords int `json:"projected_records"` + ProjectedBytes int64 `json:"projected_bytes"` +} + +// ProjectionStatus makes asynchronous query visibility explicit. Lag is the +// age of the oldest durable segment that has not yet reached the read model. +type ProjectionStatus struct { + PendingSegments int `json:"pending_segments"` + PendingBytes int64 `json:"pending_bytes"` + OldestCommitted time.Time `json:"oldest_committed_at,omitempty"` + OldestPendingLag time.Duration `json:"oldest_pending_lag"` +} + +type pendingProjection struct { + digest string + path string + uncompressedBytes int64 + sourceID string + streamID string + sequence uint64 + committedAt time.Time + catalogOrgID string + scope model.Scope +} + +type pendingProjectionGroup struct { + organizationID string + segments []pendingProjection + bytes int64 +} + +// ProjectionStatus returns a bounded control-database view and never opens an +// organization projection. +func (s *Store) ProjectionStatus(ctx context.Context, now time.Time) (ProjectionStatus, error) { + return s.projectionStatus(ctx, "", now) +} + +// OrganizationProjectionStatus restricts lag evidence to one tenant so the +// authenticated UI never discloses another organization's ingestion volume. +func (s *Store) OrganizationProjectionStatus(ctx context.Context, organizationID string, now time.Time) (ProjectionStatus, error) { + if err := model.ValidateSourceID(organizationID); err != nil { + return ProjectionStatus{}, errors.New("invalid organization identifier") + } + return s.projectionStatus(ctx, organizationID, now) +} + +func (s *Store) projectionStatus(ctx context.Context, organizationID string, now time.Time) (ProjectionStatus, error) { + if now.IsZero() { + return ProjectionStatus{}, errors.New("projection status time is required") + } + var status ProjectionStatus + var oldest sql.NullString + var err error + if organizationID == "" { + err = s.control.QueryRowContext(ctx, `SELECT COUNT(*),COALESCE(SUM(uncompressed_bytes),0),MIN(committed_at) FROM segments WHERE projected_at IS NULL`).Scan(&status.PendingSegments, &status.PendingBytes, &oldest) + } else { + err = s.control.QueryRowContext(ctx, `SELECT COUNT(*),COALESCE(SUM(uncompressed_bytes),0),MIN(committed_at) FROM segments WHERE projected_at IS NULL AND organization_id=?`, organizationID).Scan(&status.PendingSegments, &status.PendingBytes, &oldest) + } + if err != nil { + return ProjectionStatus{}, fmt.Errorf("read projection status: %w", err) + } + if oldest.Valid { + parsed, parseErr := time.Parse(time.RFC3339Nano, oldest.String) + if parseErr != nil { + return ProjectionStatus{}, errors.New("projection status contains an invalid timestamp") + } + status.OldestCommitted = parsed.UTC() + if now.After(status.OldestCommitted) { + status.OldestPendingLag = now.Sub(status.OldestCommitted) + } + } + return status, nil +} + +// ProjectPending projects a bounded set of already-durable segments. It +// groups work by organization so one SQLite transaction can safely amortize +// multiple agent batches without crossing tenant databases. +func (s *Store) ProjectPending(ctx context.Context) (ProjectionReport, error) { + s.projectorMu.Lock() + defer s.projectorMu.Unlock() + + pending, err := s.pendingProjections(ctx) + if err != nil || len(pending) == 0 { + return ProjectionReport{}, err + } + groups := boundedProjectionGroups(pending) + type projectionResult struct { + report ProjectionReport + err error + } + results := make([]projectionResult, len(groups)) + workers := min(len(groups), projectionWorkerLimit) + jobs := make(chan int) + var wait sync.WaitGroup + wait.Add(workers) + for range workers { + go func() { + defer wait.Done() + for index := range jobs { + results[index].report, results[index].err = s.projectPendingGroup(ctx, groups[index]) + } + }() + } + for index := range groups { + jobs <- index + } + close(jobs) + wait.Wait() + + var report ProjectionReport + var projectionErrors []error + for _, result := range results { + if result.err != nil { + projectionErrors = append(projectionErrors, result.err) + continue + } + report.ProjectedSegments += result.report.ProjectedSegments + report.ProjectedRecords += result.report.ProjectedRecords + report.ProjectedBytes += result.report.ProjectedBytes + } + return report, errors.Join(projectionErrors...) +} + +func (s *Store) pendingProjections(ctx context.Context) ([]pendingProjection, error) { + rows, err := s.control.QueryContext(ctx, `WITH ranked AS ( + SELECT g.digest,g.path,g.uncompressed_bytes,g.source_id,g.stream_id,g.sequence,g.committed_at,g.organization_id, + s.organization_id AS source_organization_id,s.project_id,s.environment_id,s.service_id, + ROW_NUMBER() OVER (PARTITION BY g.organization_id ORDER BY g.committed_at,g.digest) AS organization_rank + FROM segments AS g JOIN sources AS s ON s.id=g.source_id + WHERE g.projected_at IS NULL + ) + SELECT digest,path,uncompressed_bytes,source_id,stream_id,sequence,committed_at,organization_id,source_organization_id,project_id,environment_id,service_id + FROM ranked WHERE organization_rank<=? ORDER BY committed_at,digest LIMIT ?`, projectionGroupMaxSegments, recoveryPageSize) + if err != nil { + return nil, fmt.Errorf("list unprojected segments: %w", err) + } + defer rows.Close() + pending := make([]pendingProjection, 0, recoveryPageSize) + for rows.Next() { + var item pendingProjection + var committedAt string + if err = rows.Scan(&item.digest, &item.path, &item.uncompressedBytes, &item.sourceID, &item.streamID, &item.sequence, &committedAt, &item.catalogOrgID, &item.scope.OrganizationID, &item.scope.ProjectID, &item.scope.EnvironmentID, &item.scope.ServiceID); err != nil { + return nil, fmt.Errorf("scan unprojected segment: %w", err) + } + item.committedAt, err = time.Parse(time.RFC3339Nano, committedAt) + if err != nil { + return nil, errors.New("unprojected segment has an invalid committed time") + } + pending = append(pending, item) + } + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("iterate unprojected segments: %w", err) + } + return pending, nil +} + +func boundedProjectionGroups(pending []pendingProjection) []pendingProjectionGroup { + groups := make([]pendingProjectionGroup, 0) + byOrganization := make(map[string]int) + for _, item := range pending { + index, exists := byOrganization[item.scope.OrganizationID] + if !exists { + index = len(groups) + byOrganization[item.scope.OrganizationID] = index + groups = append(groups, pendingProjectionGroup{organizationID: item.scope.OrganizationID}) + } + group := &groups[index] + if len(group.segments) >= projectionGroupMaxSegments { + continue + } + if len(group.segments) > 0 && group.bytes+item.uncompressedBytes > projectionGroupMaxBytes { + continue + } + group.segments = append(group.segments, item) + group.bytes += item.uncompressedBytes + } + return groups +} + +func (s *Store) projectPendingGroup(ctx context.Context, group pendingProjectionGroup) (ProjectionReport, error) { + if len(group.segments) == 0 { + return ProjectionReport{}, nil + } + lock := s.namedLock("organization:" + group.organizationID) + lock.Lock() + defer lock.Unlock() + + items := make([]projectionItem, 0, len(group.segments)) + report := ProjectionReport{ProjectedSegments: len(group.segments), ProjectedBytes: group.bytes} + for _, pending := range group.segments { + batch, err := s.segments.Read(pending.path, pending.digest) + if err != nil { + return ProjectionReport{}, fmt.Errorf("read pending projection segment: %w", err) + } + if batch.SourceID != pending.sourceID || batch.StreamID != pending.streamID || batch.Sequence != pending.sequence || pending.catalogOrgID != group.organizationID || pending.scope.OrganizationID != group.organizationID { + return ProjectionReport{}, errors.New("pending projection identity does not match durable catalog") + } + if err = batch.Validate(batch.ObservedAt); err != nil { + return ProjectionReport{}, fmt.Errorf("validate pending projection batch: %w", err) + } + if err = validateMetricRollupCardinality(batch); err != nil { + return ProjectionReport{}, fmt.Errorf("validate pending projection cardinality: %w", err) + } + report.ProjectedRecords += len(batch.Records) + items = append(items, projectionItem{scope: pending.scope, batch: batch, digest: pending.digest}) + } + db, err := s.projection(ctx, group.organizationID) + if err != nil { + return ProjectionReport{}, err + } + if err = projectGroupWithDB(ctx, db, items); err != nil { + return ProjectionReport{}, err + } + projectedAt := time.Now().UTC() + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return ProjectionReport{}, fmt.Errorf("begin projection acknowledgement: %w", err) + } + defer tx.Rollback() + for index, item := range items { + if err = recordDescriptorProposalsTx(ctx, tx, group.organizationID, item.batch, item.digest, group.segments[index].committedAt); err != nil { + return ProjectionReport{}, err + } + if err = markProjectedTx(ctx, tx, item.batch, item.digest, projectedAt); err != nil { + return ProjectionReport{}, err + } + } + if err = tx.Commit(); err != nil { + return ProjectionReport{}, fmt.Errorf("commit projection acknowledgement: %w", err) + } + return report, nil +} + +// RunProjector continuously drains durable work, then sleeps until ingestion +// wakes it or the reconciliation interval expires. Errors leave raw segments +// pending and are retried without making acknowledgement availability depend +// on query projection health. +func (s *Store) RunProjector(ctx context.Context, interval time.Duration, onError func(error)) { + if interval <= 0 { + interval = defaultProjectorInterval + } + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + case <-s.projectionWake: + } + report, err := s.ProjectPending(ctx) + if err != nil && !errors.Is(err, context.Canceled) && onError != nil { + onError(err) + } + delay := interval + if err == nil && report.ProjectedSegments > 0 { + delay = 0 + } + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(delay) + } +} diff --git a/internal/storage/projector_test.go b/internal/storage/projector_test.go new file mode 100644 index 0000000..c522e6e --- /dev/null +++ b/internal/storage/projector_test.go @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "os" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +func TestDurableAcknowledgementPrecedesProjection(t *testing.T) { + ctx := t.Context() + store := testStore(t) + defer store.Close() + now := time.Now().UTC() + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}} + ack, err := store.Ingest(ctx, token, batch, now) + if err != nil || ack.Duplicate { + t.Fatalf("ack=%+v err=%v", ack, err) + } + duplicate, err := store.Ingest(ctx, token, batch, now) + if err != nil || !duplicate.Duplicate || duplicate.Digest != ack.Digest { + t.Fatalf("duplicate=%+v err=%v", duplicate, err) + } + status, err := store.ProjectionStatus(ctx, now.Add(time.Second)) + if err != nil || status.PendingSegments != 1 || status.PendingBytes < 1 || status.OldestPendingLag < time.Second { + t.Fatalf("status=%+v err=%v", status, err) + } + var projected int + if err = store.control.QueryRowContext(ctx, `SELECT COUNT(projected_at) FROM segments`).Scan(&projected); err != nil || projected != 0 { + t.Fatalf("projected=%d err=%v", projected, err) + } + report, err := store.ProjectPending(ctx) + if err != nil || report.ProjectedSegments != 1 || report.ProjectedRecords != 1 || report.ProjectedBytes < 1 { + t.Fatalf("report=%+v err=%v", report, err) + } + status, err = store.ProjectionStatus(ctx, now.Add(2*time.Second)) + if err != nil || status.PendingSegments != 0 || status.PendingBytes != 0 || !status.OldestCommitted.IsZero() || status.OldestPendingLag != 0 { + t.Fatalf("projected status=%+v err=%v", status, err) + } +} + +func TestProjectorGroupsBoundedSegmentsAndResumesAfterReopen(t *testing.T) { + ctx := t.Context() + store := testStore(t) + root := store.root + now := time.Now().UTC() + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + const total = projectionGroupMaxSegments + 3 + for sequence := uint64(1); sequence <= total; sequence++ { + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: sequence, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + } + first, err := store.ProjectPending(ctx) + if err != nil || first.ProjectedSegments != projectionGroupMaxSegments { + t.Fatalf("first=%+v err=%v", first, err) + } + if err = store.Close(); err != nil { + t.Fatal(err) + } + store, err = Open(root) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if err = store.RecoverRaw(ctx); err != nil { + t.Fatal(err) + } + status, err := store.ProjectionStatus(ctx, now.Add(time.Second)) + if err != nil || status.PendingSegments != total-projectionGroupMaxSegments { + t.Fatalf("status=%+v err=%v", status, err) + } + second, err := store.ProjectPending(ctx) + if err != nil || second.ProjectedSegments != total-projectionGroupMaxSegments { + t.Fatalf("second=%+v err=%v", second, err) + } + ast, err := query.Parse(`logs | limit 100`, 100) + if err != nil { + t.Fatal(err) + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now) + if err != nil || len(result.Rows) != total { + t.Fatalf("rows=%d err=%v", len(result.Rows), err) + } +} + +func TestProjectorFailureIsolatedFromAcknowledgementAndOtherOrganization(t *testing.T) { + ctx := t.Context() + store := testStore(t) + defer store.Close() + now := time.Now().UTC() + for index, organizationID := range []string{"organization-a", "organization-b"} { + sourceID := "source-" + string(rune('a'+index)) + scope := model.Scope{OrganizationID: organizationID, ProjectID: "project", EnvironmentID: "production", ServiceID: "service"} + token, err := store.CreateSource(ctx, sourceID, scope) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: sourceID, StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + } + var corruptPath string + if err := store.control.QueryRowContext(ctx, `SELECT path FROM segments WHERE organization_id='organization-a'`).Scan(&corruptPath); err != nil { + t.Fatal(err) + } + corruptBody, err := os.ReadFile(corruptPath) + if err != nil { + t.Fatal(err) + } + corruptBody[0] ^= 0xff + if err := os.WriteFile(corruptPath, corruptBody, 0o600); err != nil { + t.Fatal(err) + } + report, err := store.ProjectPending(ctx) + if err == nil || report.ProjectedSegments != 1 { + t.Fatalf("report=%+v err=%v", report, err) + } + var projectedA, projectedB int + if err = store.control.QueryRowContext(ctx, `SELECT COUNT(projected_at) FROM segments WHERE organization_id='organization-a'`).Scan(&projectedA); err != nil { + t.Fatal(err) + } + if err = store.control.QueryRowContext(ctx, `SELECT COUNT(projected_at) FROM segments WHERE organization_id='organization-b'`).Scan(&projectedB); err != nil { + t.Fatal(err) + } + if projectedA != 0 || projectedB != 1 { + t.Fatalf("projected organization-a=%d organization-b=%d", projectedA, projectedB) + } + if err = store.RecoverRaw(ctx); err != nil { + t.Fatalf("raw reconciliation decoded catalogued pending evidence: %v", err) + } +} + +func TestRunProjectorWakesOnAcceptedBatch(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + store := testStore(t) + defer store.Close() + done := make(chan struct{}) + go func() { + defer close(done) + store.RunProjector(ctx, time.Minute, func(err error) { t.Errorf("projector: %v", err) }) + }() + now := time.Now().UTC() + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for { + status, statusErr := store.ProjectionStatus(t.Context(), time.Now().UTC()) + if statusErr != nil { + t.Fatal(statusErr) + } + if status.PendingSegments == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("projection remained pending: %+v", status) + } + time.Sleep(10 * time.Millisecond) + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("projector did not stop after cancellation") + } +} diff --git a/internal/storage/push.go b/internal/storage/push.go new file mode 100644 index 0000000..45c44ef --- /dev/null +++ b/internal/storage/push.go @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "time" + "unicode/utf8" + + "gamertan.com/observatory/internal/model" +) + +const ( + MaxPushSubscriptionsPerUser = 8 + MaxPushSubscriptionsPerPass = 256 +) + +type PushSubscription struct { + OrganizationID string + ID string + UserID string + Endpoint string + P256DH []byte + Auth []byte + FailureCount int + CreatedAt time.Time + UpdatedAt time.Time + LastSentAt *time.Time +} + +type PushSubscriptionInput struct { + OrganizationID string + UserID string + Endpoint string + P256DH []byte + Auth []byte +} + +func (s *Store) SavePushSubscription(ctx context.Context, input PushSubscriptionInput, now time.Time) (PushSubscription, error) { + if err := validatePushSubscription(input); err != nil { + return PushSubscription{}, err + } + digest := sha256.Sum256([]byte(input.Endpoint)) + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return PushSubscription{}, errors.New("save push subscription") + } + defer tx.Rollback() + var endpointID, endpointUser string + err = tx.QueryRowContext(ctx, `SELECT id,user_id FROM push_endpoints WHERE endpoint_digest=?`, digest[:]).Scan(&endpointID, &endpointUser) + switch { + case err == nil: + if endpointUser != input.UserID { + return PushSubscription{}, errors.New("push subscription is already registered") + } + _, err = tx.ExecContext(ctx, `UPDATE push_endpoints SET endpoint=?,p256dh=?,auth_secret=?,active=1,failure_count=0,updated_at=? WHERE id=?`, input.Endpoint, input.P256DH, input.Auth, now.UTC().Format(time.RFC3339Nano), endpointID) + case errors.Is(err, sql.ErrNoRows): + endpointID, err = storageID("endpoint") + if err == nil { + stamp := now.UTC().Format(time.RFC3339Nano) + _, err = tx.ExecContext(ctx, `INSERT INTO push_endpoints(id,user_id,endpoint,endpoint_digest,p256dh,auth_secret,active,failure_count,created_at,updated_at) VALUES(?,?,?,?,?,?,1,0,?,?)`, endpointID, input.UserID, input.Endpoint, digest[:], input.P256DH, input.Auth, stamp, stamp) + } + default: + return PushSubscription{}, errors.New("save push subscription") + } + if err != nil { + return PushSubscription{}, errors.New("save push subscription") + } + var subscriptionID string + err = tx.QueryRowContext(ctx, `SELECT id FROM push_subscriptions WHERE organization_id=? AND user_id=? AND endpoint_id=?`, input.OrganizationID, input.UserID, endpointID).Scan(&subscriptionID) + if errors.Is(err, sql.ErrNoRows) { + var count int + if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM push_subscriptions WHERE organization_id=? AND user_id=?`, input.OrganizationID, input.UserID).Scan(&count); err != nil { + return PushSubscription{}, errors.New("save push subscription") + } + if count >= MaxPushSubscriptionsPerUser { + return PushSubscription{}, errors.New("push subscription limit reached") + } + subscriptionID, err = storageID("push") + if err == nil { + _, err = tx.ExecContext(ctx, `INSERT INTO push_subscriptions(organization_id,id,user_id,endpoint_id,created_at) VALUES(?,?,?,?,?)`, input.OrganizationID, subscriptionID, input.UserID, endpointID, now.UTC().Format(time.RFC3339Nano)) + } + } + if err != nil { + return PushSubscription{}, errors.New("save push subscription") + } + if err = tx.Commit(); err != nil { + return PushSubscription{}, errors.New("save push subscription") + } + return s.PushSubscription(ctx, input.OrganizationID, subscriptionID) +} + +func (s *Store) HasPushSubscription(ctx context.Context, organizationID, userID, endpoint string) (bool, error) { + if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(userID) != nil || !utf8.ValidString(endpoint) || len(endpoint) < 1 || len(endpoint) > 2048 { + return false, errors.New("push subscription lookup is invalid") + } + digest := sha256.Sum256([]byte(endpoint)) + var count int + err := s.control.QueryRowContext(ctx, `SELECT COUNT(*) FROM push_subscriptions s JOIN push_endpoints e ON e.id=s.endpoint_id WHERE s.organization_id=? AND s.user_id=? AND e.endpoint_digest=? AND e.active=1`, organizationID, userID, digest[:]).Scan(&count) + if err != nil { + return false, errors.New("lookup push subscription") + } + return count == 1, nil +} + +// DeletePushSubscription removes one organization mapping. The returned +// value reports whether the browser endpoint remains mapped elsewhere for +// the same user and therefore must remain subscribed in the user agent. +func (s *Store) DeletePushSubscription(ctx context.Context, organizationID, userID, endpoint string) (bool, error) { + if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(userID) != nil || !utf8.ValidString(endpoint) || len(endpoint) < 1 || len(endpoint) > 2048 { + return false, errors.New("push subscription deletion is invalid") + } + digest := sha256.Sum256([]byte(endpoint)) + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return false, errors.New("delete push subscription") + } + defer tx.Rollback() + var endpointID string + err = tx.QueryRowContext(ctx, `SELECT e.id FROM push_subscriptions s JOIN push_endpoints e ON e.id=s.endpoint_id WHERE s.organization_id=? AND s.user_id=? AND e.endpoint_digest=?`, organizationID, userID, digest[:]).Scan(&endpointID) + if errors.Is(err, sql.ErrNoRows) { + return false, errors.New("push subscription not found") + } + if err != nil { + return false, errors.New("delete push subscription") + } + if _, err = tx.ExecContext(ctx, `DELETE FROM push_subscriptions WHERE organization_id=? AND user_id=? AND endpoint_id=?`, organizationID, userID, endpointID); err != nil { + return false, errors.New("delete push subscription") + } + var remaining int + if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM push_subscriptions WHERE endpoint_id=?`, endpointID).Scan(&remaining); err != nil { + return false, errors.New("delete push subscription") + } + if remaining == 0 { + if _, err = tx.ExecContext(ctx, `DELETE FROM push_endpoints WHERE id=?`, endpointID); err != nil { + return false, errors.New("delete push subscription") + } + } + if err = tx.Commit(); err != nil { + return false, errors.New("delete push subscription") + } + return remaining > 0, nil +} + +func (s *Store) PushSubscription(ctx context.Context, organizationID, id string) (PushSubscription, error) { + if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(id) != nil { + return PushSubscription{}, errors.New("push subscription identity is invalid") + } + return scanPushSubscription(s.control.QueryRowContext(ctx, `SELECT s.organization_id,s.id,s.user_id,e.endpoint,e.p256dh,e.auth_secret,e.failure_count,s.created_at,e.updated_at,e.last_sent_at FROM push_subscriptions s JOIN push_endpoints e ON e.id=s.endpoint_id WHERE s.organization_id=? AND s.id=? AND e.active=1`, organizationID, id)) +} + +func (s *Store) PushSubscriptions(ctx context.Context, organizationID string) ([]PushSubscription, error) { + if model.ValidateSourceID(organizationID) != nil { + return nil, errors.New("organization identity is invalid") + } + rows, err := s.control.QueryContext(ctx, `SELECT s.organization_id,s.id,s.user_id,e.endpoint,e.p256dh,e.auth_secret,e.failure_count,s.created_at,e.updated_at,e.last_sent_at FROM push_subscriptions s JOIN push_endpoints e ON e.id=s.endpoint_id WHERE s.organization_id=? AND e.active=1 ORDER BY s.id LIMIT ?`, organizationID, MaxPushSubscriptionsPerPass) + if err != nil { + return nil, errors.New("list push subscriptions") + } + defer rows.Close() + var subscriptions []PushSubscription + for rows.Next() { + subscription, scanErr := scanPushSubscription(rows) + if scanErr != nil { + return nil, scanErr + } + subscriptions = append(subscriptions, subscription) + } + if err = rows.Err(); err != nil { + return nil, errors.New("list push subscriptions") + } + return subscriptions, nil +} + +func (s *Store) RecordPushResult(ctx context.Context, organizationID, id string, outcome string, now time.Time) error { + if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(id) != nil { + return errors.New("push subscription identity is invalid") + } + var result sql.Result + var err error + switch outcome { + case "sent": + result, err = s.control.ExecContext(ctx, `UPDATE push_endpoints SET failure_count=0,last_sent_at=?,updated_at=? WHERE id=(SELECT endpoint_id FROM push_subscriptions WHERE organization_id=? AND id=?) AND active=1`, now.UTC().Format(time.RFC3339Nano), now.UTC().Format(time.RFC3339Nano), organizationID, id) + case "gone": + result, err = s.control.ExecContext(ctx, `UPDATE push_endpoints SET active=0,updated_at=? WHERE id=(SELECT endpoint_id FROM push_subscriptions WHERE organization_id=? AND id=?) AND active=1`, now.UTC().Format(time.RFC3339Nano), organizationID, id) + case "failed": + result, err = s.control.ExecContext(ctx, `UPDATE push_endpoints SET failure_count=failure_count+1,active=CASE WHEN failure_count+1>=5 THEN 0 ELSE 1 END,updated_at=? WHERE id=(SELECT endpoint_id FROM push_subscriptions WHERE organization_id=? AND id=?) AND active=1`, now.UTC().Format(time.RFC3339Nano), organizationID, id) + default: + return errors.New("push delivery outcome is invalid") + } + if err != nil { + return errors.New("record push delivery result") + } + if changed, _ := result.RowsAffected(); changed != 1 { + return errors.New("push subscription not found") + } + return nil +} + +func scanPushSubscription(row rowScanner) (PushSubscription, error) { + var subscription PushSubscription + var createdAt, updatedAt string + var lastSent sql.NullString + if err := row.Scan(&subscription.OrganizationID, &subscription.ID, &subscription.UserID, &subscription.Endpoint, &subscription.P256DH, &subscription.Auth, &subscription.FailureCount, &createdAt, &updatedAt, &lastSent); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return PushSubscription{}, errors.New("push subscription not found") + } + return PushSubscription{}, errors.New("read push subscription") + } + var err error + if subscription.CreatedAt, err = time.Parse(time.RFC3339Nano, createdAt); err != nil { + return PushSubscription{}, errors.New("read push subscription") + } + if subscription.UpdatedAt, err = time.Parse(time.RFC3339Nano, updatedAt); err != nil { + return PushSubscription{}, errors.New("read push subscription") + } + if lastSent.Valid { + parsed, parseErr := time.Parse(time.RFC3339Nano, lastSent.String) + if parseErr != nil { + return PushSubscription{}, errors.New("read push subscription") + } + subscription.LastSentAt = &parsed + } + return subscription, nil +} + +func validatePushSubscription(input PushSubscriptionInput) error { + if model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.UserID) != nil { + return errors.New("push subscription scope is invalid") + } + if !utf8.ValidString(input.Endpoint) || len(input.Endpoint) < 1 || len(input.Endpoint) > 2048 || len(input.P256DH) != 65 || len(input.Auth) != 16 { + return errors.New("push subscription material is invalid") + } + return nil +} diff --git a/internal/storage/push_test.go b/internal/storage/push_test.go new file mode 100644 index 0000000..ec7d3e7 --- /dev/null +++ b/internal/storage/push_test.go @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "crypto/rand" + "testing" + "time" +) + +func TestPushSubscriptionLifecycleAndOwnership(t *testing.T) { + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC) + input := pushInput("organization-a", "user-a", "https://push.example.test/send/one") + created, err := store.SavePushSubscription(context.Background(), input, now) + if err != nil { + t.Fatal(err) + } + if created.OrganizationID != input.OrganizationID || created.UserID != input.UserID || created.Endpoint != input.Endpoint || created.FailureCount != 0 { + t.Fatalf("created=%+v", created) + } + input.P256DH[10] ^= 0xff + updated, err := store.SavePushSubscription(context.Background(), input, now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if updated.ID != created.ID || updated.P256DH[10] != input.P256DH[10] { + t.Fatalf("updated=%+v", updated) + } + claimed := input + claimed.UserID = "user-b" + if _, err = store.SavePushSubscription(context.Background(), claimed, now); err == nil { + t.Fatal("another user claimed an existing endpoint") + } + if _, err = store.DeletePushSubscription(context.Background(), input.OrganizationID, "user-b", input.Endpoint); err == nil { + t.Fatal("another user deleted an existing endpoint") + } + otherOrganization := input + otherOrganization.OrganizationID = "organization-b" + other, err := store.SavePushSubscription(context.Background(), otherOrganization, now.Add(2*time.Minute)) + if err != nil || other.ID == created.ID { + t.Fatalf("other organization=%+v err=%v", other, err) + } + for _, organizationID := range []string{input.OrganizationID, otherOrganization.OrganizationID} { + if subscribed, statusErr := store.HasPushSubscription(context.Background(), organizationID, input.UserID, input.Endpoint); statusErr != nil || !subscribed { + t.Fatalf("organization=%s subscribed=%t err=%v", organizationID, subscribed, statusErr) + } + } + remaining, err := store.DeletePushSubscription(context.Background(), input.OrganizationID, input.UserID, input.Endpoint) + if err != nil || !remaining { + t.Fatal(err) + } + if subscriptions, listErr := store.PushSubscriptions(context.Background(), input.OrganizationID); listErr != nil || len(subscriptions) != 0 { + t.Fatalf("subscriptions=%+v err=%v", subscriptions, listErr) + } + remaining, err = store.DeletePushSubscription(context.Background(), otherOrganization.OrganizationID, input.UserID, input.Endpoint) + if err != nil || remaining { + t.Fatalf("remaining=%t err=%v", remaining, err) + } +} + +func TestPushDeliveryResultsAreBounded(t *testing.T) { + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC) + created, err := store.SavePushSubscription(context.Background(), pushInput("organization-a", "user-a", "https://push.example.test/send/result"), now) + if err != nil { + t.Fatal(err) + } + for attempt := 1; attempt <= 4; attempt++ { + if err = store.RecordPushResult(context.Background(), created.OrganizationID, created.ID, "failed", now.Add(time.Duration(attempt)*time.Minute)); err != nil { + t.Fatal(err) + } + current, currentErr := store.PushSubscription(context.Background(), created.OrganizationID, created.ID) + if currentErr != nil || current.FailureCount != attempt { + t.Fatalf("attempt=%d current=%+v err=%v", attempt, current, currentErr) + } + } + if err = store.RecordPushResult(context.Background(), created.OrganizationID, created.ID, "sent", now.Add(5*time.Minute)); err != nil { + t.Fatal(err) + } + current, err := store.PushSubscription(context.Background(), created.OrganizationID, created.ID) + if err != nil || current.FailureCount != 0 || current.LastSentAt == nil { + t.Fatalf("sent current=%+v err=%v", current, err) + } + for attempt := 0; attempt < 5; attempt++ { + if err = store.RecordPushResult(context.Background(), created.OrganizationID, created.ID, "failed", now.Add(time.Duration(6+attempt)*time.Minute)); err != nil { + t.Fatal(err) + } + } + if _, err = store.PushSubscription(context.Background(), created.OrganizationID, created.ID); err == nil { + t.Fatal("five delivery failures did not disable the subscription") + } + if subscriptions, listErr := store.PushSubscriptions(context.Background(), created.OrganizationID); listErr != nil || len(subscriptions) != 0 { + t.Fatalf("subscriptions=%+v err=%v", subscriptions, listErr) + } +} + +func TestPushSubscriptionLimit(t *testing.T) { + store := testStore(t) + defer store.Close() + var err error + for index := 0; index < MaxPushSubscriptionsPerUser; index++ { + input := pushInput("organization-a", "user-a", "https://push.example.test/send/limit"+string(rune('a'+index))) + if _, err = store.SavePushSubscription(context.Background(), input, time.Now()); err != nil { + t.Fatal(err) + } + } + if _, err = store.SavePushSubscription(context.Background(), pushInput("organization-a", "user-a", "https://push.example.test/send/overflow"), time.Now()); err == nil { + t.Fatal("subscription limit was not enforced") + } +} + +func TestControlSchemaFiveMigratesToPushSchema(t *testing.T) { + database := openSchemaDatabase(t, 5) + defer database.Close() + if err := migrateControl(database); err != nil { + t.Fatal(err) + } + var version int + if err := database.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != controlSchema { + t.Fatalf("version=%d err=%v", version, err) + } + for _, name := range []string{"push_endpoints", "push_subscriptions"} { + var table int + if err := database.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, name).Scan(&table); err != nil || table != 1 { + t.Fatalf("table=%s count=%d err=%v", name, table, err) + } + } +} + +func pushInput(organizationID, userID, endpoint string) PushSubscriptionInput { + key := make([]byte, 65) + auth := make([]byte, 16) + _, _ = rand.Read(key) + _, _ = rand.Read(auth) + key[0] = 4 + return PushSubscriptionInput{OrganizationID: organizationID, UserID: userID, Endpoint: endpoint, P256DH: key, Auth: auth} +} diff --git a/internal/storage/query.go b/internal/storage/query.go new file mode 100644 index 0000000..f2cc95b --- /dev/null +++ b/internal/storage/query.go @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" +) + +type projectedRecord struct { + projectID, environmentID, serviceID string + sourceID, streamID string + sequence uint64 + recordIndex int + signal model.Signal + timestamp time.Time + name, severity, body string + value *float64 + traceID, spanID, correlationID string + attributes map[string]string +} + +// Query executes a validated typed AST against one organization projection. +// Authorization scope is supplied by the server, never by stored telemetry. +func (s *Store) Query(ctx context.Context, ast query.AST, scope query.Scope, budget query.Budget, now time.Time) (query.Result, error) { + if now.IsZero() { + return query.Result{}, errors.New("query time is required") + } + registry, activeVersion, err := s.ActiveDescriptors(ctx, scope.OrganizationID) + if err != nil { + return query.Result{}, err + } + coldSegments, coldEstimate, err := s.coldSegmentsForQuery(ctx, ast, scope, now) + if err != nil { + return query.Result{}, err + } + useMetricRollups := metricRollupQueryEligible(ast, registry) && len(coldSegments) == 0 + useLogRollups := indexedLogCountSummaryEligible(ast) && len(coldSegments) == 0 + var estimated int64 + if useMetricRollups { + estimated, err = s.estimateMetricRollupBytes(ctx, scope, ast, now) + } else if useLogRollups { + estimated, err = s.estimateLogRollupBytes(ctx, scope, ast, now) + } else { + estimated, err = s.EstimateOrganizationBytes(scope.OrganizationID) + if err == nil { + // Record queries whose predicates are fully pushed into SQLite stop + // after limit+1 rows. Their logical scan is therefore bounded by + // the existing result-memory guard, not by the size of every signal + // and index in the organization's projection file. Keep whole-file + // planning for summaries, alternate sorts, regular expressions, and + // cold-segment reads; execution continues to enforce the exact scan, + // row, memory, and duration budgets in every case. + if len(coldSegments) == 0 && projectionRowLimit(ast) > 0 && estimated > budget.MaxMemoryBytes { + estimated = budget.MaxMemoryBytes + } + if coldEstimate > math.MaxInt64-estimated { + return query.Result{}, errors.New("query scan estimate overflow") + } + estimated += coldEstimate + } + } + if err != nil { + return query.Result{}, err + } + explain, err := query.Plan(ast, scope, registry, estimated, budget) + if err != nil { + return query.Result{}, err + } + columns, err := resultColumns(ast, registry) + if err != nil { + return query.Result{}, err + } + result := query.Result{Version: query.ResultVersion, Explain: explain, Columns: columns, Rows: []query.Row{}} + if len(coldSegments) > 0 && !useMetricRollups { + for _, source := range append([]string(nil), result.Explain.ProjectedSources...) { + result.Explain.ProjectedSources = append(result.Explain.ProjectedSources, source+"/cold:raw") + } + } + path := filepath.Join(s.root, "organizations", scope.OrganizationID, "projection.sqlite") + info, err := os.Lstat(path) + projectionExists := err == nil + if (err != nil && !errors.Is(err, os.ErrNotExist)) || (projectionExists && (!info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0)) { + return query.Result{}, errors.New("organization projection is unavailable") + } + if useMetricRollups { + if !projectionExists { + return result, nil + } + for index := range result.Explain.ProjectedSources { + result.Explain.ProjectedSources[index] += "/rollup:5m" + } + return s.queryMetricRollups(ctx, path, ast, scope, registry, budget, now, result) + } + if projectionExists && useLogRollups { + for index := range result.Explain.ProjectedSources { + result.Explain.ProjectedSources[index] += "/rollup:http-status-route:5m" + } + return s.queryIndexedLogCountSummary(ctx, path, ast, scope, budget, now, result) + } + + runContext, cancel := context.WithTimeout(ctx, budget.MaxDuration) + defer cancel() + started := time.Now() + var records []projectedRecord + var memoryBytes int64 + earlyLimit := ast.Summary == nil && (ast.Sort == nil || query.CanonicalField(ast.Sort.Field) == "timestamp" && ast.Sort.Descending) + if projectionExists { + dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String() + db, openErr := sql.Open("sqlite", dsn) + if openErr != nil { + return query.Result{}, errors.New("open organization projection") + } + defer db.Close() + db.SetMaxOpenConns(1) + statement, arguments, selectionErr := projectionSelection(ast, scope, registry, activeVersion, now) + if selectionErr != nil { + return query.Result{}, selectionErr + } + rows, queryErr := db.QueryContext(runContext, statement, arguments...) + if queryErr != nil { + return query.Result{}, queryExecutionError(runContext, queryErr) + } + for rows.Next() { + record, readBytes, scanErr := scanProjected(rows) + if scanErr != nil { + _ = rows.Close() + if runContext.Err() != nil { + return query.Result{}, query.ErrBudgetExceeded + } + return query.Result{}, errors.New("read organization projection") + } + if readBytes > budget.MaxScannedBytes-result.Stats.ScannedBytes { + _ = rows.Close() + return query.Result{}, query.ErrBudgetExceeded + } + if result.Stats.ScannedRows == math.MaxInt64 { + _ = rows.Close() + return query.Result{}, query.ErrBudgetExceeded + } + result.Stats.ScannedRows++ + result.Stats.ScannedBytes += readBytes + matched, matchErr := matchesRecord(record, ast, registry) + if matchErr != nil { + _ = rows.Close() + return query.Result{}, matchErr + } + if !matched { + continue + } + if result.Stats.MatchedRows == math.MaxInt64 { + _ = rows.Close() + return query.Result{}, query.ErrBudgetExceeded + } + result.Stats.MatchedRows++ + if readBytes+256 > budget.MaxMemoryBytes-memoryBytes { + _ = rows.Close() + return query.Result{}, query.ErrBudgetExceeded + } + memoryBytes += readBytes + 256 + records = append(records, record) + if earlyLimit && len(records) > ast.Limit { + result.Stats.Truncated = true + break + } + if err = runContext.Err(); err != nil { + _ = rows.Close() + return query.Result{}, query.ErrBudgetExceeded + } + } + if err = rows.Err(); err != nil { + _ = rows.Close() + return query.Result{}, queryExecutionError(runContext, err) + } + if err = rows.Close(); err != nil { + return query.Result{}, errors.New("close organization projection query") + } + } + if !(earlyLimit && result.Stats.Truncated) { + records, memoryBytes, err = s.appendRawQuerySegments(runContext, coldSegments, ast, registry, budget, now, &result, records, memoryBytes) + if err != nil { + return query.Result{}, err + } + } + if len(coldSegments) > 0 { + sortRawQueryRecords(records) + } + return finishQueryResult(result, records, ast, columns, registry, memoryBytes, budget.MaxMemoryBytes, started) +} + +func (s *Store) appendRawQuerySegments(ctx context.Context, segments []rawQuerySegment, ast query.AST, registry query.Registry, budget query.Budget, now time.Time, result *query.Result, records []projectedRecord, memoryBytes int64) ([]projectedRecord, int64, error) { + for _, segment := range segments { + if segment.uncompressedBytes > budget.MaxMemoryBytes || segment.uncompressedBytes > budget.MaxScannedBytes-result.Stats.ScannedBytes { + return nil, 0, query.ErrBudgetExceeded + } + batch, err := s.segments.Read(segment.path, segment.digest) + if err != nil { + return nil, 0, errors.New("read raw query segment") + } + if batch.SourceID != segment.sourceID || batch.StreamID != segment.streamID || batch.Sequence != segment.sequence || batch.Signal != ast.Signal || batch.Validate(batch.ObservedAt) != nil || validateMetricRollupCardinality(batch) != nil { + return nil, 0, errors.New("raw query segment is invalid") + } + first, last := observationRange(batch) + if !first.Equal(segment.firstObservedAt) || !last.Equal(segment.lastObservedAt) { + return nil, 0, errors.New("raw query segment range does not match its catalog") + } + result.Stats.ScannedBytes += segment.uncompressedBytes + for index := range batch.Records { + if result.Stats.ScannedRows == math.MaxInt64 { + return nil, 0, query.ErrBudgetExceeded + } + result.Stats.ScannedRows++ + record := rawRecord(segment, batch, index) + if ast.Window > 0 && record.timestamp.Before(now.UTC().Add(-ast.Window)) { + continue + } + matched, matchErr := matchesRecord(record, ast, registry) + if matchErr != nil { + return nil, 0, matchErr + } + if !matched { + continue + } + if result.Stats.MatchedRows == math.MaxInt64 { + return nil, 0, query.ErrBudgetExceeded + } + result.Stats.MatchedRows++ + recordMemory := rawRecordMemory(record) + if recordMemory > budget.MaxMemoryBytes-memoryBytes { + return nil, 0, query.ErrBudgetExceeded + } + memoryBytes += recordMemory + records = append(records, record) + if ctx.Err() != nil { + return nil, 0, query.ErrBudgetExceeded + } + } + } + return records, memoryBytes, nil +} + +func sortRawQueryRecords(records []projectedRecord) { + sort.SliceStable(records, func(left, right int) bool { + if records[left].timestamp.Equal(records[right].timestamp) { + if records[left].sourceID == records[right].sourceID { + if records[left].streamID == records[right].streamID { + if records[left].sequence == records[right].sequence { + return records[left].recordIndex > records[right].recordIndex + } + return records[left].sequence > records[right].sequence + } + return records[left].streamID < records[right].streamID + } + return records[left].sourceID < records[right].sourceID + } + return records[left].timestamp.After(records[right].timestamp) + }) +} + +func finishQueryResult(result query.Result, records []projectedRecord, ast query.AST, columns []query.Column, registry query.Registry, memoryBytes, maxMemoryBytes int64, started time.Time) (query.Result, error) { + var err error + if ast.Summary == nil { + result.Rows, err = materializeRecords(records, columns) + } else { + result.Rows, memoryBytes, err = summarizeRecords(records, ast, columns, registry, memoryBytes, maxMemoryBytes) + } + if err != nil { + return query.Result{}, err + } + if ast.Sort != nil { + if err = sortRows(result.Rows, columns, ast.Sort.Field, ast.Sort.Descending); err != nil { + return query.Result{}, err + } + } + if len(result.Rows) > ast.Limit { + result.Rows = result.Rows[:ast.Limit] + result.Stats.Truncated = true + } + result.Stats.DurationNS = time.Since(started).Nanoseconds() + return result, nil +} + +func projectionSelection(ast query.AST, scope query.Scope, registry query.Registry, activeVersion int, now time.Time) (string, []any, error) { + statement := `SELECT o.project_id,o.environment_id,o.service_id,o.source_id,o.stream_id,o.sequence,o.record_index,o.signal,o.timestamp,o.name,o.severity,o.body,o.value,o.trace_id,o.span_id,o.correlation_id,o.attributes_json FROM observations o` + if index := summaryProjectionIndex(ast); index != "" { + statement += " INDEXED BY " + index + } + var joins, predicates []string + var joinArguments, predicateArguments []any + indexedJoin := 0 + for _, filter := range ast.Filters { + if filter.Op == "=~" { + continue + } + field := query.CanonicalField(filter.Field) + expression, expressionArguments := sqlFieldExpression(field) + descriptor, unknown := query.ResolveDescriptor(ast.Signal, field, registry) + value, err := typedFilterValue(filter.Value, descriptor.Type) + if err != nil { + return "", nil, err + } + operator := map[string]string{"==": "=", "!=": "!=", ">": ">", ">=": ">=", "<": "<", "<=": "<="}[filter.Op] + if operator == "" { + return "", nil, query.ErrTypeMismatch + } + _, builtin := query.BuiltinDescriptor(ast.Signal, field) + if !unknown && !builtin && descriptor.Index != schema.IndexNone && activeVersion > 1 { + table, tableErr := projectionIndexTable(activeVersion) + if tableErr != nil { + return "", nil, tableErr + } + column := "value_text" + if descriptor.Type == schema.TypeInteger || descriptor.Type == schema.TypeFloat || descriptor.Type == schema.TypeDuration { + column = "value_number" + } + if descriptor.Type == schema.TypeTime { + parsed, parseErr := time.Parse(time.RFC3339Nano, filter.Value) + if parseErr != nil { + return "", nil, query.ErrTypeMismatch + } + value = parsed.UTC().Format(indexedTimeFormat) + } + alias := fmt.Sprintf("idx%d", indexedJoin) + indexedJoin++ + joins = append(joins, " JOIN "+table+" "+alias+" ON "+alias+".signal=o.signal AND "+alias+".field=? AND "+alias+".source_id=o.source_id AND "+alias+".stream_id=o.stream_id AND "+alias+".sequence=o.sequence AND "+alias+".record_index=o.record_index AND "+alias+"."+column+operator+"?") + joinArguments = append(joinArguments, descriptor.Field, value) + } else { + predicates = append(predicates, expression+operator+"?") + predicateArguments = append(predicateArguments, expressionArguments...) + predicateArguments = append(predicateArguments, value) + } + } + statement += strings.Join(joins, "") + ` WHERE o.organization_id=? AND o.signal=?` + arguments := append(joinArguments, scope.OrganizationID, string(ast.Signal)) + for _, selected := range []struct { + column, value string + }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} { + if selected.value != "" { + statement += " AND o." + selected.column + "=?" + arguments = append(arguments, selected.value) + } + } + if ast.Window > 0 { + statement += " AND o.timestamp>=?" + arguments = append(arguments, now.UTC().Add(-ast.Window).Format(time.RFC3339Nano)) + } + for _, predicate := range predicates { + statement += " AND " + predicate + } + arguments = append(arguments, predicateArguments...) + if ast.Summary == nil { + statement += ` ORDER BY o.timestamp DESC,o.source_id,o.stream_id,o.sequence DESC,o.record_index DESC` + } + if limit := projectionRowLimit(ast); limit > 0 { + statement += ` LIMIT ?` + arguments = append(arguments, limit) + } + return statement, arguments, nil +} + +// projectionRowLimit returns the number of rows SQLite may return for a +// record query whose final result can be decided in timestamp order. The +// extra row preserves the result's truncated signal. Regular expressions are +// evaluated in Go, so they cannot safely use a pre-match SQL limit. +func projectionRowLimit(ast query.AST) int { + if ast.Summary != nil || ast.Sort != nil && (query.CanonicalField(ast.Sort.Field) != "timestamp" || !ast.Sort.Descending) { + return 0 + } + for _, filter := range ast.Filters { + if filter.Op == "=~" { + return 0 + } + } + return ast.Limit + 1 +} + +// summaryProjectionIndex keeps selective built-in filters on their reviewed +// projection index. Summary execution is independent of input order and later +// sorts its deterministic result rows, so it must not trade the selective +// filter path for the record-level timestamp order used by ordinary queries. +func summaryProjectionIndex(ast query.AST) string { + if ast.Summary == nil { + return "" + } + for _, filter := range ast.Filters { + if filter.Op == "=~" || filter.Op == "!=" { + continue + } + switch query.CanonicalField(filter.Field) { + case "http.status_code": + return "observations_http_status" + case "http.route": + return "observations_http_route" + case "duration_ns": + return "observations_duration" + case "name": + return "observations_name" + case "severity": + return "observations_severity" + case "value": + return "observations_value" + } + } + return "" +} + +func sqlFieldExpression(field string) (string, []any) { + switch query.CanonicalField(field) { + case "project.id": + return "o.project_id", nil + case "environment.id": + return "o.environment_id", nil + case "service.id": + return "o.service_id", nil + case "source.id": + return "o.source_id", nil + case "stream.id": + return "o.stream_id", nil + case "timestamp", "name", "severity", "body", "value", "trace_id", "span_id", "correlation_id": + return "o." + query.CanonicalField(field), nil + case "http.route": + return `json_extract(o.attributes_json,'$."http.route"')`, nil + case "http.status_code": + return `CAST(json_extract(o.attributes_json,'$."http.status_code"') AS INTEGER)`, nil + case "duration_ns": + return `CAST(json_extract(o.attributes_json,'$."duration_ns"') AS REAL)`, nil + default: + return "json_extract(o.attributes_json,?)", []any{`$."` + query.CanonicalField(field) + `"`} + } +} + +func typedFilterValue(value string, valueType schema.Type) (any, error) { + switch valueType { + case schema.TypeInteger: + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, query.ErrTypeMismatch + } + return parsed, nil + case schema.TypeFloat, schema.TypeDuration: + parsed, err := strconv.ParseFloat(value, 64) + if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) { + return nil, query.ErrTypeMismatch + } + return parsed, nil + case schema.TypeBoolean: + parsed, err := strconv.ParseBool(value) + if err != nil { + return nil, query.ErrTypeMismatch + } + return strconv.FormatBool(parsed), nil + case schema.TypeTime: + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return nil, query.ErrTypeMismatch + } + return parsed.UTC().Format(time.RFC3339Nano), nil + default: + return value, nil + } +} + +func scanProjected(rows *sql.Rows) (projectedRecord, int64, error) { + var record projectedRecord + var timestamp, signal, attributes string + var severity, body, traceID, spanID, correlationID sql.NullString + var value sql.NullFloat64 + err := rows.Scan(&record.projectID, &record.environmentID, &record.serviceID, &record.sourceID, &record.streamID, &record.sequence, &record.recordIndex, &signal, ×tamp, &record.name, &severity, &body, &value, &traceID, &spanID, &correlationID, &attributes) + if err != nil { + return projectedRecord{}, 0, err + } + record.signal = model.Signal(signal) + record.timestamp, err = time.Parse(time.RFC3339Nano, timestamp) + if err != nil { + return projectedRecord{}, 0, err + } + record.severity, record.body = severity.String, body.String + record.traceID, record.spanID, record.correlationID = traceID.String, spanID.String, correlationID.String + if value.Valid { + record.value = &value.Float64 + } + if err = json.Unmarshal([]byte(attributes), &record.attributes); err != nil { + return projectedRecord{}, 0, errors.New("invalid projected attributes") + } + if record.attributes == nil { + record.attributes = map[string]string{} + } + readBytes := int64(len(record.projectID) + len(record.environmentID) + len(record.serviceID) + len(record.sourceID) + len(record.streamID) + len(signal) + len(timestamp) + len(record.name) + len(record.severity) + len(record.body) + len(record.traceID) + len(record.spanID) + len(record.correlationID) + len(attributes) + 64) + return record, readBytes, nil +} + +func matchesRecord(record projectedRecord, ast query.AST, registry query.Registry) (bool, error) { + return query.MatchesFilters(ast, registry, record.field) +} + +func (record projectedRecord) field(field string) (string, bool) { + switch query.CanonicalField(field) { + case "project.id": + return record.projectID, record.projectID != "" + case "environment.id": + return record.environmentID, record.environmentID != "" + case "service.id": + return record.serviceID, record.serviceID != "" + case "source.id": + return record.sourceID, record.sourceID != "" + case "stream.id": + return record.streamID, record.streamID != "" + case "timestamp": + return record.timestamp.UTC().Format(time.RFC3339Nano), true + case "name": + return record.name, record.name != "" + case "severity": + return record.severity, record.severity != "" + case "body": + return record.body, record.body != "" + case "value": + if record.value == nil { + return "", false + } + return strconv.FormatFloat(*record.value, 'g', -1, 64), true + case "trace_id": + return record.traceID, record.traceID != "" + case "span_id": + return record.spanID, record.spanID != "" + case "correlation_id": + return record.correlationID, record.correlationID != "" + default: + value, ok := record.attributes[query.CanonicalField(field)] + return value, ok + } +} + +func resultColumns(ast query.AST, registry query.Registry) ([]query.Column, error) { + if ast.Summary != nil { + var columns []query.Column + if ast.Bucket > 0 { + columns = append(columns, query.Column{Field: "window_start", Type: schema.TypeTime, Unit: "s"}) + } + for _, field := range ast.Summary.GroupBy { + canonical := query.CanonicalField(field) + descriptor, _ := query.ResolveDescriptor(ast.Signal, canonical, registry) + columns = append(columns, query.Column{Field: canonical, Type: descriptor.Type, Unit: descriptor.Unit}) + } + for _, aggregate := range ast.Summary.Aggregates { + valueType := schema.TypeFloat + if aggregate.Function == "count" { + valueType = schema.TypeInteger + } + unit := "" + if aggregate.Field != "" { + descriptor, _ := query.ResolveDescriptor(ast.Signal, aggregate.Field, registry) + unit = descriptor.Unit + } + columns = append(columns, query.Column{Field: aggregate.Alias, Type: valueType, Unit: unit}) + } + return columns, nil + } + fields := []string{"timestamp", "service.id", "name"} + switch ast.Signal { + case model.SignalLogs: + fields = append(fields, "severity") + case model.SignalMetrics: + fields = append(fields, "value") + case model.SignalTraces: + fields = append(fields, "trace_id", "span_id") + case model.SignalDeployments: + fields = append(fields, "correlation_id") + } + fields = append(fields, query.ReferencedFields(ast)...) + seen := map[string]bool{} + columns := make([]query.Column, 0, len(fields)) + for _, field := range fields { + canonical := query.CanonicalField(field) + if seen[canonical] { + continue + } + seen[canonical] = true + descriptor, _ := query.ResolveDescriptor(ast.Signal, canonical, registry) + columns = append(columns, query.Column{Field: canonical, Type: descriptor.Type, Unit: descriptor.Unit}) + } + return columns, nil +} + +func materializeRecords(records []projectedRecord, columns []query.Column) ([]query.Row, error) { + result := make([]query.Row, 0, len(records)) + for _, record := range records { + row := query.Row{Values: make([]*string, len(columns))} + for index, column := range columns { + if value, ok := record.field(column.Field); ok { + if canonical, valid := canonicalResultValue(value, column.Type); valid { + row.Values[index] = stringPointer(canonical) + } + } + } + result = append(result, row) + } + return result, nil +} + +type aggregateState struct { + function string + count int64 + sum, min, max float64 + values []float64 +} + +type summaryGroup struct { + key string + values []*string + aggregates []aggregateState +} + +func summarizeRecords(records []projectedRecord, ast query.AST, columns []query.Column, registry query.Registry, memoryBytes, maxMemory int64) ([]query.Row, int64, error) { + groups := map[string]*summaryGroup{} + for _, record := range records { + var values []*string + if ast.Bucket > 0 { + bucket := record.timestamp.UTC().Truncate(ast.Bucket).Format(time.RFC3339Nano) + values = append(values, stringPointer(bucket)) + } + for _, field := range ast.Summary.GroupBy { + value, ok := record.field(field) + if ok { + column := columns[len(values)] + if canonical, valid := canonicalResultValue(value, column.Type); valid { + values = append(values, stringPointer(canonical)) + } else { + values = append(values, nil) + } + } else { + values = append(values, nil) + } + } + key := groupKey(values) + group := groups[key] + if group == nil { + group = &summaryGroup{key: key, values: values, aggregates: make([]aggregateState, len(ast.Summary.Aggregates))} + for index, aggregate := range ast.Summary.Aggregates { + group.aggregates[index].function = aggregate.Function + } + groups[key] = group + memoryBytes += int64(len(key) + len(values)*16 + len(group.aggregates)*64) + } + for index, aggregate := range ast.Summary.Aggregates { + state := &group.aggregates[index] + if aggregate.Function == "count" { + state.count++ + continue + } + value, ok := record.field(aggregate.Field) + if !ok { + continue + } + descriptor, _ := query.ResolveDescriptor(ast.Signal, aggregate.Field, registry) + if descriptor.Type != schema.TypeInteger && descriptor.Type != schema.TypeFloat && descriptor.Type != schema.TypeDuration { + return nil, memoryBytes, query.ErrTypeMismatch + } + number, err := strconv.ParseFloat(value, 64) + if err != nil || math.IsNaN(number) || math.IsInf(number, 0) { + continue + } + if state.count == 0 { + state.min, state.max = number, number + } else { + state.min = math.Min(state.min, number) + state.max = math.Max(state.max, number) + } + state.count++ + state.sum += number + if aggregate.Function == "p50" || aggregate.Function == "p95" || aggregate.Function == "p99" { + state.values = append(state.values, number) + memoryBytes += 8 + } + } + if memoryBytes > maxMemory { + return nil, memoryBytes, query.ErrBudgetExceeded + } + } + ordered := make([]*summaryGroup, 0, len(groups)) + for _, group := range groups { + ordered = append(ordered, group) + } + sort.Slice(ordered, func(i, j int) bool { return ordered[i].key < ordered[j].key }) + rows := make([]query.Row, 0, len(ordered)) + for _, group := range ordered { + row := query.Row{Values: append([]*string(nil), group.values...)} + for _, state := range group.aggregates { + value, ok := aggregateValue(state) + if ok { + row.Values = append(row.Values, stringPointer(value)) + } else { + row.Values = append(row.Values, nil) + } + } + if len(row.Values) != len(columns) { + return nil, memoryBytes, errors.New("summary result shape is invalid") + } + rows = append(rows, row) + } + return rows, memoryBytes, nil +} + +func aggregateValue(state aggregateState) (string, bool) { + if state.function == "count" { + return strconv.FormatInt(state.count, 10), true + } + if state.count == 0 { + return "", false + } + var value float64 + switch state.function { + case "min": + value = state.min + case "max": + value = state.max + case "sum": + value = state.sum + case "avg": + value = state.sum / float64(state.count) + case "p50", "p95", "p99": + sort.Float64s(state.values) + percentile := map[string]float64{"p50": .50, "p95": .95, "p99": .99}[state.function] + index := max(0, int(math.Ceil(percentile*float64(len(state.values))))-1) + value = state.values[index] + default: + return "", false + } + return strconv.FormatFloat(value, 'g', -1, 64), true +} + +func sortRows(rows []query.Row, columns []query.Column, field string, descending bool) error { + canonical := query.CanonicalField(field) + column := -1 + for index, candidate := range columns { + if candidate.Field == canonical || candidate.Field == field { + column = index + break + } + } + if column < 0 { + return errors.New("query sort field is unavailable") + } + sort.SliceStable(rows, func(i, j int) bool { + left, right := rows[i].Values[column], rows[j].Values[column] + if left == nil { + return false + } + if right == nil { + return true + } + comparison, leftValid, rightValid := compareTyped(*left, *right, columns[column].Type) + if !leftValid { + return false + } + if !rightValid { + return true + } + if descending { + return comparison > 0 + } + return comparison < 0 + }) + return nil +} + +func compareTyped(left, right string, valueType schema.Type) (int, bool, bool) { + if valueType == schema.TypeInteger || valueType == schema.TypeFloat || valueType == schema.TypeDuration { + leftNumber, leftErr := strconv.ParseFloat(left, 64) + rightNumber, rightErr := strconv.ParseFloat(right, 64) + leftValid := leftErr == nil && !math.IsNaN(leftNumber) && !math.IsInf(leftNumber, 0) + rightValid := rightErr == nil && !math.IsNaN(rightNumber) && !math.IsInf(rightNumber, 0) + return compareFloat(leftNumber, rightNumber), leftValid, rightValid + } + if valueType == schema.TypeTime { + leftTime, leftErr := time.Parse(time.RFC3339Nano, left) + rightTime, rightErr := time.Parse(time.RFC3339Nano, right) + return leftTime.Compare(rightTime), leftErr == nil, rightErr == nil + } + return strings.Compare(left, right), true, true +} + +func canonicalResultValue(value string, valueType schema.Type) (string, bool) { + switch valueType { + case schema.TypeInteger: + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return "", false + } + return strconv.FormatInt(parsed, 10), true + case schema.TypeFloat, schema.TypeDuration: + parsed, err := strconv.ParseFloat(value, 64) + if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) { + return "", false + } + return strconv.FormatFloat(parsed, 'g', -1, 64), true + case schema.TypeTime: + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return "", false + } + return parsed.UTC().Format(time.RFC3339Nano), true + case schema.TypeBoolean: + parsed, err := strconv.ParseBool(value) + if err != nil { + return "", false + } + return strconv.FormatBool(parsed), true + default: + return value, true + } +} + +func compareFloat(left, right float64) int { + if left < right { + return -1 + } + if left > right { + return 1 + } + return 0 +} + +func groupKey(values []*string) string { + var builder strings.Builder + for _, value := range values { + if value == nil { + builder.WriteString("-1:") + continue + } + builder.WriteString(strconv.Itoa(len(*value))) + builder.WriteByte(':') + builder.WriteString(*value) + } + return builder.String() +} + +func stringPointer(value string) *string { + copy := value + return © +} + +func queryExecutionError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return query.ErrBudgetExceeded + } + return fmt.Errorf("query projection: %w", err) +} diff --git a/internal/storage/query_test.go b/internal/storage/query_test.go new file mode 100644 index 0000000..6dda9ab --- /dev/null +++ b/internal/storage/query_test.go @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +func TestQueryExecutesScopedTypedFiltersAndSorting(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 7, 0, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + records := []model.Observation{ + requestObservation(now.Add(-time.Minute), "/ok", 200, 50), + requestObservation(now.Add(-2*time.Minute), "/broken", 503, 300), + requestObservation(now.Add(-3*time.Minute), "/slow", 500, 200), + {Timestamp: now.Add(-4 * time.Minute), Name: "application.http.request", Attributes: map[string]string{"http.route": "/invalid", "http.status_code": "not-a-number", "duration_ns": "invalid"}}, + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: records} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + otherToken, err := store.CreateSource(ctx, "source-b", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-b"}) + if err != nil { + t.Fatal(err) + } + other := model.Batch{Version: model.BatchVersion, SourceID: "source-b", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{requestObservation(now, "/other", 599, 999)}} + if _, err = store.Ingest(ctx, otherToken, other, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + projectionPath := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite") + projectionBefore, err := os.ReadFile(projectionPath) + if err != nil { + t.Fatal(err) + } + infoBefore, err := os.Stat(projectionPath) + if err != nil { + t.Fatal(err) + } + + ast, err := query.Parse(`logs | where status >= 500 | window 1h | sort duration desc | limit 1`, 100) + if err != nil { + t.Fatal(err) + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}, testQueryBudget(), now) + if err != nil { + t.Fatal(err) + } + if len(result.Rows) != 1 || result.Stats.MatchedRows != 2 || !result.Stats.Truncated { + t.Fatalf("result=%+v", result) + } + duration := columnValue(t, result, 0, "duration_ns") + service := columnValue(t, result, 0, "service.id") + if duration != "300" || service != "service-a" { + t.Fatalf("duration=%q service=%q result=%+v", duration, service, result) + } + ast, err = query.Parse(`logs | sort duration desc | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + result, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a", ServiceID: "service-a"}, testQueryBudget(), now) + if err != nil { + t.Fatal(err) + } + for index, column := range result.Columns { + if column.Field == "duration_ns" && (len(result.Rows) != 4 || result.Rows[3].Values[index] != nil) { + t.Fatalf("invalid numeric value was not sorted last: %+v", result) + } + } + projectionAfter, err := os.ReadFile(projectionPath) + if err != nil { + t.Fatal(err) + } + infoAfter, err := os.Stat(projectionPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(projectionBefore, projectionAfter) || !infoBefore.ModTime().Equal(infoAfter.ModTime()) { + t.Fatal("query execution modified the organization projection") + } +} + +func TestQuerySummarizesThroughSharedBucketAST(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 7, 2, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + requestObservation(now.Add(-time.Minute), "/items", 200, 100), + requestObservation(now.Add(-2*time.Minute), "/items", 500, 300), + requestObservation(now.Add(-time.Minute), "/about", 200, 20), + }} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + ast, err := query.Parse(`logs | summarize count(), p95(duration) by route, window(5m) | sort count desc | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + if ast.Window != 0 || ast.Bucket != 5*time.Minute { + t.Fatalf("window=%s bucket=%s", ast.Window, ast.Bucket) + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now) + if err != nil { + t.Fatal(err) + } + if len(result.Rows) != 2 || columnValue(t, result, 0, "http.route") != "/items" || columnValue(t, result, 0, "count") != "2" || columnValue(t, result, 0, "p95_duration") != "300" { + t.Fatalf("result=%+v", result) + } + filtered, err := query.Parse(`logs | where status >= 500 | summarize count() by route, window(5m) | sort count desc | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + result, err = store.Query(ctx, filtered, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now) + if err != nil { + t.Fatal(err) + } + if len(result.Rows) != 1 || columnValue(t, result, 0, "http.route") != "/items" || columnValue(t, result, 0, "count") != "1" { + t.Fatalf("filtered result=%+v", result) + } +} + +func TestSummaryProjectionUsesSelectiveIndexWithoutRecordOrder(t *testing.T) { + now := time.Date(2026, 8, 17, 7, 2, 0, 0, time.UTC) + ast, err := query.Parse(`logs | where status >= 500 | window 24h | summarize count() by route, window(5m) | sort count desc | limit 50`, 100) + if err != nil { + t.Fatal(err) + } + statement, _, err := projectionSelection(ast, query.Scope{OrganizationID: "organization-a"}, nil, 1, now) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(statement, "FROM observations o INDEXED BY observations_http_status") { + t.Fatalf("summary did not pin the selective status index: %s", statement) + } + if strings.Contains(statement, " ORDER BY o.timestamp") { + t.Fatalf("summary retained unnecessary record ordering: %s", statement) + } + + ordinary, err := query.Parse(`logs | where status >= 500 | window 24h | limit 50`, 100) + if err != nil { + t.Fatal(err) + } + statement, arguments, err := projectionSelection(ordinary, query.Scope{OrganizationID: "organization-a"}, nil, 1, now) + if err != nil { + t.Fatal(err) + } + if strings.Contains(statement, " INDEXED BY ") || !strings.Contains(statement, " ORDER BY o.timestamp") || !strings.HasSuffix(statement, " LIMIT ?") || arguments[len(arguments)-1] != 51 { + t.Fatalf("ordinary record query changed its ordered plan: %s", statement) + } + + regularExpression, err := query.Parse(`logs | where route =~ "^/items" | window 24h | limit 50`, 100) + if err != nil { + t.Fatal(err) + } + statement, _, err = projectionSelection(regularExpression, query.Scope{OrganizationID: "organization-a"}, nil, 1, now) + if err != nil { + t.Fatal(err) + } + if strings.Contains(statement, " LIMIT ?") { + t.Fatalf("regular-expression query limited rows before Go evaluation: %s", statement) + } +} + +func TestQueryPlansBoundedRecordLimitIndependentlyOfProjectionSize(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 18, 12, 0, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + requestObservation(now.Add(-time.Minute), "/one", 200, 10), + requestObservation(now.Add(-2*time.Minute), "/two", 200, 20), + requestObservation(now.Add(-3*time.Minute), "/three", 200, 30), + }} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + projectionBytes, err := store.EstimateOrganizationBytes("organization-a") + if err != nil || projectionBytes < 4 { + t.Fatalf("projection bytes=%d err=%v", projectionBytes, err) + } + budget := testQueryBudget() + budget.MaxScannedBytes = projectionBytes - 1 + budget.MaxMemoryBytes = min(projectionBytes/2, 1<<20) + ast, err := query.Parse(`logs | window 1h | limit 1`, 100) + if err != nil { + t.Fatal(err) + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, budget, now) + if err != nil { + t.Fatal(err) + } + if len(result.Rows) != 1 || result.Stats.ScannedRows != 2 || !result.Stats.Truncated || result.Explain.EstimatedScanBytes != budget.MaxMemoryBytes { + t.Fatalf("result=%+v", result) + } + + regex, err := query.Parse(`logs | where route =~ "^/" | window 1h | limit 1`, 100) + if err != nil { + t.Fatal(err) + } + if _, err = store.Query(ctx, regex, query.Scope{OrganizationID: "organization-a"}, budget, now); err == nil || !strings.Contains(err.Error(), "estimated query scan exceeds budget") { + t.Fatalf("unbounded regex planning err=%v", err) + } +} + +func TestQueryEnforcesSensitiveAndExecutionBudgets(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 7, 0, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.http.request", Body: "private evidence"}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + ast, _ := query.Parse(`logs | where body == "private evidence" | limit 10`, 100) + if _, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now); !errors.Is(err, query.ErrSensitivePermissionRequired) { + t.Fatalf("sensitive err=%v", err) + } + ast, _ = query.Parse(`logs | limit 10`, 100) + budget := testQueryBudget() + budget.MaxMemoryBytes = 1 + if _, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a", Sensitive: true}, budget, now); !errors.Is(err, query.ErrBudgetExceeded) { + t.Fatalf("budget err=%v", err) + } +} + +func requestObservation(timestamp time.Time, route string, status int, duration int) model.Observation { + return model.Observation{Timestamp: timestamp, Name: "application.http.request", Attributes: map[string]string{"http.route": route, "http.status_code": strconv.Itoa(status), "duration_ns": strconv.Itoa(duration)}} +} + +func testQueryBudget() query.Budget { + return query.Budget{MaxDuration: 5 * time.Second, MaxRows: 100, MaxScannedBytes: 100 << 20, MaxMemoryBytes: 10 << 20} +} + +func columnValue(t *testing.T, result query.Result, row int, field string) string { + t.Helper() + for index, column := range result.Columns { + if column.Field == field { + if result.Rows[row].Values[index] == nil { + t.Fatalf("field %s is nil", field) + } + return *result.Rows[row].Values[index] + } + } + t.Fatalf("field %s is absent: %+v", field, result.Columns) + return "" +} diff --git a/internal/storage/raw_query_candidate.go b/internal/storage/raw_query_candidate.go new file mode 100644 index 0000000..1145ed5 --- /dev/null +++ b/internal/storage/raw_query_candidate.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "errors" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +// queryRawCandidate executes one bounded log query directly from the complete +// retained raw-segment catalogue. It is deliberately not a public Store or +// HTTP surface: the current projection path remains the production oracle +// while differential tests establish the adaptive read path's semantics. +// +// The organization lock prevents in-process retention from moving or deleting +// a selected object during this first proof. A future leased materialization +// needs an explicit catalogue snapshot/high-watermark rather than holding this +// lock across a potentially long historical scan. +func (s *Store) queryRawCandidate(ctx context.Context, ast query.AST, scope query.Scope, budget query.Budget, now time.Time) (query.Result, error) { + if now.IsZero() { + return query.Result{}, errors.New("query time is required") + } + if ast.Signal != model.SignalLogs { + return query.Result{}, errors.New("raw query candidate supports logs only") + } + lock := s.namedLock("organization:" + scope.OrganizationID) + lock.Lock() + defer lock.Unlock() + + registry, _, err := s.ActiveDescriptors(ctx, scope.OrganizationID) + if err != nil { + return query.Result{}, err + } + segments, estimated, err := s.allRawSegmentsForQuery(ctx, ast, scope, now) + if err != nil { + return query.Result{}, err + } + explain, err := query.Plan(ast, scope, registry, estimated, budget) + if err != nil { + return query.Result{}, err + } + for index := range explain.ProjectedSources { + explain.ProjectedSources[index] += "/raw:catalog" + } + columns, err := resultColumns(ast, registry) + if err != nil { + return query.Result{}, err + } + result := query.Result{Version: query.ResultVersion, Explain: explain, Columns: columns, Rows: []query.Row{}} + runContext, cancel := context.WithTimeout(ctx, budget.MaxDuration) + defer cancel() + started := time.Now() + records, memoryBytes, err := s.appendRawQuerySegments(runContext, segments, ast, registry, budget, now, &result, nil, 0) + if err != nil { + return query.Result{}, err + } + sortRawQueryRecords(records) + return finishQueryResult(result, records, ast, columns, registry, memoryBytes, budget.MaxMemoryBytes, started) +} diff --git a/internal/storage/raw_query_candidate_test.go b/internal/storage/raw_query_candidate_test.go new file mode 100644 index 0000000..4c3215a --- /dev/null +++ b/internal/storage/raw_query_candidate_test.go @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +func TestRawQueryCandidateMatchesProjectionOracle(t *testing.T) { + ctx := t.Context() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 18, 22, 0, 0, 0, time.UTC) + for _, source := range []struct { + id string + service string + batches []model.Batch + }{ + {id: "source-a", service: "service-a", batches: []model.Batch{ + {Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now.Add(-2 * time.Minute), Signal: model.SignalLogs, Records: []model.Observation{ + requestObservation(now.Add(-3*time.Minute), "/items", 503, 300), + requestObservation(now.Add(-4*time.Minute), "/items", 200, 80), + }}, + {Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 2, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + requestObservation(now.Add(-time.Minute), "/checkout", 500, 900), + {Timestamp: now.Add(-30 * time.Second), Name: "application.note", Body: "safe", Severity: "information"}, + }}, + }}, + {id: "source-b", service: "service-b", batches: []model.Batch{ + {Version: model.BatchVersion, SourceID: "source-b", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + requestObservation(now.Add(-90*time.Second), "/items", 502, 450), + }}, + }}, + } { + token, err := store.CreateSource(ctx, source.id, model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: source.service}) + if err != nil { + t.Fatal(err) + } + for _, batch := range source.batches { + if _, err = store.Ingest(ctx, token, batch, batch.ObservedAt); err != nil { + t.Fatal(err) + } + } + } + projectAll(t, store) + + queries := []struct { + text string + scope query.Scope + }{ + {`logs | where status >= 500 | window 1h | sort duration desc | limit 2`, query.Scope{OrganizationID: "organization-a"}}, + {`logs | where status >= 500 | summarize count(), p95(duration) by route | sort count desc | limit 10`, query.Scope{OrganizationID: "organization-a"}}, + {`logs | where route =~ "^/item" | window 1h | limit 10`, query.Scope{OrganizationID: "organization-a", ServiceID: "service-a"}}, + } + for _, candidate := range queries { + ast, err := query.Parse(candidate.text, 100) + if err != nil { + t.Fatalf("parse %q: %v", candidate.text, err) + } + projected, err := store.Query(ctx, ast, candidate.scope, testQueryBudget(), now) + if err != nil { + t.Fatalf("projected %q: %v", candidate.text, err) + } + raw, err := store.queryRawCandidate(ctx, ast, candidate.scope, testQueryBudget(), now) + if err != nil { + t.Fatalf("raw %q: %v", candidate.text, err) + } + if !reflect.DeepEqual(projected.Columns, raw.Columns) || !reflect.DeepEqual(projected.Rows, raw.Rows) || projected.Stats.Truncated != raw.Stats.Truncated || projected.Stats.MatchedRows != raw.Stats.MatchedRows { + t.Fatalf("query %q diverged\nprojected=%+v\nraw=%+v", candidate.text, projected, raw) + } + } +} + +func TestRawQueryCandidateReadsDurableUnprojectedBatchesWithoutWriting(t *testing.T) { + ctx := t.Context() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 18, 22, 15, 0, 0, time.UTC) + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{requestObservation(now, "/ready", 200, 25)}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + projectionPath := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite") + if _, err = os.Lstat(projectionPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("projection unexpectedly exists: %v", err) + } + ast, err := query.Parse(`logs | where route == "/ready" | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + result, err := store.queryRawCandidate(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now) + if err != nil || len(result.Rows) != 1 || columnValue(t, result, 0, "http.route") != "/ready" { + t.Fatalf("result=%+v err=%v", result, err) + } + if _, err = os.Lstat(projectionPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("raw query created a projection: %v", err) + } + var projected int + if err = store.control.QueryRow(`SELECT COUNT(projected_at) FROM segments`).Scan(&projected); err != nil || projected != 0 { + t.Fatalf("projected=%d err=%v", projected, err) + } + budget := testQueryBudget() + budget.MaxScannedBytes = 1 + if _, err = store.queryRawCandidate(ctx, ast, query.Scope{OrganizationID: "organization-a"}, budget, now); err == nil { + t.Fatal("raw query ignored the scan budget") + } + if _, err = store.control.Exec(`UPDATE segments SET archiving_at=?,archive_path=path WHERE source_id='source-a'`, now.Format(time.RFC3339Nano)); err != nil { + t.Fatal(err) + } + if _, err = store.queryRawCandidate(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now); err == nil || err.Error() != "raw query segment transition is incomplete" { + t.Fatalf("transition error=%v", err) + } +} + +func TestRawQueryCandidateRejectsNonLogSignals(t *testing.T) { + store := testStore(t) + defer store.Close() + ast, err := query.Parse(`metrics | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + if _, err = store.queryRawCandidate(t.Context(), ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), time.Now().UTC()); err == nil { + t.Fatal("non-log raw query candidate was accepted") + } +} diff --git a/internal/storage/rebuild.go b/internal/storage/rebuild.go new file mode 100644 index 0000000..98c4330 --- /dev/null +++ b/internal/storage/rebuild.go @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" +) + +const maxRebuildSegments = 1_000_000 + +type RebuildReport struct { + OrganizationID string `json:"organization_id"` + Segments int `json:"segments"` + Observations int64 `json:"observations"` + ActiveVersion int `json:"active_projection_version"` + IndexedRows int64 `json:"indexed_rows"` +} + +type rebuildSegment struct { + digest string + path string +} + +// RebuildOrganization reconstructs one disposable organization projection +// from checksummed raw truth beside the live database, then atomically replaces +// it. The caller must hold the data directory's exclusive process lock. +func (s *Store) RebuildOrganization(ctx context.Context, organizationID string, now time.Time) (RebuildReport, error) { + if err := model.ValidateSourceID(organizationID); err != nil || now.IsZero() { + return RebuildReport{}, errors.New("projection rebuild input is invalid") + } + lock := s.namedLock("organization:" + organizationID) + lock.Lock() + defer lock.Unlock() + + dir := filepath.Join(s.root, "organizations", organizationID) + if err := os.MkdirAll(dir, 0o700); err != nil { + return RebuildReport{}, errors.New("create projection rebuild directory") + } + live := filepath.Join(dir, "projection.sqlite") + if err := requireProjectionTarget(live); err != nil { + return RebuildReport{}, err + } + exists, err := s.organizationHasSource(ctx, organizationID) + if err != nil { + return RebuildReport{}, err + } + if !exists { + return RebuildReport{}, errors.New("projection rebuild organization has no enrolled sources") + } + segments, err := s.rebuildSegments(ctx, organizationID) + if err != nil { + return RebuildReport{}, err + } + temporary, err := os.CreateTemp(dir, ".projection-rebuild-*.sqlite") + if err != nil { + return RebuildReport{}, errors.New("create projection rebuild target") + } + stage := temporary.Name() + if err = temporary.Close(); err != nil { + _ = os.Remove(stage) + return RebuildReport{}, errors.New("close projection rebuild target") + } + if err = os.Remove(stage); err != nil { + return RebuildReport{}, errors.New("prepare projection rebuild target") + } + defer removeProjectionFiles(stage) + + db, err := openProjection(ctx, stage) + if err != nil { + return RebuildReport{}, err + } + if err = db.Close(); err != nil { + return RebuildReport{}, errors.New("close empty projection rebuild target") + } + report := RebuildReport{OrganizationID: organizationID, ActiveVersion: 1} + for _, entry := range segments { + if err = ctx.Err(); err != nil { + return RebuildReport{}, err + } + batch, readErr := s.segments.Read(entry.path, entry.digest) + if readErr != nil { + return RebuildReport{}, fmt.Errorf("read projection rebuild segment: %w", readErr) + } + if err = batch.Validate(batch.ObservedAt); err != nil { + return RebuildReport{}, errors.New("projection rebuild segment is invalid") + } + source, sourceErr := s.sourceByID(ctx, batch.SourceID) + if sourceErr != nil { + return RebuildReport{}, sourceErr + } + if source.Scope.OrganizationID != organizationID { + return RebuildReport{}, errors.New("projection rebuild segment organization mismatch") + } + if err = projectAt(ctx, stage, source.Scope, batch, entry.digest); err != nil { + return RebuildReport{}, err + } + if int64(len(batch.Records)) > math.MaxInt64-report.Observations { + return RebuildReport{}, errors.New("projection rebuild observation count overflow") + } + report.Observations += int64(len(batch.Records)) + report.Segments++ + } + descriptors, err := s.activatedDescriptors(ctx, organizationID) + if err != nil { + return RebuildReport{}, err + } + if len(descriptors) > 0 { + report.ActiveVersion, report.IndexedRows, err = activateRebuiltDescriptors(ctx, stage, descriptors, now) + if err != nil { + return RebuildReport{}, err + } + } + if err = finalizeProjection(stage); err != nil { + return RebuildReport{}, err + } + if err = s.closeProjection(organizationID); err != nil { + return RebuildReport{}, errors.New("close live projection before replacement") + } + if err = requireProjectionTarget(live); err != nil { + return RebuildReport{}, err + } + if err = removeProjectionSidecars(live); err != nil { + return RebuildReport{}, err + } + if err = os.Rename(stage, live); err != nil { + return RebuildReport{}, errors.New("activate rebuilt projection") + } + if err = syncProjectionDirectory(dir); err != nil { + return RebuildReport{}, errors.New("sync rebuilt projection directory") + } + return report, nil +} + +func (s *Store) organizationHasSource(ctx context.Context, organizationID string) (bool, error) { + var exists int + err := s.control.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM sources WHERE organization_id=? LIMIT 1)`, organizationID).Scan(&exists) + if err != nil { + return false, errors.New("verify projection rebuild organization") + } + return exists == 1, nil +} + +func (s *Store) rebuildSegments(ctx context.Context, organizationID string) ([]rebuildSegment, error) { + rows, err := s.control.QueryContext(ctx, `SELECT digest,path FROM segments WHERE organization_id=? AND tier='hot' AND retiring_at IS NULL ORDER BY source_id,stream_id,sequence`, organizationID) + if err != nil { + return nil, errors.New("list projection rebuild segments") + } + defer rows.Close() + segments := make([]rebuildSegment, 0) + for rows.Next() { + if len(segments) >= maxRebuildSegments { + return nil, errors.New("projection rebuild segment limit exceeded") + } + var entry rebuildSegment + if err = rows.Scan(&entry.digest, &entry.path); err != nil { + return nil, errors.New("read projection rebuild segment") + } + segments = append(segments, entry) + } + if err = rows.Err(); err != nil { + return nil, errors.New("list projection rebuild segments") + } + return segments, nil +} + +func (s *Store) activatedDescriptors(ctx context.Context, organizationID string) ([]schema.Descriptor, error) { + rows, err := s.control.QueryContext(ctx, `SELECT descriptor_json FROM descriptor_proposals WHERE organization_id=? AND status='activated' ORDER BY signal,field`, organizationID) + if err != nil { + return nil, errors.New("list activated descriptors for rebuild") + } + defer rows.Close() + descriptors := make([]schema.Descriptor, 0) + seen := map[string]bool{} + for rows.Next() { + if len(descriptors) >= model.MaxDistinctFields { + return nil, errors.New("activated descriptor rebuild limit exceeded") + } + var encoded string + var descriptor schema.Descriptor + if err = rows.Scan(&encoded); err != nil || json.Unmarshal([]byte(encoded), &descriptor) != nil || descriptor.Validate() != nil { + return nil, errors.New("activated descriptor rebuild data is invalid") + } + key := string(descriptor.Signal) + ":" + query.CanonicalField(descriptor.Field) + if seen[key] { + return nil, errors.New("activated descriptor rebuild data is duplicated") + } + seen[key] = true + descriptor.ProjectionVersion = 2 + descriptors = append(descriptors, descriptor) + } + if err = rows.Err(); err != nil { + return nil, errors.New("list activated descriptors for rebuild") + } + sort.Slice(descriptors, func(left, right int) bool { + if descriptors[left].Signal == descriptors[right].Signal { + return descriptors[left].Field < descriptors[right].Field + } + return descriptors[left].Signal < descriptors[right].Signal + }) + return descriptors, nil +} + +func activateRebuiltDescriptors(ctx context.Context, path string, descriptors []schema.Descriptor, now time.Time) (int, int64, error) { + db, err := openProjection(ctx, path) + if err != nil { + return 0, 0, err + } + defer db.Close() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return 0, 0, errors.New("begin rebuilt descriptor activation") + } + defer tx.Rollback() + timestamp := now.UTC().Format(time.RFC3339Nano) + if _, err = tx.ExecContext(ctx, `INSERT INTO projection_versions(version,created_at,activated_at) VALUES(2,?,?)`, timestamp, timestamp); err != nil { + return 0, 0, errors.New("create rebuilt projection version") + } + for _, descriptor := range descriptors { + encoded, marshalErr := json.Marshal(descriptor) + if marshalErr != nil { + return 0, 0, errors.New("encode rebuilt active descriptor") + } + if _, err = tx.ExecContext(ctx, `INSERT INTO projection_descriptors(version,signal,field,descriptor_json) VALUES(2,?,?,?)`, descriptor.Signal, descriptor.Field, string(encoded)); err != nil { + return 0, 0, errors.New("store rebuilt active descriptor") + } + } + indexed, err := buildProjectionIndex(ctx, tx, 2, descriptors) + if err != nil { + return 0, 0, err + } + if _, err = tx.ExecContext(ctx, `UPDATE projection_state SET active_version=2 WHERE id=1 AND active_version=1`); err != nil { + return 0, 0, errors.New("activate rebuilt projection version") + } + if err = tx.Commit(); err != nil { + return 0, 0, errors.New("commit rebuilt descriptor activation") + } + return 2, indexed, nil +} + +func requireProjectionTarget(path string) error { + if info, err := os.Lstat(path); err == nil { + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("organization projection must be a regular non-symlink file") + } + } else if !errors.Is(err, os.ErrNotExist) { + return errors.New("inspect organization projection") + } + return nil +} + +func removeProjectionSidecars(path string) error { + for _, suffix := range []string{"-wal", "-shm"} { + sidecar := path + suffix + info, err := os.Lstat(sidecar) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("projection sidecar is not a regular non-symlink file") + } + if err = os.Remove(sidecar); err != nil { + return errors.New("remove inactive projection sidecar") + } + } + return nil +} + +func finalizeProjection(path string) error { + db, err := sql.Open("sqlite", path) + if err != nil { + return errors.New("open rebuilt projection for finalization") + } + if _, err = db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err == nil { + _, err = db.Exec(`PRAGMA journal_mode=DELETE`) + } + closeErr := db.Close() + if err != nil || closeErr != nil { + return errors.New("finalize rebuilt projection") + } + if err = os.Chmod(path, 0o600); err != nil { + return errors.New("set rebuilt projection mode") + } + for _, suffix := range []string{"-wal", "-shm"} { + if _, err = os.Lstat(path + suffix); !errors.Is(err, os.ErrNotExist) { + return errors.New("rebuilt projection retained a sidecar") + } + } + return nil +} + +func removeProjectionFiles(path string) { + _ = os.Remove(path) + _ = os.Remove(path + "-wal") + _ = os.Remove(path + "-shm") +} + +func syncProjectionDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/internal/storage/rebuild_test.go b/internal/storage/rebuild_test.go new file mode 100644 index 0000000..5b0ad6f --- /dev/null +++ b/internal/storage/rebuild_test.go @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "bytes" + "context" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" +) + +func TestProjectionRebuildRestoresRawTruthAndActivatedDescriptorsAtomically(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{ + {Timestamp: now.Add(-time.Minute), Name: "queue.depth", Value: floatPointer(1), Attributes: map[string]string{"workshop.queue_depth": "1"}}, + {Timestamp: now, Name: "queue.depth", Value: floatPointer(2), Attributes: map[string]string{"workshop.queue_depth": "2"}}, + }} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + logBatch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{ + requestObservation(now.Add(-time.Minute), "/broken", 503, 10), + requestObservation(now, "/healthy", 200, 5), + }} + if _, err = store.Ingest(ctx, token, logBatch, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + reviewed := schema.Descriptor{Version: schema.DescriptorVersion, Signal: model.SignalMetrics, Field: "workshop.queue_depth", Type: schema.TypeInteger, Meaning: "Number of work items waiting in the selected service queue.", Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, Index: schema.IndexRange, Retention: schema.RetentionRaw, ProjectionVersion: 1} + if _, err = store.ActivateDescriptor(ctx, scope.OrganizationID, reviewed, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + ast, err := query.Parse(`metrics | where workshop.queue_depth >= 1 | sort workshop.queue_depth desc | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + before, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + logAST, err := query.Parse(`logs | where status >= 500 | window 1h | summarize count() by route, window(5m) | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + logBefore, err := store.Query(ctx, logAST, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + + otherScope := model.Scope{OrganizationID: "organization-b", ProjectID: "project-b", EnvironmentID: "production", ServiceID: "service-b"} + otherToken, err := store.CreateSource(ctx, "source-b", otherScope) + if err != nil { + t.Fatal(err) + } + otherBatch := model.Batch{Version: model.BatchVersion, SourceID: "source-b", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "other.request"}}} + if _, err = store.Ingest(ctx, otherToken, otherBatch, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + otherPath := filepath.Join(store.root, "organizations", otherScope.OrganizationID, "projection.sqlite") + otherBefore, otherInfo := readFileAndInfo(t, otherPath) + + live := filepath.Join(store.root, "organizations", scope.OrganizationID, "projection.sqlite") + if err = os.WriteFile(live, []byte("corrupt disposable projection"), 0o600); err != nil { + t.Fatal(err) + } + report, err := store.RebuildOrganization(ctx, scope.OrganizationID, now.Add(2*time.Minute)) + if err != nil { + t.Fatal(err) + } + if report.Segments != 2 || report.Observations != 4 || report.ActiveVersion != 2 || report.IndexedRows != 2 { + t.Fatalf("report=%+v", report) + } + after, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before.Columns, after.Columns) || !reflect.DeepEqual(before.Rows, after.Rows) || before.Stats.ScannedRows != after.Stats.ScannedRows || before.Stats.MatchedRows != after.Stats.MatchedRows { + t.Fatalf("before=%+v after=%+v", before, after) + } + logAfter, err := store.Query(ctx, logAST, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(logBefore.Columns, logAfter.Columns) || !reflect.DeepEqual(logBefore.Rows, logAfter.Rows) || logBefore.Stats.ScannedRows != logAfter.Stats.ScannedRows || logBefore.Stats.MatchedRows != logAfter.Stats.MatchedRows { + t.Fatalf("log before=%+v after=%+v", logBefore, logAfter) + } + otherAfter, otherAfterInfo := readFileAndInfo(t, otherPath) + if !bytes.Equal(otherBefore, otherAfter) || !otherInfo.ModTime().Equal(otherAfterInfo.ModTime()) { + t.Fatal("rebuilding one organization modified another organization projection") + } + if _, err = os.Lstat(live + "-wal"); !os.IsNotExist(err) { + t.Fatalf("rebuilt projection retained WAL: %v", err) + } + store.projectionMu.Lock() + _, cached := store.projections[scope.OrganizationID] + store.projectionMu.Unlock() + if cached { + t.Fatal("rebuilt projection retained its replaced database handle") + } + batch.Sequence = 2 + batch.ObservedAt = now.Add(time.Minute) + batch.Records = []model.Observation{{Timestamp: now.Add(time.Minute), Name: "queue.depth", Value: floatPointer(3), Attributes: map[string]string{"workshop.queue_depth": "3"}}} + if _, err = store.Ingest(ctx, token, batch, now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + projectAll(t, store) + store.projectionMu.Lock() + refreshed := store.projections[scope.OrganizationID] + store.projectionMu.Unlock() + if refreshed.db == nil { + t.Fatal("rebuilt projection did not receive a fresh database handle") + } +} + +func TestProjectionRebuildFailurePreservesLiveProjection(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 12, 30, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + projectAll(t, store) + live := filepath.Join(store.root, "organizations", scope.OrganizationID, "projection.sqlite") + liveBefore, liveInfo := readFileAndInfo(t, live) + var rawPath string + if err = store.control.QueryRow(`SELECT path FROM segments WHERE organization_id=?`, scope.OrganizationID).Scan(&rawPath); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(rawPath, []byte("corrupt raw truth"), 0o600); err != nil { + t.Fatal(err) + } + if _, err = store.RebuildOrganization(ctx, scope.OrganizationID, now.Add(time.Minute)); err == nil { + t.Fatal("corrupt raw segment was accepted") + } + liveAfter, liveAfterInfo := readFileAndInfo(t, live) + if !bytes.Equal(liveBefore, liveAfter) || !liveInfo.ModTime().Equal(liveAfterInfo.ModTime()) { + t.Fatal("failed rebuild modified the live projection") + } + stages, err := filepath.Glob(filepath.Join(filepath.Dir(live), ".projection-rebuild-*")) + if err != nil || len(stages) != 0 { + t.Fatalf("stages=%v err=%v", stages, err) + } +} + +func TestProjectionRebuildRejectsUnknownOrganization(t *testing.T) { + store := testStore(t) + defer store.Close() + if _, err := store.RebuildOrganization(context.Background(), "unknown-organization", time.Now().UTC()); err == nil { + t.Fatal("unknown organization projection was created") + } + path := filepath.Join(store.root, "organizations", "unknown-organization", "projection.sqlite") + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("unknown organization projection exists: %v", err) + } +} + +func TestProjectionRebuildRefusesSymlinkProjectionOrSidecar(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := context.Background() + now := time.Date(2026, 8, 17, 13, 0, 0, 0, time.UTC) + dir := filepath.Join(store.root, "organizations", "organization-a") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + live := filepath.Join(dir, "projection.sqlite") + if err := os.WriteFile(live, []byte("placeholder"), 0o600); err != nil { + t.Fatal(err) + } + target := filepath.Join(dir, "target.sqlite") + if err := os.WriteFile(target, []byte("target"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, live+"-wal"); err != nil { + t.Fatal(err) + } + if _, err := store.RebuildOrganization(ctx, "organization-a", now); err == nil { + t.Fatal("symlink projection sidecar was accepted") + } + if err := os.Remove(live + "-wal"); err != nil { + t.Fatal(err) + } + if err := os.Remove(live); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, live); err != nil { + t.Fatal(err) + } + if _, err := store.RebuildOrganization(ctx, "organization-a", now); err == nil { + t.Fatal("symlink projection was accepted") + } +} + +func readFileAndInfo(t *testing.T, path string) ([]byte, os.FileInfo) { + t.Helper() + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return body, info +} diff --git a/internal/storage/retention.go b/internal/storage/retention.go new file mode 100644 index 0000000..5ef43ba --- /dev/null +++ b/internal/storage/retention.go @@ -0,0 +1,855 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/segment" +) + +const maximumRetentionDays = 3650 + +var ErrOrganizationStorageQuotaExceeded = errors.New("organization storage quota exceeded") + +// RetentionPolicy is the complete storage lifecycle for one organization. +// Raw metric samples may expire before their five-minute aggregate projection. +type RetentionPolicy struct { + RawLogsDays int `json:"raw_logs_days"` + RawTracesDays int `json:"raw_traces_days"` + RawMetricsDays int `json:"raw_metrics_days"` + ColdRawDays int `json:"cold_raw_days"` + DeleteColdRaw bool `json:"delete_cold_raw"` + MetricRollupsDays int `json:"metric_rollups_days"` + EvidenceDays int `json:"evidence_days"` +} + +func (policy RetentionPolicy) Validate() error { + for _, days := range []int{policy.RawLogsDays, policy.RawTracesDays, policy.RawMetricsDays, policy.ColdRawDays, policy.MetricRollupsDays, policy.EvidenceDays} { + if days < 1 || days > maximumRetentionDays { + return errors.New("retention values must be between 1 and 3650 days") + } + } + if policy.MetricRollupsDays < policy.RawMetricsDays { + return errors.New("metric rollup retention cannot be shorter than raw metric retention") + } + if policy.ColdRawDays < policy.RawLogsDays || policy.ColdRawDays < policy.RawTracesDays || policy.ColdRawDays < policy.RawMetricsDays || policy.ColdRawDays < policy.EvidenceDays { + return errors.New("cold raw retention cannot be shorter than a hot raw or evidence retention window") + } + return nil +} + +type OrganizationRetention struct { + OrganizationID string `json:"organization_id"` + Policy RetentionPolicy `json:"policy"` + QuotaBytes int64 `json:"quota_bytes,omitempty"` + ExtensionApproved bool `json:"extension_approved"` + ExtensionApprovedBy string `json:"extension_approved_by,omitempty"` + UpdatedBy string `json:"updated_by"` + UpdatedAt time.Time `json:"updated_at"` +} + +type SetRetentionInput struct { + OrganizationID string + Policy RetentionPolicy + Defaults RetentionPolicy + ActorUserID string + ApproveExtensionFor string + QuotaBytes int64 +} + +type RetentionReport struct { + Version int `json:"version"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + Organizations int `json:"organizations"` + RawSegmentsRemoved int `json:"raw_segments_removed"` + RawBytesRemoved int64 `json:"raw_bytes_removed"` + RawSegmentsArchived int `json:"raw_segments_archived"` + RawBytesArchived int64 `json:"raw_bytes_archived"` + ProjectedObservationsRemoved int64 `json:"projected_observations_removed"` + MetricRollupsRemoved int64 `json:"metric_rollups_removed"` + LogRollupsRemoved int64 `json:"log_rollups_removed"` + ResolvedIncidentsRemoved int64 `json:"resolved_incidents_removed"` + PolicyEventsRemoved int64 `json:"policy_events_removed"` +} + +func migrateControlRetention(db *sql.DB) error { + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("begin retention migration: %w", err) + } + defer tx.Rollback() + var segmentTable int + if err = tx.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='segments'`).Scan(&segmentTable); err != nil { + return fmt.Errorf("inspect segment schema: %w", err) + } + if segmentTable == 0 { + if _, err = tx.Exec(`CREATE TABLE segments ( + digest TEXT PRIMARY KEY, + organization_id TEXT NOT NULL, + source_id TEXT NOT NULL, + stream_id TEXT NOT NULL, + sequence INTEGER NOT NULL, + path TEXT NOT NULL UNIQUE, + compressed_bytes INTEGER NOT NULL, + uncompressed_bytes INTEGER NOT NULL, + committed_at TEXT NOT NULL, + projected_at TEXT, + signal TEXT NOT NULL DEFAULT '', + first_observed_at TEXT NOT NULL DEFAULT '', + last_observed_at TEXT NOT NULL DEFAULT '', + tier TEXT NOT NULL DEFAULT 'hot' CHECK(tier IN ('hot','cold')), + archiving_at TEXT, + archive_path TEXT, + cold_at TEXT, + retiring_at TEXT, + UNIQUE(source_id,stream_id,sequence) + )`); err != nil { + return fmt.Errorf("create retained segment schema: %w", err) + } + } else { + columns, columnErr := sqliteColumns(tx, "segments") + if columnErr != nil { + return columnErr + } + for _, column := range []struct{ name, definition string }{ + {"signal", `TEXT NOT NULL DEFAULT ''`}, + {"first_observed_at", `TEXT NOT NULL DEFAULT ''`}, + {"last_observed_at", `TEXT NOT NULL DEFAULT ''`}, + {"tier", `TEXT NOT NULL DEFAULT 'hot' CHECK(tier IN ('hot','cold'))`}, + {"archiving_at", `TEXT`}, + {"archive_path", `TEXT`}, + {"cold_at", `TEXT`}, + {"retiring_at", `TEXT`}, + } { + if columns[column.name] { + continue + } + if _, err = tx.Exec(`ALTER TABLE segments ADD COLUMN ` + column.name + ` ` + column.definition); err != nil { + return fmt.Errorf("add segment retention column: %w", err) + } + } + } + for _, statement := range []string{ + `CREATE INDEX IF NOT EXISTS segments_retention ON segments(organization_id,tier,signal,last_observed_at,retiring_at)`, + `CREATE INDEX IF NOT EXISTS segments_archiving ON segments(archiving_at,organization_id)`, + `CREATE TABLE IF NOT EXISTS organization_retention_policies ( + organization_id TEXT PRIMARY KEY, + raw_logs_days INTEGER NOT NULL CHECK(raw_logs_days BETWEEN 1 AND 3650), + raw_traces_days INTEGER NOT NULL CHECK(raw_traces_days BETWEEN 1 AND 3650), + raw_metrics_days INTEGER NOT NULL CHECK(raw_metrics_days BETWEEN 1 AND 3650), + cold_raw_days INTEGER NOT NULL CHECK(cold_raw_days BETWEEN 1 AND 3650), + delete_cold_raw INTEGER NOT NULL DEFAULT 0 CHECK(delete_cold_raw IN (0,1)), + metric_rollups_days INTEGER NOT NULL CHECK(metric_rollups_days BETWEEN 1 AND 3650), + evidence_days INTEGER NOT NULL CHECK(evidence_days BETWEEN 1 AND 3650), + quota_bytes INTEGER CHECK(quota_bytes IS NULL OR quota_bytes > 0), + extension_approved_by TEXT, + updated_by TEXT NOT NULL, + updated_at TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS retention_policy_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + organization_id TEXT NOT NULL, + actor_user_id TEXT NOT NULL, + action TEXT NOT NULL CHECK(action IN ('created','updated')), + summary TEXT NOT NULL, + created_at TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS retention_policy_events_age ON retention_policy_events(organization_id,created_at)`, + `UPDATE schema_version SET version=7 WHERE version=6`, + } { + if _, err = tx.Exec(statement); err != nil { + return fmt.Errorf("migrate retention schema: %w", err) + } + } + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit retention migration: %w", err) + } + return nil +} + +func migrateControlForensicRetention(db *sql.DB) error { + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("begin forensic retention migration: %w", err) + } + defer tx.Rollback() + columns, err := sqliteColumns(tx, "organization_retention_policies") + if err != nil { + return err + } + if !columns["delete_cold_raw"] { + if _, err = tx.Exec(`ALTER TABLE organization_retention_policies ADD COLUMN delete_cold_raw INTEGER NOT NULL DEFAULT 0 CHECK(delete_cold_raw IN (0,1))`); err != nil { + return fmt.Errorf("add forensic retention policy: %w", err) + } + } + if _, err = tx.Exec(`UPDATE schema_version SET version=8 WHERE version=7`); err != nil { + return fmt.Errorf("advance forensic retention schema: %w", err) + } + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit forensic retention migration: %w", err) + } + return nil +} + +type columnQuery interface { + Query(string, ...any) (*sql.Rows, error) +} + +func sqliteColumns(db columnQuery, table string) (map[string]bool, error) { + rows, err := db.Query(`PRAGMA table_info(` + table + `)`) + if err != nil { + return nil, fmt.Errorf("inspect SQLite columns: %w", err) + } + defer rows.Close() + columns := map[string]bool{} + for rows.Next() { + var index, notNull, primaryKey int + var name, valueType string + var defaultValue any + if err = rows.Scan(&index, &name, &valueType, ¬Null, &defaultValue, &primaryKey); err != nil { + return nil, fmt.Errorf("read SQLite columns: %w", err) + } + columns[name] = true + } + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("read SQLite columns: %w", err) + } + return columns, nil +} + +func retentionExtended(policy, defaults RetentionPolicy) bool { + return policy.RawLogsDays > defaults.RawLogsDays || + policy.RawTracesDays > defaults.RawTracesDays || + policy.RawMetricsDays > defaults.RawMetricsDays || + (policy.DeleteColdRaw && defaults.DeleteColdRaw && policy.ColdRawDays > defaults.ColdRawDays) || + (!policy.DeleteColdRaw && defaults.DeleteColdRaw) || + policy.MetricRollupsDays > defaults.MetricRollupsDays || + policy.EvidenceDays > defaults.EvidenceDays +} + +// SetOrganizationRetention records a policy selected by an organization +// owner. Extending a server default additionally requires an exact approval +// string and a quota larger than current organization storage. +func (s *Store) SetOrganizationRetention(ctx context.Context, input SetRetentionInput, now time.Time) (OrganizationRetention, error) { + if err := model.ValidateSourceID(input.OrganizationID); err != nil || model.ValidateSourceID(input.ActorUserID) != nil || input.Policy.Validate() != nil || input.Defaults.Validate() != nil || now.IsZero() { + return OrganizationRetention{}, errors.New("retention policy input is invalid") + } + extended := retentionExtended(input.Policy, input.Defaults) + if extended { + if input.ApproveExtensionFor != input.OrganizationID || input.QuotaBytes <= 0 { + return OrganizationRetention{}, errors.New("retention extension requires exact organization approval and a positive quota") + } + used, err := s.organizationStorageBytes(input.OrganizationID) + if err != nil { + return OrganizationRetention{}, err + } + if used >= input.QuotaBytes { + return OrganizationRetention{}, errors.New("organization storage already exceeds the approved quota") + } + } else if input.QuotaBytes != 0 || input.ApproveExtensionFor != "" { + return OrganizationRetention{}, errors.New("retention extension approval is not applicable") + } + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return OrganizationRetention{}, errors.New("begin retention policy update") + } + defer tx.Rollback() + var existed int + if err = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM organization_retention_policies WHERE organization_id=?)`, input.OrganizationID).Scan(&existed); err != nil { + return OrganizationRetention{}, errors.New("inspect retention policy") + } + var quota any + var approvedBy any + if extended { + quota = input.QuotaBytes + approvedBy = input.ActorUserID + } + timestamp := now.UTC().Format(time.RFC3339Nano) + _, err = tx.ExecContext(ctx, `INSERT INTO organization_retention_policies(organization_id,raw_logs_days,raw_traces_days,raw_metrics_days,cold_raw_days,delete_cold_raw,metric_rollups_days,evidence_days,quota_bytes,extension_approved_by,updated_by,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(organization_id) DO UPDATE SET raw_logs_days=excluded.raw_logs_days,raw_traces_days=excluded.raw_traces_days,raw_metrics_days=excluded.raw_metrics_days,cold_raw_days=excluded.cold_raw_days,delete_cold_raw=excluded.delete_cold_raw,metric_rollups_days=excluded.metric_rollups_days,evidence_days=excluded.evidence_days,quota_bytes=excluded.quota_bytes,extension_approved_by=excluded.extension_approved_by,updated_by=excluded.updated_by,updated_at=excluded.updated_at`, input.OrganizationID, input.Policy.RawLogsDays, input.Policy.RawTracesDays, input.Policy.RawMetricsDays, input.Policy.ColdRawDays, input.Policy.DeleteColdRaw, input.Policy.MetricRollupsDays, input.Policy.EvidenceDays, quota, approvedBy, input.ActorUserID, timestamp) + if err != nil { + return OrganizationRetention{}, errors.New("store retention policy") + } + action := "created" + if existed == 1 { + action = "updated" + } + summary := fmt.Sprintf("Organization retention policy changed; delete_cold_raw=%t", input.Policy.DeleteColdRaw) + if _, err = tx.ExecContext(ctx, `INSERT INTO retention_policy_events(organization_id,actor_user_id,action,summary,created_at) VALUES(?,?,?,?,?)`, input.OrganizationID, input.ActorUserID, action, summary, timestamp); err != nil { + return OrganizationRetention{}, errors.New("record retention policy event") + } + if err = tx.Commit(); err != nil { + return OrganizationRetention{}, errors.New("commit retention policy update") + } + return OrganizationRetention{OrganizationID: input.OrganizationID, Policy: input.Policy, QuotaBytes: input.QuotaBytes, ExtensionApproved: extended, ExtensionApprovedBy: func() string { + if extended { + return input.ActorUserID + } + return "" + }(), UpdatedBy: input.ActorUserID, UpdatedAt: now.UTC()}, nil +} + +func (s *Store) organizationStorageBytes(organizationID string) (int64, error) { + if err := model.ValidateSourceID(organizationID); err != nil { + return 0, errors.New("invalid organization identifier") + } + var raw int64 + if err := s.control.QueryRow(`SELECT COALESCE(SUM(compressed_bytes),0) FROM segments WHERE organization_id=?`, organizationID).Scan(&raw); err != nil { + return 0, errors.New("measure organization raw storage") + } + projected, err := s.EstimateOrganizationBytes(organizationID) + if err != nil { + return 0, err + } + if raw > int64(^uint64(0)>>1)-projected { + return 0, errors.New("organization storage size overflow") + } + return raw + projected, nil +} + +func (s *Store) checkCommittedQuota(ctx context.Context, organizationID string, committed segment.Committed) error { + var recorded int + if err := s.control.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM segments WHERE digest=?)`, committed.Digest).Scan(&recorded); err != nil { + return errors.New("inspect committed segment quota state") + } + if recorded == 1 { + return nil + } + var quota sql.NullInt64 + err := s.control.QueryRowContext(ctx, `SELECT quota_bytes FROM organization_retention_policies WHERE organization_id=?`, organizationID).Scan("a) + if errors.Is(err, sql.ErrNoRows) || !quota.Valid { + return nil + } + if err != nil { + return errors.New("read organization storage quota") + } + used, err := s.organizationStorageBytes(organizationID) + if err != nil { + return err + } + addition := committed.Compressed + committed.Uncompressed + if addition < 0 || used > quota.Int64-addition { + return ErrOrganizationStorageQuotaExceeded + } + return nil +} + +// admitCommitted serializes quota admission across every source belonging to +// one organization. The raw object is already durable at this point; a quota +// rejection removes that exact checksummed object before returning. The +// segment catalog row and stream watermark are committed together before an +// acknowledgement can be returned. Projection is intentionally independent. +func (s *Store) admitCommitted(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, committedAt time.Time) error { + return s.admitCommittedEnvelope(ctx, scope, batch, committed, nil, committedAt) +} + +func (s *Store) admitCommittedEnvelope(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, envelope *model.BatchEnvelope, committedAt time.Time) error { + lock := s.namedLock("quota:" + scope.OrganizationID) + lock.Lock() + defer lock.Unlock() + if err := s.checkCommittedQuota(ctx, scope.OrganizationID, committed); err != nil { + if deleteErr := s.segments.Delete(committed.Path, committed.Digest); deleteErr != nil { + return fmt.Errorf("%w; remove rejected segment: %v", err, deleteErr) + } + return err + } + return s.recordCommittedAtEnvelope(ctx, scope, batch, committed, envelope, committedAt) +} + +func (s *Store) effectiveRetention(ctx context.Context, organizationID string, defaults RetentionPolicy) (RetentionPolicy, error) { + if err := defaults.Validate(); err != nil { + return RetentionPolicy{}, err + } + policy := defaults + err := s.control.QueryRowContext(ctx, `SELECT raw_logs_days,raw_traces_days,raw_metrics_days,cold_raw_days,delete_cold_raw,metric_rollups_days,evidence_days FROM organization_retention_policies WHERE organization_id=?`, organizationID).Scan(&policy.RawLogsDays, &policy.RawTracesDays, &policy.RawMetricsDays, &policy.ColdRawDays, &policy.DeleteColdRaw, &policy.MetricRollupsDays, &policy.EvidenceDays) + if errors.Is(err, sql.ErrNoRows) { + return defaults, nil + } + if err != nil || policy.Validate() != nil { + return RetentionPolicy{}, errors.New("organization retention policy is invalid") + } + return policy, nil +} + +// ApplyRetention materializes five-minute metric rollups before removing +// expired raw projections, then retires only raw segments whose newest record +// is outside the applicable window. Segment retirement is crash-recoverable. +func (s *Store) ApplyRetention(ctx context.Context, defaults RetentionPolicy, now time.Time) (RetentionReport, error) { + if defaults.Validate() != nil || now.IsZero() { + return RetentionReport{}, errors.New("retention run input is invalid") + } + report := RetentionReport{Version: 1, StartedAt: now.UTC()} + organizations, err := s.retentionOrganizations(ctx) + if err != nil { + return report, err + } + for _, organizationID := range organizations { + if err = ctx.Err(); err != nil { + return report, err + } + policy, policyErr := s.effectiveRetention(ctx, organizationID, defaults) + if policyErr != nil { + return report, policyErr + } + lock := s.namedLock("organization:" + organizationID) + lock.Lock() + organizationReport, applyErr := s.applyOrganizationRetention(ctx, organizationID, policy, now.UTC()) + lock.Unlock() + if applyErr != nil { + return report, applyErr + } + report.Organizations++ + report.RawSegmentsRemoved += organizationReport.RawSegmentsRemoved + report.RawBytesRemoved += organizationReport.RawBytesRemoved + report.RawSegmentsArchived += organizationReport.RawSegmentsArchived + report.RawBytesArchived += organizationReport.RawBytesArchived + report.ProjectedObservationsRemoved += organizationReport.ProjectedObservationsRemoved + report.MetricRollupsRemoved += organizationReport.MetricRollupsRemoved + report.LogRollupsRemoved += organizationReport.LogRollupsRemoved + report.ResolvedIncidentsRemoved += organizationReport.ResolvedIncidentsRemoved + report.PolicyEventsRemoved += organizationReport.PolicyEventsRemoved + } + report.CompletedAt = time.Now().UTC() + return report, nil +} + +func (s *Store) retentionOrganizations(ctx context.Context) ([]string, error) { + rows, err := s.control.QueryContext(ctx, `SELECT organization_id FROM sources UNION SELECT organization_id FROM segments UNION SELECT organization_id FROM organization_retention_policies ORDER BY organization_id`) + if err != nil { + return nil, errors.New("list retention organizations") + } + defer rows.Close() + var organizations []string + for rows.Next() { + var organizationID string + if err = rows.Scan(&organizationID); err != nil || model.ValidateSourceID(organizationID) != nil { + return nil, errors.New("retention organization is invalid") + } + organizations = append(organizations, organizationID) + } + if err = rows.Err(); err != nil { + return nil, errors.New("list retention organizations") + } + return organizations, nil +} + +func retentionCutoff(now time.Time, days int) string { + return now.Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339Nano) +} + +func (s *Store) applyOrganizationRetention(ctx context.Context, organizationID string, policy RetentionPolicy, now time.Time) (RetentionReport, error) { + report := RetentionReport{Version: 1} + path := s.organizationProjectionPath(organizationID) + if info, err := os.Lstat(path); err == nil { + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return report, errors.New("organization projection is unavailable") + } + db, openErr := openProjection(ctx, path) + if openErr != nil { + return report, openErr + } + tx, beginErr := db.BeginTx(ctx, nil) + if beginErr != nil { + _ = db.Close() + return report, errors.New("begin organization retention") + } + for _, expiration := range []struct { + signal model.Signal + days int + }{ + {model.SignalLogs, policy.RawLogsDays}, + {model.SignalTraces, policy.RawTracesDays}, + {model.SignalMetrics, policy.RawMetricsDays}, + {model.SignalDeployments, policy.EvidenceDays}, + } { + result, deleteErr := tx.ExecContext(ctx, `DELETE FROM observations WHERE organization_id=? AND signal=? AND timestamp 0 || report.MetricRollupsRemoved > 0 || report.LogRollupsRemoved > 0 { + if _, err = db.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + _ = db.Close() + return report, errors.New("checkpoint retained organization projection") + } + if _, err = db.ExecContext(ctx, `VACUUM`); err != nil { + _ = db.Close() + return report, errors.New("compact retained organization projection") + } + } + if err = db.Close(); err != nil { + return report, errors.New("close retained organization projection") + } + } else if !errors.Is(err, os.ErrNotExist) { + return report, errors.New("inspect organization projection for retention") + } + + evidenceCutoff := retentionCutoff(now, policy.EvidenceDays) + result, err := s.control.ExecContext(ctx, `DELETE FROM incidents WHERE organization_id=? AND state='resolved' AND updated_at math.MaxInt64-archivedBytes { + return archived, archivedBytes, errors.New("cold archive byte count overflow") + } + archivedBytes += candidate.bytes + removeEmptyPrivateDirectory(filepath.Dir(candidate.path)) + removeEmptyPrivateDirectory(filepath.Dir(filepath.Dir(candidate.path))) + } + return archived, archivedBytes, nil +} + +func (s *Store) finalizeRetiring(ctx context.Context, organizationID string) (int, int64, error) { + rows, err := s.control.QueryContext(ctx, `SELECT digest,path,compressed_bytes FROM segments WHERE organization_id=? AND retiring_at IS NOT NULL ORDER BY retiring_at,digest`, organizationID) + if err != nil { + return 0, 0, errors.New("list retiring raw segments") + } + type retiringSegment struct { + digest, path string + bytes int64 + } + var pending []retiringSegment + for rows.Next() { + var segment retiringSegment + if err = rows.Scan(&segment.digest, &segment.path, &segment.bytes); err != nil { + _ = rows.Close() + return 0, 0, errors.New("read retiring raw segment") + } + pending = append(pending, segment) + } + if err = rows.Close(); err != nil { + return 0, 0, errors.New("close retiring raw segments") + } + path := s.organizationProjectionPath(organizationID) + var projection *sql.DB + if _, err = os.Lstat(path); err == nil { + projection, err = openProjection(ctx, path) + if err != nil { + return 0, 0, err + } + defer projection.Close() + } else if !errors.Is(err, os.ErrNotExist) { + return 0, 0, errors.New("inspect retiring segment projection") + } + removed := 0 + var removedBytes int64 + for _, segment := range pending { + if projection != nil { + tx, beginErr := projection.BeginTx(ctx, nil) + if beginErr != nil { + return removed, removedBytes, errors.New("begin retiring segment projection") + } + if _, err = tx.ExecContext(ctx, `DELETE FROM observations WHERE organization_id=? AND segment_digest=?`, organizationID, segment.digest); err != nil { + _ = tx.Rollback() + return removed, removedBytes, errors.New("remove retiring segment projection") + } + if _, err = tx.ExecContext(ctx, `DELETE FROM metric_rollup_segments WHERE segment_digest=?`, segment.digest); err != nil { + _ = tx.Rollback() + return removed, removedBytes, errors.New("remove retiring segment rollup ledger") + } + if _, err = tx.ExecContext(ctx, `DELETE FROM log_rollup_segments WHERE segment_digest=?`, segment.digest); err != nil { + _ = tx.Rollback() + return removed, removedBytes, errors.New("remove retiring segment log rollup ledger") + } + if err = tx.Commit(); err != nil { + return removed, removedBytes, errors.New("commit retiring segment projection") + } + } + if err = s.segments.Delete(segment.path, segment.digest); err != nil { + return removed, removedBytes, err + } + tx, beginErr := s.control.BeginTx(ctx, nil) + if beginErr != nil { + return removed, removedBytes, errors.New("begin retiring segment acknowledgement") + } + if _, err = tx.ExecContext(ctx, `DELETE FROM descriptor_proposal_segments WHERE segment_digest=?`, segment.digest); err != nil { + _ = tx.Rollback() + return removed, removedBytes, errors.New("remove retired segment proposal evidence") + } + result, deleteErr := tx.ExecContext(ctx, `DELETE FROM segments WHERE digest=? AND organization_id=? AND retiring_at IS NOT NULL`, segment.digest, organizationID) + if deleteErr != nil { + _ = tx.Rollback() + return removed, removedBytes, errors.New("acknowledge retired segment") + } + if changed, _ := result.RowsAffected(); changed != 1 { + _ = tx.Rollback() + return removed, removedBytes, errors.New("retired segment state changed") + } + if err = tx.Commit(); err != nil { + return removed, removedBytes, errors.New("commit retired segment acknowledgement") + } + removed++ + removedBytes += segment.bytes + removeEmptyPrivateDirectory(filepath.Dir(segment.path)) + removeEmptyPrivateDirectory(filepath.Dir(filepath.Dir(segment.path))) + } + return removed, removedBytes, nil +} + +func (s *Store) finishInterruptedRetention(ctx context.Context) error { + rows, err := s.control.QueryContext(ctx, `SELECT DISTINCT organization_id FROM segments WHERE retiring_at IS NOT NULL ORDER BY organization_id`) + if err != nil { + return errors.New("list interrupted retention organizations") + } + var organizations []string + for rows.Next() { + var organizationID string + if err = rows.Scan(&organizationID); err != nil { + _ = rows.Close() + return errors.New("read interrupted retention organization") + } + organizations = append(organizations, organizationID) + } + if err = rows.Close(); err != nil { + return errors.New("close interrupted retention organizations") + } + for _, organizationID := range organizations { + lock := s.namedLock("organization:" + organizationID) + lock.Lock() + _, _, finalizeErr := s.finalizeRetiring(ctx, organizationID) + lock.Unlock() + if finalizeErr != nil { + return finalizeErr + } + } + return nil +} + +func (s *Store) finishInterruptedArchival(ctx context.Context) error { + rows, err := s.control.QueryContext(ctx, `SELECT DISTINCT organization_id FROM segments WHERE archiving_at IS NOT NULL ORDER BY organization_id`) + if err != nil { + return errors.New("list interrupted archive organizations") + } + var organizations []string + for rows.Next() { + var organizationID string + if err = rows.Scan(&organizationID); err != nil || model.ValidateSourceID(organizationID) != nil { + _ = rows.Close() + return errors.New("read interrupted archive organization") + } + organizations = append(organizations, organizationID) + } + if err = rows.Close(); err != nil { + return errors.New("close interrupted archive organizations") + } + for _, organizationID := range organizations { + lock := s.namedLock("organization:" + organizationID) + lock.Lock() + _, _, finalizeErr := s.finalizeArchiving(ctx, organizationID) + lock.Unlock() + if finalizeErr != nil { + return finalizeErr + } + } + return nil +} + +func (s *Store) organizationProjectionPath(organizationID string) string { + return filepath.Join(s.root, "organizations", organizationID, "projection.sqlite") +} + +func (s *Store) backfillSegmentRetentionMetadata(ctx context.Context) error { + rows, err := s.control.QueryContext(ctx, `SELECT digest,path FROM segments WHERE signal='' OR first_observed_at='' OR last_observed_at='' ORDER BY committed_at,digest`) + if err != nil { + return errors.New("list legacy segment metadata") + } + type legacy struct{ digest, path string } + var pending []legacy + for rows.Next() { + var item legacy + if err = rows.Scan(&item.digest, &item.path); err != nil { + _ = rows.Close() + return errors.New("read legacy segment metadata") + } + pending = append(pending, item) + } + if err = rows.Close(); err != nil { + return errors.New("close legacy segment metadata") + } + for _, item := range pending { + batch, readErr := s.segments.Read(item.path, item.digest) + if readErr != nil { + return fmt.Errorf("backfill segment metadata: %w", readErr) + } + if len(batch.Records) == 0 { + return errors.New("backfill segment metadata: segment has no records") + } + first, last := observationRange(batch) + result, updateErr := s.control.ExecContext(ctx, `UPDATE segments SET signal=?,first_observed_at=?,last_observed_at=? WHERE digest=? AND (signal='' OR first_observed_at='' OR last_observed_at='')`, batch.Signal, first.Format(time.RFC3339Nano), last.Format(time.RFC3339Nano), item.digest) + if updateErr != nil { + return errors.New("backfill segment metadata") + } + if changed, _ := result.RowsAffected(); changed != 1 { + return errors.New("segment metadata changed during backfill") + } + } + return nil +} + +func removeEmptyPrivateDirectory(path string) { + info, err := os.Lstat(path) + if err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 { + _ = os.Remove(path) + } +} diff --git a/internal/storage/retention_test.go b/internal/storage/retention_test.go new file mode 100644 index 0000000..deacb90 --- /dev/null +++ b/internal/storage/retention_test.go @@ -0,0 +1,564 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "math" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/segment" +) + +var previewRetention = RetentionPolicy{RawLogsDays: 30, RawTracesDays: 30, RawMetricsDays: 14, ColdRawDays: 400, MetricRollupsDays: 400, EvidenceDays: 400} + +func TestMetricRollupsAggregateAndBackfill(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := t.Context() + scope := model.Scope{OrganizationID: "org-rollup", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-rollup", scope) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 4, 8, 0, 0, time.UTC) + values := []float64{-2, 0, 3, 100} + records := make([]model.Observation, 0, len(values)) + for index := range values { + value := values[index] + records = append(records, model.Observation{Timestamp: now.Add(time.Duration(index) * time.Second), Name: "http.duration", Value: &value, Attributes: map[string]string{"http.route": "/", "private.token_hint": "must-not-roll-up"}}) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-rollup", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: records} + ack, err := store.Ingest(ctx, token, batch, now) + if err != nil { + t.Fatal(err) + } + projectAll(t, store) + if err = projectAt(ctx, store.organizationProjectionPath(scope.OrganizationID), scope, batch, ack.Digest); err != nil { + t.Fatal(err) + } + db := openTestProjection(t, store.organizationProjectionPath(scope.OrganizationID)) + defer db.Close() + var count int64 + var sum, minimum, maximum float64 + var histogram, dimensions string + if err = db.QueryRow(`SELECT sample_count,value_sum,value_min,value_max,histogram_json,attributes_json FROM metric_rollups_5m`).Scan(&count, &sum, &minimum, &maximum, &histogram, &dimensions); err != nil { + t.Fatal(err) + } + if count != 4 || sum != 101 || minimum != -2 || maximum != 100 { + t.Fatalf("count=%d sum=%g min=%g max=%g", count, sum, minimum, maximum) + } + var ledger int + if err = db.QueryRow(`SELECT COUNT(*) FROM metric_rollup_segments WHERE segment_digest=?`, ack.Digest).Scan(&ledger); err != nil || ledger != 1 { + t.Fatalf("rollup ledger=%d err=%v", ledger, err) + } + if dimensions != `{"http.route":"/"}` { + t.Fatalf("rollup dimensions=%s", dimensions) + } + bins, err := decodeHistogram(histogram) + if err != nil { + t.Fatal(err) + } + p95, ok := histogramPercentile(bins, .95) + if !ok || math.Abs(p95-100) > 2 { + t.Fatalf("p95=%g ok=%t", p95, ok) + } + + // Simulate the projection shape from a prior private preview. The first + // open performs one transactional backfill and records its version. + legacyRoot := filepath.Join(t.TempDir(), "data") + if err = os.MkdirAll(filepath.Join(legacyRoot, "organizations", "legacy"), 0o700); err != nil { + t.Fatal(err) + } + legacyPath := filepath.Join(legacyRoot, "organizations", "legacy", "projection.sqlite") + legacy, err := sql.Open("sqlite", legacyPath) + if err != nil { + t.Fatal(err) + } + if _, err = legacy.Exec(`CREATE TABLE observations ( + organization_id TEXT NOT NULL, project_id TEXT NOT NULL, environment_id TEXT NOT NULL, service_id TEXT NOT NULL, + source_id TEXT NOT NULL, stream_id TEXT NOT NULL, sequence INTEGER NOT NULL, record_index INTEGER NOT NULL, + signal TEXT NOT NULL, timestamp TEXT NOT NULL, name TEXT NOT NULL, severity TEXT, body TEXT, value REAL, + trace_id TEXT, span_id TEXT, correlation_id TEXT, attributes_json TEXT NOT NULL, segment_digest TEXT NOT NULL, + PRIMARY KEY(source_id,stream_id,sequence,record_index)); + INSERT INTO observations VALUES('legacy','project','prod','web','source','metrics',1,0,'metrics','2026-08-17T04:01:00Z','queue',NULL,NULL,5,NULL,NULL,NULL,'{}','digest')`); err != nil { + t.Fatal(err) + } + if err = legacy.Close(); err != nil { + t.Fatal(err) + } + legacy, err = openProjection(ctx, legacyPath) + if err != nil { + t.Fatal(err) + } + defer legacy.Close() + if err = legacy.QueryRow(`SELECT sample_count,value_sum FROM metric_rollups_5m`).Scan(&count, &sum); err != nil || count != 1 || sum != 5 { + t.Fatalf("backfilled count=%d sum=%g err=%v", count, sum, err) + } +} + +func TestRetentionPreservesRollupsAndRetiresRawSegments(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := t.Context() + scope := model.Scope{OrganizationID: "org-retain", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-retain", scope) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 5, 0, 0, 0, time.UTC) + ingest := func(stream string, signal model.Signal, timestamp time.Time, value *float64) string { + t.Helper() + observation := model.Observation{Timestamp: timestamp, Name: "sample", Value: value} + if signal == model.SignalLogs { + observation = requestObservation(timestamp, "/retained", 503, 1) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-retain", StreamID: stream, Sequence: 1, ObservedAt: now, Signal: signal, Records: []model.Observation{observation}} + ack, ingestErr := store.Ingest(ctx, token, batch, now) + if ingestErr != nil { + t.Fatal(ingestErr) + } + var path string + if ingestErr = store.control.QueryRow(`SELECT path FROM segments WHERE digest=?`, ack.Digest).Scan(&path); ingestErr != nil { + t.Fatal(ingestErr) + } + return path + } + metricValue := 42.0 + logPath := ingest("logs", model.SignalLogs, now.Add(-31*24*time.Hour), nil) + metricPath := ingest("metrics", model.SignalMetrics, now.Add(-15*24*time.Hour), &metricValue) + deploymentPath := ingest("deployments", model.SignalDeployments, now.Add(-399*24*time.Hour), nil) + projectAll(t, store) + report, err := store.ApplyRetention(ctx, previewRetention, now) + if err != nil { + t.Fatal(err) + } + if report.RawSegmentsRemoved != 0 || report.RawSegmentsArchived != 2 || report.ProjectedObservationsRemoved != 2 || report.MetricRollupsRemoved != 0 || report.LogRollupsRemoved != 1 { + t.Fatalf("report=%+v", report) + } + for _, hot := range []string{logPath, metricPath} { + if _, err = os.Lstat(hot); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("hot path still exists: %s err=%v", hot, err) + } + } + for _, signal := range []model.Signal{model.SignalLogs, model.SignalMetrics} { + var path, tier string + if err = store.control.QueryRow(`SELECT path,tier FROM segments WHERE organization_id=? AND signal=?`, scope.OrganizationID, signal).Scan(&path, &tier); err != nil { + t.Fatal(err) + } + if tier != "cold" || !strings.Contains(filepath.ToSlash(path), "/cold/") { + t.Fatalf("signal=%s tier=%s path=%s", signal, tier, path) + } + if _, err = os.Stat(path); err != nil { + t.Fatal(err) + } + } + if _, err = os.Lstat(deploymentPath); err != nil { + t.Fatal(err) + } + metricAST, err := query.Parse(`metrics | window 720h | summarize count(), p95(value) | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + metricResult, err := store.Query(ctx, metricAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now) + if err != nil { + t.Fatal(err) + } + if metricResult.Stats.Approximate || len(metricResult.Rows) != 1 || metricResult.Rows[0].Values[0] == nil || *metricResult.Rows[0].Values[0] != "1" || metricResult.Rows[0].Values[1] == nil || *metricResult.Rows[0].Values[1] != "42" || len(metricResult.Explain.ProjectedSources) != 2 || !strings.HasSuffix(metricResult.Explain.ProjectedSources[1], "/cold:raw") { + t.Fatalf("cold metric result=%+v", metricResult) + } + logAST, err := query.Parse(`logs | window 960h | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + logResult, err := store.Query(ctx, logAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now) + if err != nil || len(logResult.Rows) != 1 || len(logResult.Explain.ProjectedSources) != 2 || !strings.HasSuffix(logResult.Explain.ProjectedSources[1], "/cold:raw") { + t.Fatalf("cold log result=%+v err=%v", logResult, err) + } + if _, err = store.Query(ctx, logAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1}, now); !errors.Is(err, query.ErrBudgetExceeded) { + t.Fatalf("cold query memory budget err=%v", err) + } + db := openTestProjection(t, store.organizationProjectionPath(scope.OrganizationID)) + var observations, rollups, logRollups int + if err = db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&observations); err != nil { + t.Fatal(err) + } + if err = db.QueryRow(`SELECT COUNT(*) FROM metric_rollups_5m`).Scan(&rollups); err != nil { + t.Fatal(err) + } + if err = db.QueryRow(`SELECT COUNT(*) FROM log_status_route_rollups_5m`).Scan(&logRollups); err != nil { + t.Fatal(err) + } + if observations != 1 || rollups != 1 || logRollups != 0 { + t.Fatalf("observations=%d metric_rollups=%d log_rollups=%d", observations, rollups, logRollups) + } + info, statErr := os.Stat(store.organizationProjectionPath(scope.OrganizationID)) + if statErr != nil { + t.Fatal(statErr) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("projection mode=%v", info.Mode().Perm()) + } + var rawSegments, ledger int + if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&rawSegments); err != nil || rawSegments != 3 { + t.Fatalf("raw_segments=%d err=%v", rawSegments, err) + } + if err = db.QueryRow(`SELECT COUNT(*) FROM metric_rollup_segments`).Scan(&ledger); err != nil || ledger != 1 { + t.Fatalf("rollup ledger=%d err=%v", ledger, err) + } + if err = db.QueryRow(`SELECT COUNT(*) FROM log_rollup_segments`).Scan(&ledger); err != nil || ledger != 1 { + t.Fatalf("log rollup ledger=%d err=%v", ledger, err) + } + if err = db.Close(); err != nil { + t.Fatal(err) + } + finalReport, err := store.ApplyRetention(ctx, previewRetention, now.Add(401*24*time.Hour)) + if err != nil { + t.Fatal(err) + } + if finalReport.RawSegmentsRemoved != 0 || finalReport.MetricRollupsRemoved != 1 { + t.Fatalf("final report=%+v", finalReport) + } + if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&rawSegments); err != nil || rawSegments != 3 { + t.Fatalf("preserved raw_segments=%d err=%v", rawSegments, err) + } + deletingPolicy := previewRetention + deletingPolicy.DeleteColdRaw = true + deletionReport, err := store.ApplyRetention(ctx, deletingPolicy, now.Add(401*24*time.Hour)) + if err != nil { + t.Fatal(err) + } + if deletionReport.RawSegmentsRemoved != 3 { + t.Fatalf("deletion report=%+v", deletionReport) + } + if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&rawSegments); err != nil || rawSegments != 0 { + t.Fatalf("final raw_segments=%d err=%v", rawSegments, err) + } + db = openTestProjection(t, store.organizationProjectionPath(scope.OrganizationID)) + defer db.Close() + if err = db.QueryRow(`SELECT COUNT(*) FROM log_rollup_segments`).Scan(&ledger); err != nil || ledger != 0 { + t.Fatalf("final log rollup ledger=%d err=%v", ledger, err) + } +} + +func TestRetainingColdRawBeyondDefaultRequiresApprovalOnlyWhenServerDeletes(t *testing.T) { + store := testStore(t) + defer store.Close() + defaults := previewRetention + defaults.DeleteColdRaw = true + policy := defaults + policy.DeleteColdRaw = false + input := SetRetentionInput{OrganizationID: "org-forensic", Policy: policy, Defaults: defaults, ActorUserID: "owner"} + if _, err := store.SetOrganizationRetention(t.Context(), input, time.Now().UTC()); err == nil { + t.Fatal("indefinite forensic retention without extension approval was accepted") + } + input.ApproveExtensionFor = input.OrganizationID + input.QuotaBytes = 1 + retained, err := store.SetOrganizationRetention(t.Context(), input, time.Now().UTC()) + if err != nil || !retained.ExtensionApproved || retained.Policy.DeleteColdRaw { + t.Fatalf("retained=%+v err=%v", retained, err) + } + var summary string + if err = store.control.QueryRow(`SELECT summary FROM retention_policy_events WHERE organization_id=?`, input.OrganizationID).Scan(&summary); err != nil || !strings.Contains(summary, "delete_cold_raw=false") { + t.Fatalf("summary=%q err=%v", summary, err) + } +} + +func TestColdCutoffDoesNotExtendAnIndefinitePolicy(t *testing.T) { + store := testStore(t) + defer store.Close() + policy := previewRetention + policy.ColdRawDays++ + retained, err := store.SetOrganizationRetention(t.Context(), SetRetentionInput{ + OrganizationID: "org-indefinite", Policy: policy, Defaults: previewRetention, ActorUserID: "owner", + }, time.Now().UTC()) + if err != nil || retained.ExtensionApproved { + t.Fatalf("retained=%+v err=%v", retained, err) + } +} + +func TestMetricRollupRejectsOutOfRangeValueBeforeRawCommit(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := t.Context() + scope := model.Scope{OrganizationID: "org-range", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-range", scope) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + value := maxMetricMagnitude * 2 + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-range", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: now, Name: "sample", Value: &value}}} + if _, err = store.Ingest(ctx, token, batch, now); err == nil { + t.Fatal("out-of-range metric value was accepted") + } + entries, err := store.segments.List() + if err != nil || len(entries) != 0 { + t.Fatalf("raw entries=%d err=%v", len(entries), err) + } +} + +func TestColdQueryCombinesHotAndColdWithoutDuplication(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := t.Context() + scope := model.Scope{OrganizationID: "org-hot-cold", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-hot-cold", scope) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 6, 0, 0, 0, time.UTC) + for index, sample := range []struct { + stream string + at time.Time + }{{"old", now.Add(-31 * 24 * time.Hour)}, {"new", now.Add(-time.Hour)}} { + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-hot-cold", StreamID: sample.stream, Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: sample.at, Name: "sample", Body: string(rune('a' + index))}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + } + projectAll(t, store) + if _, err = store.ApplyRetention(ctx, previewRetention, now); err != nil { + t.Fatal(err) + } + ast, err := query.Parse(`logs | window 960h | limit 10`, 100) + if err != nil { + t.Fatal(err) + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now) + if err != nil { + t.Fatal(err) + } + if len(result.Rows) != 2 || result.Stats.MatchedRows != 2 || len(result.Explain.ProjectedSources) != 2 || !strings.HasSuffix(result.Explain.ProjectedSources[1], "/cold:raw") { + t.Fatalf("combined result=%+v", result) + } +} + +func TestRetentionExtensionRequiresApprovalAndEnforcesQuota(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := t.Context() + now := time.Date(2026, 8, 17, 5, 0, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "org-quota", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-quota", scope) + if err != nil { + t.Fatal(err) + } + extended := previewRetention + extended.RawLogsDays++ + input := SetRetentionInput{OrganizationID: scope.OrganizationID, Policy: extended, Defaults: previewRetention, ActorUserID: "owner"} + if _, err = store.SetOrganizationRetention(ctx, input, now); err == nil { + t.Fatal("retention extension without approval was accepted") + } + used, err := store.organizationStorageBytes(scope.OrganizationID) + if err != nil { + t.Fatal(err) + } + input.ApproveExtensionFor, input.QuotaBytes = scope.OrganizationID, used+1 + policy, err := store.SetOrganizationRetention(ctx, input, now) + if err != nil || !policy.ExtensionApproved { + t.Fatalf("policy=%+v err=%v", policy, err) + } + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-quota", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}} + if _, err = store.Ingest(ctx, token, batch, now); err == nil || err.Error() != "organization storage quota exceeded" { + t.Fatalf("quota ingest err=%v", err) + } + var segments int + if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&segments); err != nil || segments != 0 { + t.Fatalf("segments=%d err=%v", segments, err) + } +} + +func TestQuotaAdmissionSerializesSourcesAndRecoveryCannotBypassQuota(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := t.Context() + now := time.Date(2026, 8, 17, 5, 30, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "org-quota-race", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + for _, sourceID := range []string{"source-race-a", "source-race-b"} { + if _, err := store.CreateSource(ctx, sourceID, scope); err != nil { + t.Fatal(err) + } + } + batches := []model.Batch{ + {Version: model.BatchVersion, SourceID: "source-race-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}}, + {Version: model.BatchVersion, SourceID: "source-race-b", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}}, + } + committed := make([]segment.Committed, len(batches)) + var err error + for index := range batches { + committed[index], err = store.segments.Commit(scope, batches[index]) + if err != nil { + t.Fatal(err) + } + } + quota := committed[0].Compressed + committed[0].Uncompressed + other := committed[1].Compressed + committed[1].Uncompressed + if other > quota { + quota = other + } + extended := previewRetention + extended.RawLogsDays++ + if _, err = store.SetOrganizationRetention(ctx, SetRetentionInput{OrganizationID: scope.OrganizationID, Policy: extended, Defaults: previewRetention, ActorUserID: "owner", ApproveExtensionFor: scope.OrganizationID, QuotaBytes: quota}, now); err != nil { + t.Fatal(err) + } + + errorsBySource := make(chan error, len(batches)) + var group sync.WaitGroup + for index := range batches { + group.Add(1) + go func(index int) { + defer group.Done() + errorsBySource <- store.admitCommitted(ctx, scope, batches[index], committed[index], time.Now().UTC()) + }(index) + } + group.Wait() + close(errorsBySource) + accepted, rejected := 0, 0 + for admissionErr := range errorsBySource { + switch { + case admissionErr == nil: + accepted++ + case errors.Is(admissionErr, ErrOrganizationStorageQuotaExceeded): + rejected++ + default: + t.Fatalf("admission error=%v", admissionErr) + } + } + if accepted != 1 || rejected != 1 { + t.Fatalf("accepted=%d rejected=%d", accepted, rejected) + } + var recorded int + if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments WHERE organization_id=?`, scope.OrganizationID).Scan(&recorded); err != nil || recorded != 1 { + t.Fatalf("recorded=%d err=%v", recorded, err) + } + + recoveryScope := model.Scope{OrganizationID: "org-quota-recovery", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + if _, err = store.CreateSource(ctx, "source-recovery", recoveryScope); err != nil { + t.Fatal(err) + } + recoveryBatch := model.Batch{Version: model.BatchVersion, SourceID: "source-recovery", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}} + recoverySegment, err := store.segments.Commit(recoveryScope, recoveryBatch) + if err != nil { + t.Fatal(err) + } + if _, err = store.SetOrganizationRetention(ctx, SetRetentionInput{OrganizationID: recoveryScope.OrganizationID, Policy: extended, Defaults: previewRetention, ActorUserID: "owner", ApproveExtensionFor: recoveryScope.OrganizationID, QuotaBytes: 1}, now); err != nil { + t.Fatal(err) + } + if err = store.Recover(ctx); err != nil { + t.Fatal(err) + } + if _, err = os.Lstat(recoverySegment.Path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("over-quota recovery segment still exists: %v", err) + } + if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments WHERE organization_id=?`, recoveryScope.OrganizationID).Scan(&recorded); err != nil || recorded != 0 { + t.Fatalf("recovered records=%d err=%v", recorded, err) + } +} + +func TestRecoverFinishesInterruptedRetention(t *testing.T) { + store := testStore(t) + ctx := context.Background() + scope := model.Scope{OrganizationID: "org-recover", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-recover", scope) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-recover", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}} + ack, err := store.Ingest(ctx, token, batch, now) + if err != nil { + t.Fatal(err) + } + var path string + if _, err = store.control.Exec(`UPDATE segments SET retiring_at=? WHERE digest=?`, now.Format(time.RFC3339Nano), ack.Digest); err != nil { + t.Fatal(err) + } + if err = store.control.QueryRow(`SELECT path FROM segments WHERE digest=?`, ack.Digest).Scan(&path); err != nil { + t.Fatal(err) + } + if err = store.Recover(ctx); err != nil { + t.Fatal(err) + } + if _, err = os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("retiring segment exists after recovery: %v", err) + } + var segments int + if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&segments); err != nil || segments != 0 { + t.Fatalf("segments=%d err=%v", segments, err) + } + if err = store.Close(); err != nil { + t.Fatal(err) + } +} + +func TestRecoverFinishesInterruptedColdArchivesBeforeProjectionRecovery(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := t.Context() + scope := model.Scope{OrganizationID: "org-archive-recover", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-archive", scope) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + for sequence, stream := range []string{"before-move", "after-move"} { + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-archive", StreamID: stream, Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}} + ack, ingestErr := store.Ingest(ctx, token, batch, now) + if ingestErr != nil { + t.Fatal(ingestErr) + } + var candidate archivingSegment + candidate.digest, candidate.sourceID, candidate.streamID, candidate.signal = ack.Digest, batch.SourceID, batch.StreamID, batch.Signal + if ingestErr = store.control.QueryRow(`SELECT path,compressed_bytes FROM segments WHERE digest=?`, ack.Digest).Scan(&candidate.path, &candidate.bytes); ingestErr != nil { + t.Fatal(ingestErr) + } + candidate.archivePath, ingestErr = store.coldArchivePath(scope.OrganizationID, candidate) + if ingestErr != nil { + t.Fatal(ingestErr) + } + if _, ingestErr = store.control.Exec(`UPDATE segments SET archiving_at=?,archive_path=? WHERE digest=?`, now.Format(time.RFC3339Nano), candidate.archivePath, ack.Digest); ingestErr != nil { + t.Fatal(ingestErr) + } + if sequence == 1 { + if ingestErr = store.segments.MoveToCold(candidate.path, candidate.archivePath, candidate.digest); ingestErr != nil { + t.Fatal(ingestErr) + } + } + } + if err = store.Recover(ctx); err != nil { + t.Fatal(err) + } + rows, err := store.control.Query(`SELECT path,tier,archiving_at,archive_path FROM segments WHERE organization_id=? ORDER BY stream_id`, scope.OrganizationID) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + count := 0 + for rows.Next() { + var path, tier string + var archivingAt, archivePath sql.NullString + if err = rows.Scan(&path, &tier, &archivingAt, &archivePath); err != nil { + t.Fatal(err) + } + if tier != "cold" || archivingAt.Valid || archivePath.Valid || !strings.Contains(filepath.ToSlash(path), "/cold/") { + t.Fatalf("path=%s tier=%s archiving=%+v archive_path=%+v", path, tier, archivingAt, archivePath) + } + if _, err = os.Stat(path); err != nil { + t.Fatal(err) + } + count++ + } + if err = rows.Err(); err != nil || count != 2 { + t.Fatalf("archives=%d err=%v", count, err) + } +} diff --git a/internal/storage/rollup.go b/internal/storage/rollup.go new file mode 100644 index 0000000..82cdfe9 --- /dev/null +++ b/internal/storage/rollup.go @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" +) + +const ( + metricRollupVersion = 1 + metricRollupWindow = 5 * time.Minute + maxMetricRollupGroupsBatch = 1024 + maxHistogramBins = 8192 + maxHistogramJSON = 256 << 10 + metricHistogramScale = 64.0 + maxMetricMagnitude = 1e25 +) + +type histogramBin struct { + Bucket int64 `json:"bucket"` + Count int64 `json:"count"` +} + +type metricRollup struct { + bucket, projectID, environmentID, serviceID, name string + dimensionsDigest, attributesJSON string + sampleCount, valueCount int64 + sum, minimum, maximum, lastValue float64 + lastTimestamp string + bins map[int64]int64 +} + +func validateMetricRollupCardinality(batch model.Batch) error { + if batch.Signal != model.SignalMetrics { + return nil + } + groups := map[string]struct{}{} + sums := map[string]float64{} + for _, observation := range batch.Records { + if observation.Value == nil || math.Abs(*observation.Value) > maxMetricMagnitude { + return errors.New("metric value exceeds rollup numeric range") + } + dimensions, err := retainedMetricDimensions(observation.Attributes, nil) + if err != nil { + return err + } + attributes, err := json.Marshal(dimensions) + if err != nil { + return errors.New("encode metric rollup dimensions") + } + bucket := observation.Timestamp.UTC().Truncate(metricRollupWindow).Format(time.RFC3339Nano) + key := bucket + "\x00" + observation.Name + "\x00" + string(attributes) + groups[key] = struct{}{} + sums[key] += *observation.Value + if math.IsNaN(sums[key]) || math.IsInf(sums[key], 0) { + return errors.New("metric batch sum exceeds rollup numeric range") + } + if len(groups) > maxMetricRollupGroupsBatch { + return fmt.Errorf("metric batch exceeds %d rollup groups", maxMetricRollupGroupsBatch) + } + } + return nil +} + +func ensureMetricRollups(ctx context.Context, db *sql.DB) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin metric rollup migration: %w", err) + } + defer tx.Rollback() + for _, statement := range []string{ + `CREATE TABLE IF NOT EXISTS storage_projection_state (id INTEGER PRIMARY KEY CHECK(id=1), metric_rollup_version INTEGER NOT NULL CHECK(metric_rollup_version BETWEEN 0 AND 1), base_index_version INTEGER NOT NULL DEFAULT 0 CHECK(base_index_version BETWEEN 0 AND 1))`, + // Keep this insert compatible with preview databases whose state table + // predates the independently versioned base-index migration. That + // migration adds and initializes its own column transactionally. + `INSERT OR IGNORE INTO storage_projection_state(id,metric_rollup_version) VALUES(1,0)`, + `CREATE TABLE IF NOT EXISTS metric_rollups_5m ( + organization_id TEXT NOT NULL, + project_id TEXT NOT NULL, + environment_id TEXT NOT NULL, + service_id TEXT NOT NULL, + bucket_start TEXT NOT NULL, + name TEXT NOT NULL, + dimensions_digest TEXT NOT NULL, + attributes_json TEXT NOT NULL, + sample_count INTEGER NOT NULL CHECK(sample_count > 0), + value_count INTEGER NOT NULL CHECK(value_count > 0), + value_sum REAL NOT NULL, + value_min REAL NOT NULL, + value_max REAL NOT NULL, + last_value REAL NOT NULL, + last_timestamp TEXT NOT NULL, + histogram_json TEXT NOT NULL, + PRIMARY KEY(organization_id,project_id,environment_id,service_id,bucket_start,name,dimensions_digest) + )`, + `CREATE TABLE IF NOT EXISTS metric_rollup_segments ( + segment_digest TEXT PRIMARY KEY + )`, + `CREATE INDEX IF NOT EXISTS metric_rollups_time ON metric_rollups_5m(organization_id,bucket_start)`, + `CREATE INDEX IF NOT EXISTS metric_rollups_scope ON metric_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start)`, + `CREATE INDEX IF NOT EXISTS metric_rollups_name ON metric_rollups_5m(organization_id,name,bucket_start)`, + } { + if _, err = tx.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("migrate metric rollups: %w", err) + } + } + var version int + if err = tx.QueryRowContext(ctx, `SELECT metric_rollup_version FROM storage_projection_state WHERE id=1`).Scan(&version); err != nil { + return errors.New("read metric rollup migration state") + } + if version == 0 { + if _, err = tx.ExecContext(ctx, `DELETE FROM metric_rollups_5m`); err != nil { + return errors.New("clear incomplete metric rollup migration") + } + if _, err = tx.ExecContext(ctx, `DELETE FROM metric_rollup_segments`); err != nil { + return errors.New("clear incomplete metric rollup segment ledger") + } + _, registry, _, registryErr := activeProjection(ctx, tx) + if registryErr != nil { + return registryErr + } + if err = backfillMetricRollups(ctx, tx, registry); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `INSERT INTO metric_rollup_segments(segment_digest) SELECT DISTINCT segment_digest FROM observations WHERE signal=?`, model.SignalMetrics); err != nil { + return errors.New("record backfilled metric rollup segments") + } + if _, err = tx.ExecContext(ctx, `UPDATE storage_projection_state SET metric_rollup_version=? WHERE id=1`, metricRollupVersion); err != nil { + return errors.New("complete metric rollup migration") + } + } else if version != metricRollupVersion { + return errors.New("unsupported metric rollup projection version") + } + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit metric rollup migration: %w", err) + } + return nil +} + +func backfillMetricRollups(ctx context.Context, tx *sql.Tx, registry query.Registry) error { + rows, err := tx.QueryContext(ctx, `SELECT organization_id,project_id,environment_id,service_id,timestamp,name,value,attributes_json FROM observations WHERE signal=? ORDER BY timestamp,organization_id,project_id,environment_id,service_id,name,attributes_json`, model.SignalMetrics) + if err != nil { + return errors.New("read metrics for rollup migration") + } + defer rows.Close() + groups := map[string]*metricRollup{} + currentBucket := "" + for rows.Next() { + var organizationID, projectID, environmentID, serviceID, timestampText, name, attributesJSON string + var value float64 + if err = rows.Scan(&organizationID, &projectID, &environmentID, &serviceID, ×tampText, &name, &value, &attributesJSON); err != nil { + return errors.New("read metric rollup migration row") + } + timestamp, parseErr := time.Parse(time.RFC3339Nano, timestampText) + if parseErr != nil || math.IsNaN(value) || math.IsInf(value, 0) || math.Abs(value) > maxMetricMagnitude { + return errors.New("metric rollup migration row is invalid") + } + var attributes map[string]string + if err = json.Unmarshal([]byte(attributesJSON), &attributes); err != nil { + return errors.New("metric rollup migration dimensions are invalid") + } + retainedAttributes, retainErr := retainedMetricDimensions(attributes, registry) + if retainErr != nil { + return retainErr + } + retainedJSON, marshalErr := json.Marshal(retainedAttributes) + if marshalErr != nil { + return errors.New("encode metric rollup migration dimensions") + } + attributesJSON = string(retainedJSON) + bucket := timestamp.UTC().Truncate(metricRollupWindow).Format(time.RFC3339Nano) + if currentBucket != "" && bucket != currentBucket { + if err = flushMetricRollups(ctx, tx, groups); err != nil { + return err + } + groups = map[string]*metricRollup{} + } + currentBucket = bucket + rollupKey, digest := metricRollupKey(projectID, environmentID, serviceID, bucket, name, attributesJSON) + key := organizationID + "\x00" + rollupKey + rollup := groups[key] + if rollup == nil { + rollup = newMetricRollup(projectID, environmentID, serviceID, bucket, name, attributesJSON, digest) + groups[key] = rollup + } + addMetricValue(rollup, timestamp, value) + if len(groups) >= maxMetricRollupGroupsBatch { + if err = flushMetricRollups(ctx, tx, groups); err != nil { + return err + } + groups = map[string]*metricRollup{} + } + } + if err = rows.Err(); err != nil { + return errors.New("read metric rollup migration rows") + } + return flushMetricRollups(ctx, tx, groups) +} + +func projectMetricRollups(ctx context.Context, tx *sql.Tx, scope model.Scope, batch model.Batch, segmentDigest string, registry query.Registry) error { + if batch.Signal != model.SignalMetrics { + return nil + } + ledger, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO metric_rollup_segments(segment_digest) VALUES(?)`, segmentDigest) + if err != nil { + return errors.New("record metric rollup segment") + } + inserted, err := ledger.RowsAffected() + if err != nil { + return errors.New("inspect metric rollup segment") + } + if inserted == 0 { + return nil + } + groups := map[string]*metricRollup{} + for _, observation := range batch.Records { + retained, err := retainedMetricDimensions(observation.Attributes, registry) + if err != nil { + return err + } + attributes, err := json.Marshal(retained) + if err != nil { + return errors.New("encode metric rollup dimensions") + } + bucket := observation.Timestamp.UTC().Truncate(metricRollupWindow).Format(time.RFC3339Nano) + key, digest := metricRollupKey(scope.ProjectID, scope.EnvironmentID, scope.ServiceID, bucket, observation.Name, string(attributes)) + rollup := groups[key] + if rollup == nil { + rollup = newMetricRollup(scope.ProjectID, scope.EnvironmentID, scope.ServiceID, bucket, observation.Name, string(attributes), digest) + groups[key] = rollup + if len(groups) > maxMetricRollupGroupsBatch { + return fmt.Errorf("metric batch exceeds %d projected rollup groups", maxMetricRollupGroupsBatch) + } + } + addMetricValue(rollup, observation.Timestamp.UTC(), *observation.Value) + } + for _, rollup := range groups { + if err := mergeMetricRollup(ctx, tx, scope.OrganizationID, rollup); err != nil { + return err + } + } + return nil +} + +func retainedMetricDimensions(attributes map[string]string, registry query.Registry) (map[string]string, error) { + retained := map[string]string{} + for field, value := range attributes { + canonical := query.CanonicalField(field) + descriptor, unknown := query.ResolveDescriptor(model.SignalMetrics, canonical, registry) + if unknown || descriptor.Retention != schema.RetentionMetric || descriptor.Sensitivity == schema.SensitivitySensitive || descriptor.Cardinality == schema.CardinalityHigh { + continue + } + if len(retained) >= model.MaxAttributes { + return nil, errors.New("metric rollup dimension limit exceeded") + } + if _, exists := retained[canonical]; exists { + return nil, errors.New("metric rollup dimensions contain a canonical alias collision") + } + retained[canonical] = value + } + return retained, nil +} + +func newMetricRollup(projectID, environmentID, serviceID, bucket, name, attributesJSON, digest string) *metricRollup { + return &metricRollup{bucket: bucket, projectID: projectID, environmentID: environmentID, serviceID: serviceID, name: name, dimensionsDigest: digest, attributesJSON: attributesJSON, bins: map[int64]int64{}} +} + +func addMetricValue(rollup *metricRollup, timestamp time.Time, value float64) { + rollup.sampleCount++ + rollup.valueCount++ + rollup.sum += value + if rollup.valueCount == 1 || value < rollup.minimum { + rollup.minimum = value + } + if rollup.valueCount == 1 || value > rollup.maximum { + rollup.maximum = value + } + stamp := timestamp.UTC().Format(time.RFC3339Nano) + if rollup.lastTimestamp == "" || stamp >= rollup.lastTimestamp { + rollup.lastTimestamp, rollup.lastValue = stamp, value + } + rollup.bins[metricHistogramBucket(value)]++ +} + +func metricRollupKey(projectID, environmentID, serviceID, bucket, name, attributesJSON string) (string, string) { + dimensions := sha256.Sum256([]byte(name + "\x00" + attributesJSON)) + digest := hex.EncodeToString(dimensions[:]) + return projectID + "\x00" + environmentID + "\x00" + serviceID + "\x00" + bucket + "\x00" + name + "\x00" + digest, digest +} + +func flushMetricRollups(ctx context.Context, tx *sql.Tx, groups map[string]*metricRollup) error { + keys := make([]string, 0, len(groups)) + for key := range groups { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + // Backfill keys prefix the organization. Split once; identifiers cannot + // contain NUL under the model validation contract. + separator := -1 + for index := 0; index < len(key); index++ { + if key[index] == 0 { + separator = index + break + } + } + if separator < 1 { + return errors.New("metric rollup migration key is invalid") + } + if err := mergeMetricRollup(ctx, tx, key[:separator], groups[key]); err != nil { + return err + } + } + return nil +} + +func mergeMetricRollup(ctx context.Context, tx *sql.Tx, organizationID string, incoming *metricRollup) error { + var stored metricRollup + var histogram string + err := tx.QueryRowContext(ctx, `SELECT attributes_json,sample_count,value_count,value_sum,value_min,value_max,last_value,last_timestamp,histogram_json FROM metric_rollups_5m WHERE organization_id=? AND project_id=? AND environment_id=? AND service_id=? AND bucket_start=? AND name=? AND dimensions_digest=?`, organizationID, incoming.projectID, incoming.environmentID, incoming.serviceID, incoming.bucket, incoming.name, incoming.dimensionsDigest).Scan(&stored.attributesJSON, &stored.sampleCount, &stored.valueCount, &stored.sum, &stored.minimum, &stored.maximum, &stored.lastValue, &stored.lastTimestamp, &histogram) + if err == nil { + if stored.attributesJSON != incoming.attributesJSON { + return errors.New("metric rollup dimension digest collision") + } + stored.bins, err = decodeHistogram(histogram) + if err != nil { + return err + } + stored.projectID, stored.environmentID, stored.serviceID, stored.bucket, stored.name, stored.dimensionsDigest = incoming.projectID, incoming.environmentID, incoming.serviceID, incoming.bucket, incoming.name, incoming.dimensionsDigest + if stored.sampleCount > math.MaxInt64-incoming.sampleCount || stored.valueCount > math.MaxInt64-incoming.valueCount { + return errors.New("metric rollup count exceeds numeric range") + } + stored.sampleCount += incoming.sampleCount + stored.valueCount += incoming.valueCount + stored.sum += incoming.sum + if math.IsNaN(stored.sum) || math.IsInf(stored.sum, 0) { + return errors.New("metric rollup sum exceeds numeric range") + } + stored.minimum = math.Min(stored.minimum, incoming.minimum) + stored.maximum = math.Max(stored.maximum, incoming.maximum) + if incoming.lastTimestamp >= stored.lastTimestamp { + stored.lastTimestamp, stored.lastValue = incoming.lastTimestamp, incoming.lastValue + } + for bucket, count := range incoming.bins { + if stored.bins[bucket] > math.MaxInt64-count { + return errors.New("metric histogram count exceeds numeric range") + } + stored.bins[bucket] += count + } + incoming = &stored + } else if !errors.Is(err, sql.ErrNoRows) { + return errors.New("read metric rollup") + } + histogram, err = encodeHistogram(incoming.bins) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `INSERT INTO metric_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start,name,dimensions_digest,attributes_json,sample_count,value_count,value_sum,value_min,value_max,last_value,last_timestamp,histogram_json) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(organization_id,project_id,environment_id,service_id,bucket_start,name,dimensions_digest) DO UPDATE SET sample_count=excluded.sample_count,value_count=excluded.value_count,value_sum=excluded.value_sum,value_min=excluded.value_min,value_max=excluded.value_max,last_value=excluded.last_value,last_timestamp=excluded.last_timestamp,histogram_json=excluded.histogram_json WHERE metric_rollups_5m.attributes_json=excluded.attributes_json`, organizationID, incoming.projectID, incoming.environmentID, incoming.serviceID, incoming.bucket, incoming.name, incoming.dimensionsDigest, incoming.attributesJSON, incoming.sampleCount, incoming.valueCount, incoming.sum, incoming.minimum, incoming.maximum, incoming.lastValue, incoming.lastTimestamp, histogram) + if err != nil { + return errors.New("store metric rollup") + } + return nil +} + +func encodeHistogram(bins map[int64]int64) (string, error) { + if len(bins) < 1 || len(bins) > maxHistogramBins { + return "", errors.New("metric histogram bin count is invalid") + } + keys := make([]int64, 0, len(bins)) + for bucket, count := range bins { + if count < 1 { + return "", errors.New("metric histogram count is invalid") + } + keys = append(keys, bucket) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + encoded := make([]histogramBin, 0, len(keys)) + for _, bucket := range keys { + encoded = append(encoded, histogramBin{Bucket: bucket, Count: bins[bucket]}) + } + body, err := json.Marshal(encoded) + if err != nil || len(body) > maxHistogramJSON { + return "", errors.New("metric histogram encoding exceeds limit") + } + return string(body), nil +} + +func decodeHistogram(encoded string) (map[int64]int64, error) { + if len(encoded) < 2 || len(encoded) > maxHistogramJSON { + return nil, errors.New("metric histogram encoding is invalid") + } + var values []histogramBin + decoder := json.NewDecoder(strings.NewReader(encoded)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&values); err != nil || len(values) < 1 || len(values) > maxHistogramBins { + return nil, errors.New("metric histogram encoding is invalid") + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return nil, errors.New("metric histogram encoding is invalid") + } + bins := make(map[int64]int64, len(values)) + var previous int64 + for index, value := range values { + if value.Count < 1 || index > 0 && value.Bucket <= previous { + return nil, errors.New("metric histogram encoding is invalid") + } + bins[value.Bucket] = value.Count + previous = value.Bucket + } + return bins, nil +} + +func metricHistogramBucket(value float64) int64 { + if value == 0 { + return 0 + } + bucket := int64(math.Round(math.Log1p(math.Abs(value))*metricHistogramScale)) + 1 + if value < 0 { + return -bucket + } + return bucket +} + +func metricHistogramValue(bucket int64) float64 { + if bucket == 0 { + return 0 + } + sign := 1.0 + if bucket < 0 { + sign, bucket = -1, -bucket + } + return sign * math.Expm1(float64(bucket-1)/metricHistogramScale) +} + +func histogramPercentile(bins map[int64]int64, percentile float64) (float64, bool) { + var total int64 + keys := make([]int64, 0, len(bins)) + for bucket, count := range bins { + if count < 1 || total > math.MaxInt64-count { + return 0, false + } + total += count + keys = append(keys, bucket) + } + if total == 0 { + return 0, false + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + target := int64(math.Ceil(percentile * float64(total))) + var seen int64 + for _, bucket := range keys { + seen += bins[bucket] + if seen >= target { + return metricHistogramValue(bucket), true + } + } + return 0, false +} + +func histogramCanonical(value float64) string { + return strconv.FormatFloat(value, 'g', -1, 64) +} diff --git a/internal/storage/rollup_query.go b/internal/storage/rollup_query.go new file mode 100644 index 0000000..9e0ea2f --- /dev/null +++ b/internal/storage/rollup_query.go @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "io" + "math" + "net/url" + "os" + "sort" + "strconv" + "strings" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" + "gamertan.com/observatory/internal/schema" +) + +func metricRollupQueryEligible(ast query.AST, registry query.Registry) bool { + if ast.Signal != model.SignalMetrics || ast.Summary == nil { + return false + } + if ast.Bucket > 0 && (ast.Bucket < metricRollupWindow || ast.Bucket%metricRollupWindow != 0) { + return false + } + for _, aggregate := range ast.Summary.Aggregates { + if aggregate.Function != "count" && query.CanonicalField(aggregate.Field) != "value" { + return false + } + } + for _, filter := range ast.Filters { + field := query.CanonicalField(filter.Field) + switch field { + case "value", "timestamp", "source.id", "stream.id", "severity", "body", "trace_id", "span_id", "correlation_id": + return false + } + if !metricRollupDimensionAvailable(field, registry) { + return false + } + } + for _, field := range ast.Summary.GroupBy { + canonical := query.CanonicalField(field) + switch canonical { + case "value", "timestamp", "source.id", "stream.id", "severity", "body", "trace_id", "span_id", "correlation_id": + return false + } + if !metricRollupDimensionAvailable(canonical, registry) { + return false + } + } + return true +} + +func metricRollupDimensionAvailable(field string, registry query.Registry) bool { + switch query.CanonicalField(field) { + case "project.id", "environment.id", "service.id", "name": + return true + } + descriptor, unknown := query.ResolveDescriptor(model.SignalMetrics, field, registry) + return !unknown && descriptor.Retention == schema.RetentionMetric && descriptor.Sensitivity != schema.SensitivitySensitive && descriptor.Cardinality != schema.CardinalityHigh +} + +func (s *Store) estimateMetricRollupBytes(ctx context.Context, scope query.Scope, ast query.AST, now time.Time) (int64, error) { + path := s.organizationProjectionPath(scope.OrganizationID) + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return 0, nil + } + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return 0, errors.New("organization projection is unavailable") + } + dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) + if err != nil { + return 0, errors.New("open metric rollup estimate") + } + defer db.Close() + db.SetMaxOpenConns(1) + statement := `SELECT COALESCE(SUM(192+LENGTH(name)+LENGTH(attributes_json)+LENGTH(histogram_json)),0) FROM metric_rollups_5m WHERE organization_id=?` + arguments := []any{scope.OrganizationID} + for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} { + if selected.value != "" { + statement += " AND " + selected.column + "=?" + arguments = append(arguments, selected.value) + } + } + if ast.Window > 0 { + statement += " AND bucket_start>=?" + arguments = append(arguments, now.UTC().Add(-ast.Window).Truncate(metricRollupWindow).Format(time.RFC3339Nano)) + } + var estimated int64 + if err = db.QueryRowContext(ctx, statement, arguments...).Scan(&estimated); err != nil || estimated < 0 { + return 0, errors.New("estimate metric rollup scan") + } + return estimated, nil +} + +type rollupAggregate struct { + function string + count int64 + sum, minimum, maximum float64 + bins map[int64]int64 +} + +type rollupSummaryGroup struct { + key string + values []*string + aggregates []rollupAggregate +} + +func (s *Store) queryMetricRollups(ctx context.Context, path string, ast query.AST, scope query.Scope, registry query.Registry, budget query.Budget, now time.Time, result query.Result) (query.Result, error) { + runContext, cancel := context.WithTimeout(ctx, budget.MaxDuration) + defer cancel() + started := time.Now() + dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) + if err != nil { + return query.Result{}, errors.New("open metric rollup projection") + } + defer db.Close() + db.SetMaxOpenConns(1) + statement := `SELECT project_id,environment_id,service_id,bucket_start,name,attributes_json,sample_count,value_count,value_sum,value_min,value_max,last_value,last_timestamp,histogram_json FROM metric_rollups_5m WHERE organization_id=?` + arguments := []any{scope.OrganizationID} + for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} { + if selected.value != "" { + statement += " AND " + selected.column + "=?" + arguments = append(arguments, selected.value) + } + } + if ast.Window > 0 { + statement += " AND bucket_start>=?" + arguments = append(arguments, now.UTC().Add(-ast.Window).Truncate(metricRollupWindow).Format(time.RFC3339Nano)) + } + statement += ` ORDER BY bucket_start DESC,project_id,environment_id,service_id,name,dimensions_digest` + rows, err := db.QueryContext(runContext, statement, arguments...) + if err != nil { + return query.Result{}, queryExecutionError(runContext, err) + } + defer rows.Close() + groups := map[string]*rollupSummaryGroup{} + var memoryBytes int64 + for rows.Next() { + var projectID, environmentID, serviceID, bucketText, name, attributesJSON, lastTimestamp, histogram string + var sampleCount, valueCount int64 + var sum, minimum, maximum, lastValue float64 + if err = rows.Scan(&projectID, &environmentID, &serviceID, &bucketText, &name, &attributesJSON, &sampleCount, &valueCount, &sum, &minimum, &maximum, &lastValue, &lastTimestamp, &histogram); err != nil { + return query.Result{}, errors.New("read metric rollup projection") + } + if result.Stats.ScannedRows == math.MaxInt64 { + return query.Result{}, query.ErrBudgetExceeded + } + result.Stats.ScannedRows++ + readBytes := int64(192 + len(projectID) + len(environmentID) + len(serviceID) + len(bucketText) + len(name) + len(attributesJSON) + len(lastTimestamp) + len(histogram)) + if readBytes < 0 || readBytes > budget.MaxScannedBytes-result.Stats.ScannedBytes { + return query.Result{}, query.ErrBudgetExceeded + } + result.Stats.ScannedBytes += readBytes + bucket, parseErr := time.Parse(time.RFC3339Nano, bucketText) + if parseErr != nil || sampleCount < 1 || valueCount < 1 || valueCount > sampleCount || math.IsNaN(sum) || math.IsInf(sum, 0) || math.IsNaN(minimum) || math.IsInf(minimum, 0) || math.IsNaN(maximum) || math.IsInf(maximum, 0) || math.IsNaN(lastValue) || math.IsInf(lastValue, 0) { + return query.Result{}, errors.New("metric rollup projection is invalid") + } + attributes := map[string]string{} + decoder := json.NewDecoder(strings.NewReader(attributesJSON)) + if err = decoder.Decode(&attributes); err != nil || len(attributes) > model.MaxAttributes { + return query.Result{}, errors.New("metric rollup dimensions are invalid") + } + var trailing any + if err = decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return query.Result{}, errors.New("metric rollup dimensions are invalid") + } + record := projectedRecord{projectID: projectID, environmentID: environmentID, serviceID: serviceID, signal: model.SignalMetrics, timestamp: bucket.UTC(), name: name, value: &lastValue, attributes: attributes} + matched, matchErr := matchesRecord(record, ast, registry) + if matchErr != nil { + return query.Result{}, matchErr + } + if !matched { + continue + } + if result.Stats.MatchedRows == math.MaxInt64 { + return query.Result{}, query.ErrBudgetExceeded + } + result.Stats.MatchedRows++ + bins, decodeErr := decodeHistogram(histogram) + if decodeErr != nil { + return query.Result{}, decodeErr + } + var values []*string + if ast.Bucket > 0 { + window := bucket.UTC().Truncate(ast.Bucket).Format(time.RFC3339Nano) + values = append(values, stringPointer(window)) + } + for _, field := range ast.Summary.GroupBy { + value, present := record.field(field) + if !present { + values = append(values, nil) + continue + } + column := result.Columns[len(values)] + canonical, valid := canonicalResultValue(value, column.Type) + if !valid { + values = append(values, nil) + continue + } + values = append(values, stringPointer(canonical)) + } + key := groupKey(values) + group := groups[key] + if group == nil { + group = &rollupSummaryGroup{key: key, values: values, aggregates: make([]rollupAggregate, len(ast.Summary.Aggregates))} + for index, aggregate := range ast.Summary.Aggregates { + group.aggregates[index] = rollupAggregate{function: aggregate.Function, bins: map[int64]int64{}} + } + groups[key] = group + addition := int64(len(key) + len(values)*16 + len(group.aggregates)*96) + if addition < 0 || addition > budget.MaxMemoryBytes-memoryBytes { + return query.Result{}, query.ErrBudgetExceeded + } + memoryBytes += addition + } + for index, aggregate := range ast.Summary.Aggregates { + state := &group.aggregates[index] + if aggregate.Function == "count" { + if sampleCount > math.MaxInt64-state.count { + return query.Result{}, errors.New("metric rollup count exceeds numeric range") + } + state.count += sampleCount + continue + } + if state.count == 0 { + state.minimum, state.maximum = minimum, maximum + } else { + state.minimum = math.Min(state.minimum, minimum) + state.maximum = math.Max(state.maximum, maximum) + } + if valueCount > math.MaxInt64-state.count { + return query.Result{}, errors.New("metric rollup count exceeds numeric range") + } + state.count += valueCount + state.sum += sum + if math.IsNaN(state.sum) || math.IsInf(state.sum, 0) { + return query.Result{}, errors.New("metric rollup sum exceeds numeric range") + } + if aggregate.Function == "p50" || aggregate.Function == "p95" || aggregate.Function == "p99" { + result.Stats.Approximate = true + for histogramBucket, count := range bins { + if count > math.MaxInt64-state.bins[histogramBucket] { + return query.Result{}, errors.New("metric histogram count exceeds numeric range") + } + state.bins[histogramBucket] += count + } + addition := int64(len(bins) * 24) + if addition < 0 || addition > budget.MaxMemoryBytes-memoryBytes { + return query.Result{}, query.ErrBudgetExceeded + } + memoryBytes += addition + } + } + if memoryBytes > budget.MaxMemoryBytes { + return query.Result{}, query.ErrBudgetExceeded + } + } + if err = rows.Err(); err != nil { + return query.Result{}, queryExecutionError(runContext, err) + } + ordered := make([]*rollupSummaryGroup, 0, len(groups)) + for _, group := range groups { + ordered = append(ordered, group) + } + sort.Slice(ordered, func(i, j int) bool { return ordered[i].key < ordered[j].key }) + for _, group := range ordered { + row := query.Row{Values: append([]*string(nil), group.values...)} + for _, aggregate := range group.aggregates { + value, present := rollupAggregateValue(aggregate) + if present { + row.Values = append(row.Values, stringPointer(value)) + } else { + row.Values = append(row.Values, nil) + } + } + result.Rows = append(result.Rows, row) + } + if ast.Sort != nil { + if err = sortRows(result.Rows, result.Columns, ast.Sort.Field, ast.Sort.Descending); err != nil { + return query.Result{}, err + } + } + if len(result.Rows) > ast.Limit { + result.Rows = result.Rows[:ast.Limit] + result.Stats.Truncated = true + } + result.Stats.DurationNS = time.Since(started).Nanoseconds() + return result, nil +} + +func rollupAggregateValue(state rollupAggregate) (string, bool) { + if state.function == "count" { + return strconv.FormatInt(state.count, 10), true + } + if state.count == 0 { + return "", false + } + var value float64 + switch state.function { + case "min": + value = state.minimum + case "max": + value = state.maximum + case "sum": + value = state.sum + case "avg": + value = state.sum / float64(state.count) + case "p50", "p95", "p99": + percentile := map[string]float64{"p50": .50, "p95": .95, "p99": .99}[state.function] + var ok bool + value, ok = histogramPercentile(state.bins, percentile) + if !ok { + return "", false + } + default: + return "", false + } + return strconv.FormatFloat(value, 'g', -1, 64), true +} diff --git a/internal/storage/rollup_query_test.go b/internal/storage/rollup_query_test.go new file mode 100644 index 0000000..8c5962d --- /dev/null +++ b/internal/storage/rollup_query_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "strconv" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +func TestMetricSummaryQueriesUseFiveMinuteRollups(t *testing.T) { + store := testStore(t) + defer store.Close() + ctx := t.Context() + scope := model.Scope{OrganizationID: "org-query-rollup", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-query-rollup", scope) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 17, 4, 9, 0, 0, time.UTC) + sequence := uint64(1) + for _, sample := range []struct { + at time.Time + value float64 + }{{now.Add(-8 * time.Minute), 1}, {now.Add(-7 * time.Minute), 2}, {now.Add(-2 * time.Minute), 100}} { + value := sample.value + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-query-rollup", StreamID: "metrics", Sequence: sequence, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: sample.at, Name: "request.duration", Value: &value, Attributes: map[string]string{"http.route": "/items", "private.dimension": "secret"}}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + sequence++ + } + projectAll(t, store) + ast, err := query.Parse(`metrics | where route == "/items" | window 1h | summarize count(), avg(value), p95(value) by name, window(10m) | limit 20`, 100) + if err != nil { + t.Fatal(err) + } + result, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID, ProjectID: scope.ProjectID, EnvironmentID: scope.EnvironmentID, ServiceID: scope.ServiceID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now) + if err != nil { + t.Fatal(err) + } + if len(result.Explain.ProjectedSources) != 1 || !strings.HasSuffix(result.Explain.ProjectedSources[0], "/rollup:5m") { + t.Fatalf("sources=%v", result.Explain.ProjectedSources) + } + if !result.Stats.Approximate || result.Stats.ScannedRows != 2 || result.Stats.MatchedRows != 2 || len(result.Rows) != 1 { + t.Fatalf("stats=%+v rows=%+v", result.Stats, result.Rows) + } + row := result.Rows[0] + if len(row.Values) != 5 || row.Values[2] == nil || *row.Values[2] != "3" || row.Values[3] == nil || row.Values[4] == nil { + t.Fatalf("row=%+v", row) + } + average, err := strconv.ParseFloat(*row.Values[3], 64) + if err != nil || average < 34.3 || average > 34.4 { + t.Fatalf("average=%g err=%v", average, err) + } + p95, err := strconv.ParseFloat(*row.Values[4], 64) + if err != nil || p95 < 98 || p95 > 102 { + t.Fatalf("p95=%g err=%v", p95, err) + } + + rawAST, err := query.Parse(`metrics | where value >= 2 | window 1h | summarize count() | limit 20`, 100) + if err != nil { + t.Fatal(err) + } + raw, err := store.Query(ctx, rawAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now) + if err != nil { + t.Fatal(err) + } + if strings.HasSuffix(raw.Explain.ProjectedSources[0], "/rollup:5m") || len(raw.Rows) != 1 || raw.Rows[0].Values[0] == nil || *raw.Rows[0].Values[0] != "2" { + t.Fatalf("raw=%+v", raw) + } + unknownAST, err := query.Parse(`metrics | where private.dimension == "secret" | window 1h | summarize count() | limit 20`, 100) + if err != nil { + t.Fatal(err) + } + unknown, err := store.Query(ctx, unknownAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now) + if err != nil { + t.Fatal(err) + } + if strings.HasSuffix(unknown.Explain.ProjectedSources[0], "/rollup:5m") || len(unknown.Rows) != 1 || unknown.Rows[0].Values[0] == nil || *unknown.Rows[0].Values[0] != "3" { + t.Fatalf("unknown=%+v", unknown) + } +} diff --git a/internal/storage/source_alerts.go b/internal/storage/source_alerts.go new file mode 100644 index 0000000..a0d8753 --- /dev/null +++ b/internal/storage/source_alerts.go @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "gamertan.com/observatory/internal/model" +) + +type SourceAlertTransitionAck struct { + SourceID string `json:"source_id"` + RuleID string `json:"rule_id"` + RuleRevision int `json:"rule_revision"` + AgentEpoch string `json:"agent_epoch"` + Sequence uint64 `json:"sequence"` + Digest string `json:"digest"` + Duplicate bool `json:"duplicate"` +} + +func migrateControlSourceAlertTransitions(db *sql.DB) error { + tx, err := db.Begin() + if err != nil { + return errors.New("begin source alert transition migration") + } + defer tx.Rollback() + for _, statement := range []string{ + `CREATE TABLE source_alert_transitions ( + organization_id TEXT NOT NULL, + source_id TEXT NOT NULL, + rule_id TEXT NOT NULL, + rule_revision INTEGER NOT NULL CHECK(rule_revision BETWEEN 1 AND 1000000), + agent_epoch TEXT NOT NULL CHECK(length(agent_epoch)=32), + transition_sequence INTEGER NOT NULL CHECK(transition_sequence >= 1), + stream_id TEXT NOT NULL, + batch_sequence INTEGER NOT NULL CHECK(batch_sequence >= 1), + segment_digest TEXT NOT NULL, + window_start TEXT NOT NULL, + window_end TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('matched','clear','error')), + observed_at TEXT NOT NULL, + received_at TEXT NOT NULL, + transition_digest TEXT NOT NULL CHECK(length(transition_digest)=64), + PRIMARY KEY(source_id,rule_id,rule_revision,agent_epoch,transition_sequence), + FOREIGN KEY(source_id) REFERENCES sources(id) ON DELETE RESTRICT, + FOREIGN KEY(organization_id,rule_id) REFERENCES alert_rules(organization_id,id) ON DELETE RESTRICT, + FOREIGN KEY(segment_digest) REFERENCES segments(digest) ON DELETE RESTRICT + )`, + `CREATE INDEX source_alert_transitions_by_rule ON source_alert_transitions(organization_id,rule_id,observed_at,source_id,agent_epoch,transition_sequence)`, + `UPDATE schema_version SET version=9 WHERE version=8`, + } { + if _, err = tx.Exec(statement); err != nil { + return fmt.Errorf("migrate source alert transitions: %w", err) + } + } + if err = tx.Commit(); err != nil { + return errors.New("commit source alert transition migration") + } + return nil +} + +func (s *Store) RecordSourceAlertTransition(ctx context.Context, token string, transition model.AlertTransition, now time.Time) (SourceAlertTransitionAck, error) { + if err := transition.Validate(now); err != nil { + return SourceAlertTransitionAck{}, err + } + digest, err := transition.Digest() + if err != nil { + return SourceAlertTransitionAck{}, err + } + source, err := s.Authenticate(ctx, token) + if err != nil { + return SourceAlertTransitionAck{}, err + } + ack := SourceAlertTransitionAck{SourceID: source.ID, RuleID: transition.RuleID, RuleRevision: transition.RuleRevision, AgentEpoch: transition.AgentEpoch, Sequence: transition.Sequence, Digest: digest} + lock := s.namedLock("source-alert:" + source.ID + ":" + transition.RuleID + ":" + transition.AgentEpoch) + lock.Lock() + defer lock.Unlock() + + var existing string + err = s.control.QueryRowContext(ctx, `SELECT transition_digest FROM source_alert_transitions WHERE source_id=? AND rule_id=? AND rule_revision=? AND agent_epoch=? AND transition_sequence=?`, source.ID, transition.RuleID, transition.RuleRevision, transition.AgentEpoch, transition.Sequence).Scan(&existing) + if err == nil { + if existing != digest { + return SourceAlertTransitionAck{}, errors.New("alert transition sequence reused with different content") + } + ack.Duplicate = true + return ack, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return SourceAlertTransitionAck{}, errors.New("read source alert transition") + } + + rule, err := s.AlertRule(ctx, source.Scope.OrganizationID, transition.RuleID) + if err != nil || !rule.Enabled || rule.Revision != transition.RuleRevision { + return SourceAlertTransitionAck{}, errors.New("source alert rule is unavailable") + } + saved, err := s.SavedQuery(ctx, source.Scope.OrganizationID, rule.SavedQueryID) + if err != nil || saved.AST.Signal != model.SignalLogs || saved.Scope.ProjectID != source.Scope.ProjectID || saved.Scope.EnvironmentID != source.Scope.EnvironmentID || saved.Scope.ServiceID != source.Scope.ServiceID { + return SourceAlertTransitionAck{}, errors.New("source alert rule is not scoped to this source") + } + + var segmentOrganization, segmentSignal, firstText, lastText string + err = s.control.QueryRowContext(ctx, `SELECT organization_id,signal,first_observed_at,last_observed_at FROM segments WHERE digest=? AND source_id=? AND stream_id=? AND sequence=? AND retiring_at IS NULL`, transition.SegmentDigest, source.ID, transition.StreamID, transition.BatchSequence).Scan(&segmentOrganization, &segmentSignal, &firstText, &lastText) + if errors.Is(err, sql.ErrNoRows) { + return SourceAlertTransitionAck{}, errors.New("source alert evidence is unavailable") + } + if err != nil { + return SourceAlertTransitionAck{}, errors.New("read source alert evidence") + } + first, firstErr := time.Parse(time.RFC3339Nano, firstText) + last, lastErr := time.Parse(time.RFC3339Nano, lastText) + if firstErr != nil || lastErr != nil || segmentOrganization != source.Scope.OrganizationID || segmentSignal != string(model.SignalLogs) || transition.WindowStart.After(first) || transition.WindowEnd.Before(last) { + return SourceAlertTransitionAck{}, errors.New("source alert evidence does not match transition") + } + + var previous uint64 + if err = s.control.QueryRowContext(ctx, `SELECT COALESCE(MAX(transition_sequence),0) FROM source_alert_transitions WHERE source_id=? AND rule_id=? AND rule_revision=? AND agent_epoch=?`, source.ID, transition.RuleID, transition.RuleRevision, transition.AgentEpoch).Scan(&previous); err != nil { + return SourceAlertTransitionAck{}, errors.New("read source alert transition watermark") + } + if previous != 0 && transition.Sequence != previous+1 { + return SourceAlertTransitionAck{}, errors.New("source alert transition sequence gap") + } + _, err = s.control.ExecContext(ctx, `INSERT INTO source_alert_transitions(organization_id,source_id,rule_id,rule_revision,agent_epoch,transition_sequence,stream_id,batch_sequence,segment_digest,window_start,window_end,state,observed_at,received_at,transition_digest) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, source.Scope.OrganizationID, source.ID, transition.RuleID, transition.RuleRevision, transition.AgentEpoch, transition.Sequence, transition.StreamID, transition.BatchSequence, transition.SegmentDigest, transition.WindowStart.UTC().Format(time.RFC3339Nano), transition.WindowEnd.UTC().Format(time.RFC3339Nano), transition.State, transition.ObservedAt.UTC().Format(time.RFC3339Nano), now.UTC().Format(time.RFC3339Nano), digest) + if err != nil { + return SourceAlertTransitionAck{}, errors.New("record source alert transition") + } + return ack, nil +} diff --git a/internal/storage/source_alerts_test.go b/internal/storage/source_alerts_test.go new file mode 100644 index 0000000..cc93b1f --- /dev/null +++ b/internal/storage/source_alerts_test.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/query" +) + +func TestSourceAlertTransitionBindsAuthenticatedBatchEvidence(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 18, 23, 0, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + saved, err := store.SaveQuery(ctx, SavedQueryInput{ + OrganizationID: scope.OrganizationID, + ActorUserID: "operator-a", + MaxRows: 100, + Name: "Source failures", + Description: "Exact source-scoped failures for differential evaluation.", + Query: "logs | where status >= 500 | window 1h | limit 50", + Scope: ResourceScope{ProjectID: scope.ProjectID, EnvironmentID: scope.EnvironmentID, ServiceID: scope.ServiceID}, + }, now) + if err != nil { + t.Fatal(err) + } + rule, err := store.SaveAlertRule(ctx, AlertRuleInput{ + OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", SavedQueryID: saved.ID, + Name: "Source failures", Description: "Source and server comparison rule.", Severity: "warning", + MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true, + }, now) + if err != nil { + t.Fatal(err) + } + observed := now.Add(time.Second) + batch := model.Batch{ + Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, + ObservedAt: observed, Signal: model.SignalLogs, + Records: []model.Observation{{Timestamp: observed, Name: "http.request", Attributes: map[string]string{"http.status_code": "503", "http.route": "/failed"}}}, + } + ingested, err := store.Ingest(ctx, token, batch, observed) + if err != nil { + t.Fatal(err) + } + projectAll(t, store) + budget := query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 16 << 20, MaxMemoryBytes: 8 << 20} + central, err := store.EvaluateDueAlertRules(ctx, budget, now.Add(15*time.Second)) + if err != nil || len(central) != 1 || !central[0].Matched { + t.Fatalf("central=%+v err=%v", central, err) + } + + transition := model.AlertTransition{ + Version: model.AlertTransitionVersion, RuleID: rule.ID, RuleRevision: rule.Revision, + AgentEpoch: strings.Repeat("a", 32), Sequence: 7, StreamID: batch.StreamID, + BatchSequence: batch.Sequence, SegmentDigest: ingested.Digest, + WindowStart: observed, WindowEnd: observed, State: "matched", ObservedAt: now.Add(16 * time.Second), + } + ack, err := store.RecordSourceAlertTransition(ctx, token, transition, now.Add(16*time.Second)) + if err != nil || ack.SourceID != batch.SourceID || ack.RuleID != rule.ID || ack.Sequence != 7 || ack.Digest == "" || ack.Duplicate { + t.Fatalf("ack=%+v err=%v", ack, err) + } + replay, err := store.RecordSourceAlertTransition(ctx, token, transition, now.Add(17*time.Second)) + if err != nil || !replay.Duplicate || replay.Digest != ack.Digest { + t.Fatalf("replay=%+v err=%v", replay, err) + } + + conflict := transition + conflict.State = "clear" + if _, err = store.RecordSourceAlertTransition(ctx, token, conflict, now.Add(17*time.Second)); err == nil || !strings.Contains(err.Error(), "reused with different content") { + t.Fatalf("conflicting replay err=%v", err) + } + gap := transition + gap.Sequence = 9 + if _, err = store.RecordSourceAlertTransition(ctx, token, gap, now.Add(17*time.Second)); err == nil || !strings.Contains(err.Error(), "sequence gap") { + t.Fatalf("sequence gap err=%v", err) + } + wrongEvidence := transition + wrongEvidence.Sequence = 8 + wrongEvidence.SegmentDigest = strings.Repeat("b", 64) + if _, err = store.RecordSourceAlertTransition(ctx, token, wrongEvidence, now.Add(17*time.Second)); err == nil || !strings.Contains(err.Error(), "evidence is unavailable") { + t.Fatalf("wrong evidence err=%v", err) + } + + otherToken, err := store.CreateSource(ctx, "source-b", model.Scope{OrganizationID: scope.OrganizationID, ProjectID: "project-b", EnvironmentID: "production", ServiceID: "service-b"}) + if err != nil { + t.Fatal(err) + } + other := transition + other.Sequence = 8 + if _, err = store.RecordSourceAlertTransition(ctx, otherToken, other, now.Add(17*time.Second)); err == nil || !strings.Contains(err.Error(), "not scoped to this source") { + t.Fatalf("cross-source transition err=%v", err) + } +} + +func TestSourceAlertTransitionRequiresEvidenceCoveringItsWindow(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 18, 23, 30, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + saved, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", MaxRows: 100, Name: "Logs", Query: "logs | limit 10", Scope: ResourceScope{ProjectID: scope.ProjectID, EnvironmentID: scope.EnvironmentID, ServiceID: scope.ServiceID}}, now) + if err != nil { + t.Fatal(err) + } + rule, err := store.SaveAlertRule(ctx, AlertRuleInput{OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", SavedQueryID: saved.ID, Name: "Logs", Severity: "warning", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true}, now) + if err != nil { + t.Fatal(err) + } + first, last := now.Add(-time.Minute), now + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: last, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: first, Name: "first"}, {Timestamp: last, Name: "last"}}} + ingested, err := store.Ingest(ctx, token, batch, now) + if err != nil { + t.Fatal(err) + } + transition := model.AlertTransition{Version: model.AlertTransitionVersion, RuleID: rule.ID, RuleRevision: rule.Revision, AgentEpoch: strings.Repeat("c", 32), Sequence: 1, StreamID: batch.StreamID, BatchSequence: batch.Sequence, SegmentDigest: ingested.Digest, WindowStart: first.Add(time.Second), WindowEnd: last, State: "matched", ObservedAt: now} + if _, err = store.RecordSourceAlertTransition(ctx, token, transition, now); err == nil || !strings.Contains(err.Error(), "does not match transition") { + t.Fatalf("partial evidence window err=%v", err) + } +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go new file mode 100644 index 0000000..ed83f1d --- /dev/null +++ b/internal/storage/storage.go @@ -0,0 +1,1180 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" + "time" + + "gamertan.com/observatory/internal/model" + "gamertan.com/observatory/internal/segment" + _ "modernc.org/sqlite" +) + +const controlSchema = 11 + +const recoveryPageSize = 128 + +type Store struct { + root string + control *sql.DB + segments *segment.Store + locks sync.Map + projectionMu sync.Mutex + projectorMu sync.Mutex + projections map[string]projectionHandle + projectionWake chan struct{} +} + +type projectionHandle struct { + db *sql.DB + device uint64 + inode uint64 +} + +type Source struct { + ID string + Scope model.Scope + Active bool +} + +type Ack struct { + SourceID string `json:"source_id"` + StreamID string `json:"stream_id"` + Sequence uint64 `json:"sequence"` + Digest string `json:"digest"` + BatchDigest string `json:"batch_digest"` + Duplicate bool `json:"duplicate"` +} + +type Enrollment struct { + SourceID, CreatedByUserID string + Scope model.Scope + CreatedAt, ExpiresAt time.Time +} + +func Open(root string) (*Store, error) { + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return nil, errors.New("data root must be an absolute clean path") + } + if err := os.MkdirAll(root, 0o700); err != nil { + return nil, fmt.Errorf("create data root: %w", err) + } + info, err := os.Lstat(root) + if err != nil { + return nil, fmt.Errorf("inspect data root: %w", err) + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil, errors.New("data root must be a non-symlink directory") + } + if info.Mode().Perm()&0o077 != 0 { + return nil, errors.New("data root must not grant group or other permissions") + } + controlPath := filepath.Join(root, "control.sqlite") + if err := validateSQLiteFileSet(controlPath); err != nil { + return nil, fmt.Errorf("inspect control database: %w", err) + } + db, err := sql.Open("sqlite", controlPath) + if err != nil { + return nil, fmt.Errorf("open control database: %w", err) + } + db.SetMaxOpenConns(1) + if err := migrateControl(db); err != nil { + _ = db.Close() + return nil, err + } + if err := os.Chmod(controlPath, 0o600); err != nil { + _ = db.Close() + return nil, fmt.Errorf("set control database mode: %w", err) + } + segments, err := segment.New(root) + if err != nil { + _ = db.Close() + return nil, err + } + store := &Store{ + root: root, + control: db, + segments: segments, + projections: make(map[string]projectionHandle), + projectionWake: make(chan struct{}, 1), + } + if err = store.backfillSegmentRetentionMetadata(context.Background()); err != nil { + _ = db.Close() + return nil, err + } + return store, nil +} + +func (s *Store) Close() error { + s.projectionMu.Lock() + handles := make([]projectionHandle, 0, len(s.projections)) + for organizationID, handle := range s.projections { + handles = append(handles, handle) + delete(s.projections, organizationID) + } + s.projectionMu.Unlock() + errs := make([]error, 0, len(handles)+1) + for _, handle := range handles { + if err := handle.db.Close(); err != nil { + errs = append(errs, err) + } + } + if err := s.control.Close(); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) +} + +// EstimateOrganizationBytes returns a conservative whole-projection scan +// estimate. Query execution will refine this with index and time-window +// statistics; planning never accepts a client-supplied cost estimate. +func (s *Store) EstimateOrganizationBytes(organizationID string) (int64, error) { + if err := model.ValidateSourceID(organizationID); err != nil { + return 0, errors.New("invalid organization identifier") + } + path := filepath.Join(s.root, "organizations", organizationID, "projection.sqlite") + var total int64 + for _, candidate := range []string{path, path + "-wal"} { + info, err := os.Lstat(candidate) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return 0, fmt.Errorf("inspect organization projection: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return 0, errors.New("organization projection is not a regular file") + } + if info.Size() > math.MaxInt64-total { + return 0, errors.New("organization projection size overflow") + } + total += info.Size() + } + return total, nil +} + +func migrateControl(db *sql.DB) error { + statements := []string{ + `PRAGMA journal_mode=WAL`, + `PRAGMA synchronous=FULL`, + `PRAGMA busy_timeout=5000`, + `PRAGMA foreign_keys=ON`, + `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)`, + `INSERT INTO schema_version(version) SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM schema_version)`, + `CREATE TABLE IF NOT EXISTS sources ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL, + project_id TEXT NOT NULL, + environment_id TEXT NOT NULL, + service_id TEXT NOT NULL, + credential_digest BLOB NOT NULL UNIQUE, + active INTEGER NOT NULL CHECK(active IN (0,1)), + created_at TEXT NOT NULL, + rotated_at TEXT + )`, + `CREATE TABLE IF NOT EXISTS streams ( + source_id TEXT NOT NULL REFERENCES sources(id), + stream_id TEXT NOT NULL, + last_sequence INTEGER NOT NULL, + last_digest TEXT NOT NULL, + PRIMARY KEY(source_id, stream_id) + )`, + `CREATE TABLE IF NOT EXISTS segments ( + digest TEXT PRIMARY KEY, + organization_id TEXT NOT NULL, + source_id TEXT NOT NULL, + stream_id TEXT NOT NULL, + sequence INTEGER NOT NULL, + path TEXT NOT NULL UNIQUE, + compressed_bytes INTEGER NOT NULL, + uncompressed_bytes INTEGER NOT NULL, + committed_at TEXT NOT NULL, + projected_at TEXT, + UNIQUE(source_id, stream_id, sequence) + )`, + `CREATE TABLE IF NOT EXISTS source_enrollments ( + credential_digest BLOB PRIMARY KEY, + source_id TEXT NOT NULL UNIQUE, + organization_id TEXT NOT NULL, + project_id TEXT NOT NULL, + environment_id TEXT NOT NULL, + service_id TEXT NOT NULL, + created_by_user_id TEXT NOT NULL, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + used_at TEXT + )`, + `UPDATE schema_version SET version=2 WHERE version=1`, + `CREATE TABLE IF NOT EXISTS descriptor_proposals ( + organization_id TEXT NOT NULL, + signal TEXT NOT NULL, + field TEXT NOT NULL, + descriptor_json TEXT NOT NULL, + observed_values INTEGER NOT NULL CHECK(observed_values > 0), + estimated_bytes INTEGER NOT NULL CHECK(estimated_bytes >= 0), + example_queries_json TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('pending','activated','rejected')), + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + PRIMARY KEY(organization_id,signal,field) + )`, + `CREATE TABLE IF NOT EXISTS descriptor_proposal_segments ( + segment_digest TEXT NOT NULL, + organization_id TEXT NOT NULL, + signal TEXT NOT NULL, + field TEXT NOT NULL, + observed_values INTEGER NOT NULL CHECK(observed_values > 0), + estimated_bytes INTEGER NOT NULL CHECK(estimated_bytes >= 0), + PRIMARY KEY(segment_digest,organization_id,field) + )`, + `UPDATE schema_version SET version=3 WHERE version=2`, + `CREATE TABLE IF NOT EXISTS saved_queries ( + organization_id TEXT NOT NULL, + id TEXT NOT NULL, + version INTEGER NOT NULL CHECK(version=1), + revision INTEGER NOT NULL CHECK(revision >= 1), + name TEXT NOT NULL, + description TEXT NOT NULL, + query_text TEXT NOT NULL, + ast_json TEXT NOT NULL, + project_id TEXT NOT NULL, + environment_id TEXT NOT NULL, + service_id TEXT NOT NULL, + created_by TEXT NOT NULL, + updated_by TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY(organization_id,id), + UNIQUE(organization_id,name) + )`, + `CREATE TABLE IF NOT EXISTS dashboards ( + organization_id TEXT NOT NULL, + id TEXT NOT NULL, + version INTEGER NOT NULL CHECK(version=1), + revision INTEGER NOT NULL CHECK(revision >= 1), + slug TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + created_by TEXT NOT NULL, + updated_by TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY(organization_id,id), + UNIQUE(organization_id,slug) + )`, + `CREATE TABLE IF NOT EXISTS dashboard_panels ( + organization_id TEXT NOT NULL, + dashboard_id TEXT NOT NULL, + id TEXT NOT NULL, + position INTEGER NOT NULL CHECK(position >= 0 AND position < 64), + title TEXT NOT NULL, + visualization TEXT NOT NULL CHECK(visualization IN ('table','stat','timeseries')), + saved_query_id TEXT NOT NULL, + PRIMARY KEY(organization_id,dashboard_id,id), + UNIQUE(organization_id,dashboard_id,position), + FOREIGN KEY(organization_id,dashboard_id) REFERENCES dashboards(organization_id,id) ON DELETE CASCADE, + FOREIGN KEY(organization_id,saved_query_id) REFERENCES saved_queries(organization_id,id) ON DELETE RESTRICT + )`, + `UPDATE schema_version SET version=4 WHERE version=3`, + `CREATE TABLE IF NOT EXISTS alert_rules ( + organization_id TEXT NOT NULL, + id TEXT NOT NULL, + version INTEGER NOT NULL CHECK(version=1), + revision INTEGER NOT NULL CHECK(revision >= 1), + name TEXT NOT NULL, + description TEXT NOT NULL, + saved_query_id TEXT NOT NULL, + severity TEXT NOT NULL CHECK(severity IN ('information','warning','critical')), + minimum_matches INTEGER NOT NULL CHECK(minimum_matches BETWEEN 1 AND 100000), + required_consecutive INTEGER NOT NULL CHECK(required_consecutive BETWEEN 1 AND 10), + evaluation_interval_seconds INTEGER NOT NULL CHECK(evaluation_interval_seconds BETWEEN 15 AND 86400), + enabled INTEGER NOT NULL CHECK(enabled IN (0,1)), + last_evaluated_at TEXT, + next_evaluation_at TEXT NOT NULL, + last_result INTEGER, + last_error TEXT NOT NULL DEFAULT '', + created_by TEXT NOT NULL, + updated_by TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY(organization_id,id), + UNIQUE(organization_id,name), + FOREIGN KEY(organization_id,saved_query_id) REFERENCES saved_queries(organization_id,id) ON DELETE RESTRICT + )`, + `CREATE INDEX IF NOT EXISTS alert_rules_due ON alert_rules(enabled,next_evaluation_at,organization_id,id)`, + `CREATE TABLE IF NOT EXISTS incidents ( + organization_id TEXT NOT NULL, + id TEXT NOT NULL, + version INTEGER NOT NULL CHECK(version=1), + rule_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('pending','firing','acknowledged','silenced','resolved')), + severity TEXT NOT NULL CHECK(severity IN ('information','warning','critical')), + title TEXT NOT NULL, + consecutive_matches INTEGER NOT NULL CHECK(consecutive_matches >= 0), + started_at TEXT NOT NULL, + last_observed_at TEXT NOT NULL, + acknowledged_by TEXT, + acknowledged_at TEXT, + silenced_by TEXT, + silenced_until TEXT, + resolved_at TEXT, + updated_at TEXT NOT NULL, + PRIMARY KEY(organization_id,id), + FOREIGN KEY(organization_id,rule_id) REFERENCES alert_rules(organization_id,id) ON DELETE RESTRICT + )`, + `CREATE UNIQUE INDEX IF NOT EXISTS one_open_incident_per_rule ON incidents(organization_id,rule_id) WHERE state!='resolved'`, + `CREATE INDEX IF NOT EXISTS incidents_by_state ON incidents(organization_id,state,updated_at DESC,id)`, + `CREATE TABLE IF NOT EXISTS incident_events ( + organization_id TEXT NOT NULL, + incident_id TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK(sequence >= 1), + event TEXT NOT NULL CHECK(event IN ('opened','promoted','acknowledged','silenced','unsilenced','resolved')), + actor TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY(organization_id,incident_id,sequence), + FOREIGN KEY(organization_id,incident_id) REFERENCES incidents(organization_id,id) ON DELETE CASCADE + )`, + `UPDATE schema_version SET version=5 WHERE version=4`, + `CREATE TABLE IF NOT EXISTS push_endpoints ( + id TEXT NOT NULL, + user_id TEXT NOT NULL, + endpoint TEXT NOT NULL, + endpoint_digest BLOB NOT NULL UNIQUE, + p256dh BLOB NOT NULL, + auth_secret BLOB NOT NULL, + active INTEGER NOT NULL CHECK(active IN (0,1)), + failure_count INTEGER NOT NULL CHECK(failure_count >= 0), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + last_sent_at TEXT, + PRIMARY KEY(id) + )`, + `CREATE TABLE IF NOT EXISTS push_subscriptions ( + organization_id TEXT NOT NULL, + id TEXT NOT NULL, + user_id TEXT NOT NULL, + endpoint_id TEXT NOT NULL REFERENCES push_endpoints(id) ON DELETE CASCADE, + created_at TEXT NOT NULL, + PRIMARY KEY(organization_id,id), + UNIQUE(organization_id,user_id,endpoint_id) + )`, + `CREATE INDEX IF NOT EXISTS push_subscriptions_by_organization ON push_subscriptions(organization_id,user_id,id)`, + `CREATE INDEX IF NOT EXISTS push_subscriptions_by_endpoint ON push_subscriptions(endpoint_id,organization_id,id)`, + `UPDATE schema_version SET version=6 WHERE version=5`, + } + for _, statement := range statements { + if _, err := db.Exec(statement); err != nil { + return fmt.Errorf("migrate control database: %w", err) + } + } + var version int + if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil { + return fmt.Errorf("read control schema: %w", err) + } + if version == 6 { + if err := migrateControlRetention(db); err != nil { + return err + } + version = 7 + } + if version == 7 { + if err := migrateControlForensicRetention(db); err != nil { + return err + } + version = 8 + } + if version == 8 { + if err := migrateControlSourceAlertTransitions(db); err != nil { + return err + } + version = 9 + } + if version == 9 { + if err := migrateControlBatchMetadata(db); err != nil { + return err + } + version = 10 + } + if version == 10 { + if err := migrateControlBatchEnvelopes(db); err != nil { + return err + } + version = 11 + } + if version != controlSchema { + return fmt.Errorf("unsupported control schema %d", version) + } + return nil +} + +func (s *Store) CreateSource(ctx context.Context, id string, scope model.Scope) (string, error) { + if err := scope.Validate(); err != nil { + return "", err + } + if err := model.ValidateSourceID(id); err != nil { + return "", err + } + token, err := sourceCredential(id) + if err != nil { + return "", err + } + digest := sha256.Sum256([]byte(token)) + _, err = s.control.ExecContext(ctx, `INSERT INTO sources(id, organization_id, project_id, environment_id, service_id, credential_digest, active, created_at) VALUES(?,?,?,?,?,?,1,?)`, id, scope.OrganizationID, scope.ProjectID, scope.EnvironmentID, scope.ServiceID, digest[:], time.Now().UTC().Format(time.RFC3339Nano)) + if err != nil { + return "", fmt.Errorf("create source: %w", err) + } + return token, nil +} + +func (s *Store) CreateEnrollment(ctx context.Context, id string, scope model.Scope, createdBy string, lifetime time.Duration, now time.Time) (string, Enrollment, error) { + if err := model.ValidateSourceID(id); err != nil { + return "", Enrollment{}, err + } + if err := scope.Validate(); err != nil { + return "", Enrollment{}, err + } + if err := model.ValidateSourceID(createdBy); err != nil || lifetime < 5*time.Minute || lifetime > 24*time.Hour || now.IsZero() { + return "", Enrollment{}, errors.New("invalid source enrollment") + } + secret := make([]byte, 32) + if _, err := rand.Read(secret); err != nil { + return "", Enrollment{}, errors.New("cryptographic randomness unavailable") + } + token := "obse1." + hex.EncodeToString(secret) + digest := sha256.Sum256([]byte(token)) + enrollment := Enrollment{SourceID: id, Scope: scope, CreatedByUserID: createdBy, CreatedAt: now.UTC(), ExpiresAt: now.UTC().Add(lifetime)} + _, err := s.control.ExecContext(ctx, `INSERT INTO source_enrollments(credential_digest,source_id,organization_id,project_id,environment_id,service_id,created_by_user_id,created_at,expires_at) VALUES(?,?,?,?,?,?,?,?,?)`, digest[:], id, scope.OrganizationID, scope.ProjectID, scope.EnvironmentID, scope.ServiceID, createdBy, enrollment.CreatedAt.Format(time.RFC3339Nano), enrollment.ExpiresAt.Format(time.RFC3339Nano)) + if err != nil { + return "", Enrollment{}, fmt.Errorf("create source enrollment: %w", err) + } + return token, enrollment, nil +} + +func (s *Store) RedeemEnrollment(ctx context.Context, token string, now time.Time) (Enrollment, string, error) { + if len(token) != len("obse1.")+64 || !strings.HasPrefix(token, "obse1.") || strings.ContainsAny(token, " \t\r\n") || now.IsZero() { + return Enrollment{}, "", errors.New("invalid or expired source enrollment") + } + digest := sha256.Sum256([]byte(token)) + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return Enrollment{}, "", err + } + defer tx.Rollback() + var enrollment Enrollment + var created, expires string + err = tx.QueryRowContext(ctx, `SELECT source_id,organization_id,project_id,environment_id,service_id,created_by_user_id,created_at,expires_at FROM source_enrollments WHERE credential_digest=? AND used_at IS NULL`, digest[:]).Scan(&enrollment.SourceID, &enrollment.Scope.OrganizationID, &enrollment.Scope.ProjectID, &enrollment.Scope.EnvironmentID, &enrollment.Scope.ServiceID, &enrollment.CreatedByUserID, &created, &expires) + if errors.Is(err, sql.ErrNoRows) { + return Enrollment{}, "", errors.New("invalid or expired source enrollment") + } + if err != nil { + return Enrollment{}, "", err + } + enrollment.CreatedAt, err = time.Parse(time.RFC3339Nano, created) + if err != nil { + return Enrollment{}, "", errors.New("stored source enrollment is invalid") + } + enrollment.ExpiresAt, err = time.Parse(time.RFC3339Nano, expires) + if err != nil { + return Enrollment{}, "", errors.New("stored source enrollment is invalid") + } + if !now.UTC().Before(enrollment.ExpiresAt) { + return Enrollment{}, "", errors.New("invalid or expired source enrollment") + } + credential, err := sourceCredential(enrollment.SourceID) + if err != nil { + return Enrollment{}, "", err + } + credentialDigest := sha256.Sum256([]byte(credential)) + if _, err = tx.ExecContext(ctx, `INSERT INTO sources(id,organization_id,project_id,environment_id,service_id,credential_digest,active,created_at) VALUES(?,?,?,?,?,?,1,?)`, enrollment.SourceID, enrollment.Scope.OrganizationID, enrollment.Scope.ProjectID, enrollment.Scope.EnvironmentID, enrollment.Scope.ServiceID, credentialDigest[:], now.UTC().Format(time.RFC3339Nano)); err != nil { + return Enrollment{}, "", errors.New("source enrollment failed") + } + result, err := tx.ExecContext(ctx, `UPDATE source_enrollments SET used_at=? WHERE credential_digest=? AND used_at IS NULL`, now.UTC().Format(time.RFC3339Nano), digest[:]) + if err != nil { + return Enrollment{}, "", err + } + if changed, _ := result.RowsAffected(); changed != 1 { + return Enrollment{}, "", errors.New("invalid or expired source enrollment") + } + if err = tx.Commit(); err != nil { + return Enrollment{}, "", err + } + return enrollment, credential, nil +} + +func (s *Store) CancelEnrollment(ctx context.Context, token string) error { + if len(token) != len("obse1.")+64 || !strings.HasPrefix(token, "obse1.") || strings.ContainsAny(token, " \t\r\n") { + return errors.New("invalid source enrollment") + } + digest := sha256.Sum256([]byte(token)) + result, err := s.control.ExecContext(ctx, `DELETE FROM source_enrollments WHERE credential_digest=? AND used_at IS NULL`, digest[:]) + if err != nil { + return err + } + if changed, _ := result.RowsAffected(); changed != 1 { + return errors.New("source enrollment not found") + } + return nil +} + +func sourceCredential(id string) (string, error) { + secret := make([]byte, 32) + if _, err := rand.Read(secret); err != nil { + return "", errors.New("cryptographic randomness unavailable") + } + return "obs1." + id + "." + hex.EncodeToString(secret), nil +} + +func (s *Store) Authenticate(ctx context.Context, token string) (Source, error) { + if len(token) < 48 || len(token) > 512 || !strings.HasPrefix(token, "obs1.") { + return Source{}, errors.New("invalid source credential") + } + digest := sha256.Sum256([]byte(token)) + var source Source + var active int + err := s.control.QueryRowContext(ctx, `SELECT id, organization_id, project_id, environment_id, service_id, active FROM sources WHERE credential_digest = ?`, digest[:]).Scan(&source.ID, &source.Scope.OrganizationID, &source.Scope.ProjectID, &source.Scope.EnvironmentID, &source.Scope.ServiceID, &active) + if errors.Is(err, sql.ErrNoRows) { + return Source{}, errors.New("invalid source credential") + } + if err != nil { + return Source{}, fmt.Errorf("authenticate source: %w", err) + } + source.Active = active == 1 + if !source.Active { + return Source{}, errors.New("source credential revoked") + } + return source, nil +} + +func (s *Store) RevokeSource(ctx context.Context, id string) error { + result, err := s.control.ExecContext(ctx, `UPDATE sources SET active=0, rotated_at=? WHERE id=? AND active=1`, time.Now().UTC().Format(time.RFC3339Nano), id) + if err != nil { + return fmt.Errorf("revoke source: %w", err) + } + n, _ := result.RowsAffected() + if n != 1 { + return errors.New("active source not found") + } + return nil +} + +func (s *Store) Ingest(ctx context.Context, token string, batch model.Batch, now time.Time) (Ack, error) { + source, err := s.Authenticate(ctx, token) + if err != nil { + return Ack{}, err + } + if batch.SourceID != source.ID { + return Ack{}, errors.New("batch source does not match credential") + } + lock := s.sourceLock(source.ID) + lock.Lock() + defer lock.Unlock() + return s.ingestAuthenticated(ctx, source, batch, nil, now) +} + +// IngestNative validates the exact encoded request against its transport +// envelope before committing a new batch. A concurrently acknowledged exact +// replay is returned without recompressing or rewriting raw evidence. +func (s *Store) IngestNative(ctx context.Context, token string, batch model.Batch, envelope model.BatchEnvelope, encoded []byte, now time.Time) (Ack, error) { + if err := envelope.Match(batch, encoded); err != nil { + return Ack{}, err + } + source, err := s.Authenticate(ctx, token) + if err != nil { + return Ack{}, err + } + if batch.SourceID != source.ID { + return Ack{}, errors.New("batch source does not match credential") + } + lock := s.sourceLock(source.ID) + lock.Lock() + defer lock.Unlock() + if ack, exact, checkErr := s.checkEnvelope(ctx, source, envelope); checkErr != nil { + return Ack{}, checkErr + } else if exact { + return ack, nil + } + return s.ingestAuthenticated(ctx, source, batch, &envelope, now) +} + +func (s *Store) IngestAuto(ctx context.Context, token, streamID string, signal model.Signal, records []model.Observation, now time.Time) (Ack, error) { + source, err := s.Authenticate(ctx, token) + if err != nil { + return Ack{}, err + } + if err = model.ValidateStreamID(streamID); err != nil { + return Ack{}, err + } + lock := s.sourceLock(source.ID) + lock.Lock() + defer lock.Unlock() + lastSequence, _, found, err := s.watermark(ctx, source.ID, streamID) + if err != nil { + return Ack{}, err + } + if lastSequence == ^uint64(0) { + return Ack{}, errors.New("stream sequence exhausted") + } + sequence := uint64(1) + if found { + sequence = lastSequence + 1 + } + if len(records) == 0 { + return Ack{}, errors.New("automatic ingestion requires records") + } + // Derive the batch observation time from its records so a retry of the + // same OTLP payload produces the same content-addressed segment even if a + // prior attempt stopped after the atomic raw commit. + observedAt := records[0].Timestamp + for _, record := range records[1:] { + if record.Timestamp.After(observedAt) { + observedAt = record.Timestamp + } + } + batch := model.Batch{Version: model.BatchVersion, SourceID: source.ID, StreamID: streamID, Sequence: sequence, ObservedAt: observedAt.UTC(), Signal: signal, Records: records} + return s.ingestAuthenticated(ctx, source, batch, nil, now) +} + +func (s *Store) sourceLock(sourceID string) *sync.Mutex { + return s.namedLock("source:" + sourceID) +} + +func (s *Store) ingestAuthenticated(ctx context.Context, source Source, batch model.Batch, envelope *model.BatchEnvelope, now time.Time) (Ack, error) { + if err := batch.Validate(now); err != nil { + return Ack{}, err + } + batchDigest, err := batch.Digest() + if err != nil { + return Ack{}, err + } + if err := validateMetricRollupCardinality(batch); err != nil { + return Ack{}, err + } + lastSequence, lastDigest, found, err := s.watermark(ctx, source.ID, batch.StreamID) + if err != nil { + return Ack{}, err + } + if found && batch.Sequence < lastSequence { + return Ack{}, errors.New("sequence replay is older than acknowledged watermark") + } + if (!found && batch.Sequence != 1) || (found && batch.Sequence > lastSequence+1) { + return Ack{}, errors.New("sequence gap") + } + committed, err := s.segments.Commit(source.Scope, batch) + if err != nil { + return Ack{}, err + } + if found && batch.Sequence == lastSequence { + if committed.Digest != lastDigest { + if deleteErr := s.segments.Delete(committed.Path, committed.Digest); deleteErr != nil { + return Ack{}, fmt.Errorf("acknowledged sequence reused with different content; remove rejected raw object: %w", deleteErr) + } + return Ack{}, errors.New("acknowledged sequence reused with different content") + } + if envelope != nil { + if err = s.backfillAcknowledgedEnvelope(ctx, batch, committed.Digest, *envelope); err != nil { + return Ack{}, err + } + } + return Ack{SourceID: source.ID, StreamID: batch.StreamID, Sequence: batch.Sequence, Digest: committed.Digest, BatchDigest: batchDigest, Duplicate: true}, nil + } + if err := s.admitCommittedEnvelope(ctx, source.Scope, batch, committed, envelope, now); err != nil { + return Ack{}, err + } + s.notifyProjector() + return Ack{SourceID: source.ID, StreamID: batch.StreamID, Sequence: batch.Sequence, Digest: committed.Digest, BatchDigest: batchDigest}, nil +} + +func (s *Store) notifyProjector() { + select { + case s.projectionWake <- struct{}{}: + default: + } +} + +func (s *Store) watermark(ctx context.Context, sourceID, streamID string) (uint64, string, bool, error) { + var sequence uint64 + var digest string + err := s.control.QueryRowContext(ctx, `SELECT last_sequence, last_digest FROM streams WHERE source_id=? AND stream_id=?`, sourceID, streamID).Scan(&sequence, &digest) + if errors.Is(err, sql.ErrNoRows) { + return 0, "", false, nil + } + if err != nil { + return 0, "", false, fmt.Errorf("read stream watermark: %w", err) + } + return sequence, digest, true, nil +} + +func (s *Store) recordCommitted(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed) error { + return s.recordCommittedAt(ctx, scope, batch, committed, time.Now().UTC()) +} + +func (s *Store) recordCommittedAt(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, committedAt time.Time) error { + return s.recordCommittedAtEnvelope(ctx, scope, batch, committed, nil, committedAt) +} + +func (s *Store) recordCommittedAtEnvelope(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, envelope *model.BatchEnvelope, committedAt time.Time) error { + if committedAt.IsZero() { + return errors.New("committed segment time is required") + } + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin committed segment record: %w", err) + } + defer tx.Rollback() + if err = recordCommittedTx(ctx, tx, scope, batch, committed, committedAt); err != nil { + return err + } + if err = advanceStreamTx(ctx, tx, batch, committed.Digest, envelope); err != nil { + return err + } + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit segment record: %w", err) + } + return nil +} + +func recordCommittedTx(ctx context.Context, tx *sql.Tx, scope model.Scope, batch model.Batch, committed segment.Committed, committedAt time.Time) error { + first, last := observationRange(batch) + _, err := tx.ExecContext(ctx, `INSERT INTO segments(digest, organization_id, source_id, stream_id, sequence, path, compressed_bytes, uncompressed_bytes, committed_at, signal, first_observed_at, last_observed_at, record_count) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`, committed.Digest, scope.OrganizationID, batch.SourceID, batch.StreamID, batch.Sequence, committed.Path, committed.Compressed, committed.Uncompressed, committedAt.UTC().Format(time.RFC3339Nano), batch.Signal, first.Format(time.RFC3339Nano), last.Format(time.RFC3339Nano), len(batch.Records)) + if err != nil { + return fmt.Errorf("record committed segment: %w", err) + } + return nil +} + +func observationRange(batch model.Batch) (time.Time, time.Time) { + first, last := batch.Records[0].Timestamp.UTC(), batch.Records[0].Timestamp.UTC() + for _, observation := range batch.Records[1:] { + timestamp := observation.Timestamp.UTC() + if timestamp.Before(first) { + first = timestamp + } + if timestamp.After(last) { + last = timestamp + } + } + return first, last +} + +func (s *Store) project(ctx context.Context, scope model.Scope, batch model.Batch, digest string) error { + lock := s.namedLock("organization:" + scope.OrganizationID) + lock.Lock() + defer lock.Unlock() + + db, err := s.projection(ctx, scope.OrganizationID) + if err != nil { + return err + } + return projectWithDB(ctx, db, scope, batch, digest) +} + +func projectAt(ctx context.Context, path string, scope model.Scope, batch model.Batch, digest string) error { + db, err := openProjection(ctx, path) + if err != nil { + return err + } + defer db.Close() + return projectWithDB(ctx, db, scope, batch, digest) +} + +type projectionItem struct { + scope model.Scope + batch model.Batch + digest string +} + +func projectWithDB(ctx context.Context, db *sql.DB, scope model.Scope, batch model.Batch, digest string) error { + return projectGroupWithDB(ctx, db, []projectionItem{{scope: scope, batch: batch, digest: digest}}) +} + +func projectGroupWithDB(ctx context.Context, db *sql.DB, items []projectionItem) error { + if len(items) == 0 { + return errors.New("projection group is empty") + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin projection: %w", err) + } + defer tx.Rollback() + activeVersion, activeRegistry, activeDescriptors, err := activeProjection(ctx, tx) + if err != nil { + return err + } + insert, err := tx.PrepareContext(ctx, `INSERT OR IGNORE INTO observations(organization_id, project_id, environment_id, service_id, source_id, stream_id, sequence, record_index, signal, timestamp, name, severity, body, value, trace_id, span_id, correlation_id, attributes_json, segment_digest) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) + if err != nil { + return fmt.Errorf("prepare observation projection: %w", err) + } + defer insert.Close() + for _, item := range items { + batch, scope, digest := item.batch, item.scope, item.digest + var observationBytes []int64 + if batch.Signal == model.SignalLogs { + observationBytes = make([]int64, 0, len(batch.Records)) + } + for index, observation := range batch.Records { + attributes, marshalErr := json.Marshal(observation.Attributes) + if marshalErr != nil { + return fmt.Errorf("encode observation attributes: %w", marshalErr) + } + _, execErr := insert.ExecContext(ctx, scope.OrganizationID, scope.ProjectID, scope.EnvironmentID, scope.ServiceID, batch.SourceID, batch.StreamID, batch.Sequence, index, batch.Signal, observation.Timestamp.UTC().Format(time.RFC3339Nano), observation.Name, observation.Severity, observation.Body, observation.Value, observation.TraceID, observation.SpanID, observation.CorrelationID, string(attributes), digest) + if execErr != nil { + return fmt.Errorf("project observation: %w", execErr) + } + if err = indexProjectedObservation(ctx, tx, activeVersion, activeDescriptors, batch, index, observation); err != nil { + return err + } + if batch.Signal == model.SignalLogs { + observationBytes = append(observationBytes, projectedObservationBytes(scope, batch, observation, len(attributes))) + } + } + if err = projectMetricRollups(ctx, tx, scope, batch, digest, activeRegistry); err != nil { + return err + } + if err = projectLogRollups(ctx, tx, scope, batch, digest, observationBytes); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit projection: %w", err) + } + return nil +} + +func (s *Store) projection(ctx context.Context, organizationID string) (*sql.DB, error) { + path := filepath.Join(s.root, "organizations", organizationID, "projection.sqlite") + s.projectionMu.Lock() + handle, exists := s.projections[organizationID] + s.projectionMu.Unlock() + if exists { + device, inode, err := projectionIdentity(path) + if err != nil || device != handle.device || inode != handle.inode { + return nil, errors.New("organization projection identity changed") + } + return handle.db, nil + } + db, err := openProjection(ctx, path) + if err != nil { + return nil, err + } + device, inode, err := projectionIdentity(path) + if err != nil { + _ = db.Close() + return nil, err + } + s.projectionMu.Lock() + s.projections[organizationID] = projectionHandle{db: db, device: device, inode: inode} + s.projectionMu.Unlock() + return db, nil +} + +func (s *Store) closeProjection(organizationID string) error { + s.projectionMu.Lock() + handle, exists := s.projections[organizationID] + if exists { + delete(s.projections, organizationID) + } + s.projectionMu.Unlock() + if !exists { + return nil + } + return handle.db.Close() +} + +func projectionIdentity(path string) (uint64, uint64, error) { + if err := validateSQLiteFileSet(path); err != nil { + return 0, 0, fmt.Errorf("inspect organization projection: %w", err) + } + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return 0, 0, errors.New("organization projection must be a regular non-symlink file") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Nlink != 1 { + return 0, 0, errors.New("organization projection identity is unavailable") + } + return uint64(stat.Dev), uint64(stat.Ino), nil +} + +func openProjection(ctx context.Context, path string) (*sql.DB, error) { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("create organization store: %w", err) + } + for _, directory := range []string{filepath.Dir(dir), dir} { + info, err := os.Lstat(directory) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return nil, errors.New("organization store path must contain private non-symlink directories") + } + } + if err := validateSQLiteFileSet(path); err != nil { + return nil, fmt.Errorf("inspect organization projection: %w", err) + } + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, fmt.Errorf("open organization projection: %w", err) + } + db.SetMaxOpenConns(1) + if _, err := db.ExecContext(ctx, `PRAGMA journal_mode=WAL`); err != nil { + _ = db.Close() + return nil, fmt.Errorf("configure organization projection: %w", err) + } + for _, statement := range []string{`PRAGMA synchronous=FULL`, `PRAGMA busy_timeout=5000`, `PRAGMA foreign_keys=ON`} { + if _, err := db.ExecContext(ctx, statement); err != nil { + _ = db.Close() + return nil, fmt.Errorf("configure organization projection: %w", err) + } + } + if _, err := db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS observations ( + organization_id TEXT NOT NULL, + project_id TEXT NOT NULL, + environment_id TEXT NOT NULL, + service_id TEXT NOT NULL, + source_id TEXT NOT NULL, + stream_id TEXT NOT NULL, + sequence INTEGER NOT NULL, + record_index INTEGER NOT NULL, + signal TEXT NOT NULL, + timestamp TEXT NOT NULL, + name TEXT NOT NULL, + severity TEXT, + body TEXT, + value REAL, + trace_id TEXT, + span_id TEXT, + correlation_id TEXT, + attributes_json TEXT NOT NULL, + segment_digest TEXT NOT NULL, + PRIMARY KEY(source_id, stream_id, sequence, record_index) + )`); err != nil { + _ = db.Close() + return nil, fmt.Errorf("migrate organization projection: %w", err) + } + for _, statement := range []string{ + `CREATE INDEX IF NOT EXISTS observations_signal_time ON observations(signal,timestamp)`, + `CREATE INDEX IF NOT EXISTS observations_scope ON observations(project_id,environment_id,service_id,signal,timestamp)`, + `CREATE INDEX IF NOT EXISTS observations_environment ON observations(environment_id,signal,timestamp)`, + `CREATE INDEX IF NOT EXISTS observations_service ON observations(service_id,signal,timestamp)`, + `CREATE INDEX IF NOT EXISTS observations_name ON observations(signal,name,timestamp)`, + `CREATE INDEX IF NOT EXISTS observations_severity ON observations(signal,severity,timestamp)`, + `CREATE INDEX IF NOT EXISTS observations_trace ON observations(trace_id) WHERE trace_id IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS observations_span ON observations(span_id) WHERE span_id IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS observations_correlation ON observations(correlation_id) WHERE correlation_id IS NOT NULL`, + } { + if _, err := db.ExecContext(ctx, statement); err != nil { + _ = db.Close() + return nil, fmt.Errorf("index organization projection: %w", err) + } + } + if err := ensureProjectionMetadata(ctx, db); err != nil { + _ = db.Close() + return nil, err + } + if err := ensureMetricRollups(ctx, db); err != nil { + _ = db.Close() + return nil, err + } + if err := ensureBaseIndexes(ctx, db); err != nil { + _ = db.Close() + return nil, err + } + if err := ensureLogRollups(ctx, db); err != nil { + _ = db.Close() + return nil, err + } + if err := os.Chmod(path, 0o600); err != nil { + _ = db.Close() + return nil, fmt.Errorf("set organization database mode: %w", err) + } + return db, nil +} + +func validateSQLiteFileSet(path string) error { + for _, candidate := range []string{path, path + "-wal", path + "-shm"} { + info, err := os.Lstat(candidate) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("SQLite file must be a regular non-symlink file") + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Nlink != 1 { + return errors.New("SQLite file must not have additional hard links") + } + } + return nil +} + +func (s *Store) markProjected(ctx context.Context, batch model.Batch, digest string) error { + tx, err := s.control.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin acknowledgement: %w", err) + } + defer tx.Rollback() + if err = markProjectedTx(ctx, tx, batch, digest, time.Now().UTC()); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit acknowledgement: %w", err) + } + return nil +} + +func markProjectedTx(ctx context.Context, tx *sql.Tx, batch model.Batch, digest string, projectedAt time.Time) error { + result, err := tx.ExecContext(ctx, `UPDATE segments SET projected_at=? WHERE digest=? AND projected_at IS NULL`, projectedAt.UTC().Format(time.RFC3339Nano), digest) + if err != nil { + return fmt.Errorf("mark segment projected: %w", err) + } + n, _ := result.RowsAffected() + if n != 1 { + return errors.New("committed segment state changed before acknowledgement") + } + return nil +} + +func advanceStreamTx(ctx context.Context, tx *sql.Tx, batch model.Batch, digest string, envelope *model.BatchEnvelope) error { + var lastSequence uint64 + var lastDigest string + err := tx.QueryRowContext(ctx, `SELECT last_sequence,last_digest FROM streams WHERE source_id=? AND stream_id=?`, batch.SourceID, batch.StreamID).Scan(&lastSequence, &lastDigest) + if errors.Is(err, sql.ErrNoRows) { + if batch.Sequence != 1 { + return errors.New("sequence gap while accepting committed segment") + } + } else if err != nil { + return fmt.Errorf("read stream watermark while accepting committed segment: %w", err) + } else { + if batch.Sequence == lastSequence && digest == lastDigest { + return nil + } + if lastSequence == ^uint64(0) || batch.Sequence != lastSequence+1 { + return errors.New("sequence gap while accepting committed segment") + } + } + batchDigest, wireDigest, signal, recordCount, encodedBytes, first, last := envelopeSQL(envelope) + _, err = tx.ExecContext(ctx, `INSERT INTO streams(source_id,stream_id,last_sequence,last_digest,last_batch_digest,last_wire_digest,last_signal,last_record_count,last_encoded_bytes,last_first_observed_at,last_last_observed_at) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_id,stream_id) DO UPDATE SET last_sequence=excluded.last_sequence,last_digest=excluded.last_digest,last_batch_digest=excluded.last_batch_digest,last_wire_digest=excluded.last_wire_digest,last_signal=excluded.last_signal,last_record_count=excluded.last_record_count,last_encoded_bytes=excluded.last_encoded_bytes,last_first_observed_at=excluded.last_first_observed_at,last_last_observed_at=excluded.last_last_observed_at`, batch.SourceID, batch.StreamID, batch.Sequence, digest, batchDigest, wireDigest, signal, recordCount, encodedBytes, first, last) + if err != nil { + return fmt.Errorf("advance stream watermark: %w", err) + } + return nil +} + +// RecoverRaw reconciles immutable raw objects with the control catalog. It +// deliberately does not project telemetry, so a large projection backlog +// cannot delay server readiness. +func (s *Store) RecoverRaw(ctx context.Context) error { + if err := s.finishInterruptedArchival(ctx); err != nil { + return err + } + if err := s.finishInterruptedRetention(ctx); err != nil { + return err + } + lookup, err := s.control.PrepareContext(ctx, `SELECT organization_id,source_id,stream_id,sequence,path,compressed_bytes,tier FROM segments WHERE digest=?`) + if err != nil { + return fmt.Errorf("prepare recovered segment lookup: %w", err) + } + var missing []segment.Metadata + walkErr := s.segments.WalkMetadata(func(metadata segment.Metadata) error { + var catalog segment.Metadata + var tier string + catalog.Digest = metadata.Digest + lookupErr := lookup.QueryRowContext(ctx, metadata.Digest).Scan(&catalog.OrganizationID, &catalog.SourceID, &catalog.StreamID, &catalog.Sequence, &catalog.Path, &catalog.Compressed, &tier) + if lookupErr == nil { + if tier != "hot" || catalog != metadata { + return errors.New("catalogued segment metadata does not match raw object") + } + return nil + } + if !errors.Is(lookupErr, sql.ErrNoRows) { + return fmt.Errorf("inspect recovered segment: %w", lookupErr) + } + missing = append(missing, metadata) + return nil + }) + if closeErr := lookup.Close(); walkErr == nil && closeErr != nil { + walkErr = fmt.Errorf("close recovered segment lookup: %w", closeErr) + } + if walkErr != nil { + return walkErr + } + sort.Slice(missing, func(i, j int) bool { + left, right := missing[i], missing[j] + if left.SourceID != right.SourceID { + return left.SourceID < right.SourceID + } + if left.StreamID != right.StreamID { + return left.StreamID < right.StreamID + } + return left.Sequence < right.Sequence + }) + for _, metadata := range missing { + entry, readErr := s.segments.ReadEntry(metadata) + if readErr != nil { + return readErr + } + source, sourceErr := s.sourceByID(ctx, entry.Batch.SourceID) + if sourceErr != nil { + return sourceErr + } + if validateErr := entry.Batch.Validate(entry.Batch.ObservedAt); validateErr != nil { + return fmt.Errorf("validate recovered segment: %w", validateErr) + } + if validateErr := validateMetricRollupCardinality(entry.Batch); validateErr != nil { + return fmt.Errorf("validate recovered metric segment: %w", validateErr) + } + if source.Scope.OrganizationID != entry.OrganizationID { + return errors.New("recovered segment organization does not match enrolled source") + } + if admitErr := s.admitCommitted(ctx, source.Scope, entry.Batch, entry.Committed, time.Now().UTC()); admitErr != nil { + if errors.Is(admitErr, ErrOrganizationStorageQuotaExceeded) { + continue + } + return admitErr + } + s.notifyProjector() + } + return nil +} + +// Recover performs full offline recovery. The server uses RecoverRaw and a +// background projector; check and migration commands retain this blocking +// form so they can prove every durable segment is queryable before returning. +func (s *Store) Recover(ctx context.Context) error { + if err := s.RecoverRaw(ctx); err != nil { + return err + } + for { + report, err := s.ProjectPending(ctx) + if err != nil { + return err + } + if report.ProjectedSegments == 0 { + return nil + } + } +} + +func (s *Store) sourceByID(ctx context.Context, id string) (Source, error) { + var source Source + var active int + err := s.control.QueryRowContext(ctx, `SELECT id, organization_id, project_id, environment_id, service_id, active FROM sources WHERE id=?`, id).Scan(&source.ID, &source.Scope.OrganizationID, &source.Scope.ProjectID, &source.Scope.EnvironmentID, &source.Scope.ServiceID, &active) + if err != nil { + return Source{}, fmt.Errorf("load source: %w", err) + } + source.Active = active == 1 + return source, nil +} diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go new file mode 100644 index 0000000..45ebb57 --- /dev/null +++ b/internal/storage/storage_test.go @@ -0,0 +1,516 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package storage + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "sort" + "sync" + "testing" + "time" + + "gamertan.com/observatory/internal/model" +) + +func TestNativeEnvelopeReplayUsesBatchIdentityAndAllowsOverlappingTime(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + token, err := store.CreateSource(ctx, "source-framed", model.Scope{OrganizationID: "organization", ProjectID: "project", EnvironmentID: "production", ServiceID: "service"}) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 18, 20, 0, 0, 0, time.UTC) + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-framed", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}} + body, _ := json.Marshal(batch) + envelope, _ := batch.Envelope(body) + ack, err := store.IngestNative(ctx, token, batch, envelope, body, now) + if err != nil || ack.Duplicate { + t.Fatalf("ack=%+v err=%v", ack, err) + } + preflight, exact, err := store.CheckNativeReplay(ctx, token, envelope) + if err != nil || !exact || !preflight.Duplicate || preflight.Digest != ack.Digest { + t.Fatalf("preflight=%+v exact=%t err=%v", preflight, exact, err) + } + // A known exact retry remains acknowledgeable after its original ingest + // clock window; the immutable bytes and persisted envelope are authoritative. + confirmed, err := store.ConfirmNativeReplay(ctx, token, envelope) + if err != nil || !confirmed.Duplicate || confirmed.BatchDigest != envelope.BatchDigest { + t.Fatalf("confirmed=%+v err=%v", confirmed, err) + } + conflict := envelope + conflict.RecordCount++ + if _, _, err = store.CheckNativeReplay(ctx, token, conflict); err == nil { + t.Fatal("same sequence with conflicting envelope was accepted") + } + + // A second batch may overlap the first batch's timestamps. Time is a + // partition hint, not a deduplication key. + batch.Sequence = 2 + batch.ObservedAt = now.Add(time.Second) + body, _ = json.Marshal(batch) + envelope, _ = batch.Envelope(body) + if _, err = store.IngestNative(ctx, token, batch, envelope, body, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + var batchDigest, wireDigest string + var recordCount int + if err = store.control.QueryRow(`SELECT last_batch_digest,last_wire_digest,last_record_count FROM streams WHERE source_id=? AND stream_id=?`, batch.SourceID, batch.StreamID).Scan(&batchDigest, &wireDigest, &recordCount); err != nil || batchDigest != envelope.BatchDigest || wireDigest != envelope.WireDigest || recordCount != 1 { + t.Fatalf("batch=%q wire=%q count=%d err=%v", batchDigest, wireDigest, recordCount, err) + } +} + +func TestIngestAutoSerializesConcurrentSequenceAssignment(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + token, err := store.CreateSource(ctx, "source-auto", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + const requests = 16 + results := make(chan Ack, requests) + errors := make(chan error, requests) + var group sync.WaitGroup + for index := 0; index < requests; index++ { + group.Add(1) + go func() { + defer group.Done() + ack, ingestErr := store.IngestAuto(ctx, token, "otlp-logs", model.SignalLogs, []model.Observation{{Timestamp: now, Name: "http.request"}}, now) + if ingestErr != nil { + errors <- ingestErr + return + } + results <- ack + }() + } + group.Wait() + close(results) + close(errors) + for ingestErr := range errors { + t.Errorf("ingest: %v", ingestErr) + } + var sequences []int + for ack := range results { + if ack.Duplicate || ack.Digest == "" || ack.StreamID != "otlp-logs" { + t.Errorf("ack=%+v", ack) + } + sequences = append(sequences, int(ack.Sequence)) + } + sort.Ints(sequences) + if len(sequences) != requests { + t.Fatalf("sequences=%v", sequences) + } + for index, sequence := range sequences { + if sequence != index+1 { + t.Fatalf("sequences=%v", sequences) + } + } +} + +func testStore(t *testing.T) *Store { + t.Helper() + root := filepath.Join(t.TempDir(), "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + store, err := Open(root) + if err != nil { + t.Fatal(err) + } + return store +} + +func projectAll(t testing.TB, store *Store) { + t.Helper() + for { + report, err := store.ProjectPending(context.Background()) + if err != nil { + t.Fatal(err) + } + if report.ProjectedSegments == 0 { + return + } + } +} + +func TestStorageRejectsSymlinkedSQLiteFilesAndProjectionDirectories(t *testing.T) { + base := t.TempDir() + root := filepath.Join(base, "data") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(base, "target.sqlite") + if err := os.WriteFile(target, []byte("target"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(root, "control.sqlite")); err != nil { + t.Fatal(err) + } + if _, err := Open(root); err == nil { + t.Fatal("symlink control database was accepted") + } + if err := os.Remove(filepath.Join(root, "control.sqlite")); err != nil { + t.Fatal(err) + } + if err := os.Link(target, filepath.Join(root, "control.sqlite")); err != nil { + t.Fatal(err) + } + if _, err := Open(root); err == nil { + t.Fatal("hard-linked control database was accepted") + } + if err := os.Remove(filepath.Join(root, "control.sqlite")); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(root, "organizations"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(base, filepath.Join(root, "organizations", "organization-a")); err != nil { + t.Fatal(err) + } + if _, err := openProjection(context.Background(), filepath.Join(root, "organizations", "organization-a", "projection.sqlite")); err == nil { + t.Fatal("symlink organization directory was accepted") + } + if err := os.Remove(filepath.Join(root, "organizations", "organization-a")); err != nil { + t.Fatal(err) + } + organization := filepath.Join(root, "organizations", "organization-a") + if err := os.Mkdir(organization, 0o700); err != nil { + t.Fatal(err) + } + projection := filepath.Join(organization, "projection.sqlite") + if err := os.Symlink(target, projection+"-wal"); err != nil { + t.Fatal(err) + } + if _, err := openProjection(context.Background(), projection); err == nil { + t.Fatal("symlink projection sidecar was accepted") + } +} + +func TestScopedIngestionDeduplicationAndReplay(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + scope := model.Scope{OrganizationID: "org-a", ProjectID: "site", EnvironmentID: "prod", ServiceID: "web"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source-a", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request", Attributes: map[string]string{"route": "/"}}}} + ack, err := store.Ingest(ctx, token, batch, now) + if err != nil { + t.Fatal(err) + } + if ack.Duplicate || ack.Digest == "" { + t.Fatalf("unexpected acknowledgement: %#v", ack) + } + duplicate, err := store.Ingest(ctx, token, batch, now) + if err != nil { + t.Fatal(err) + } + if !duplicate.Duplicate || duplicate.Digest != ack.Digest { + t.Fatalf("unexpected duplicate acknowledgement: %#v", duplicate) + } + batch.Records[0].Name = "changed" + if _, err := store.Ingest(ctx, token, batch, now); err == nil { + t.Fatal("expected conflicting duplicate rejection") + } + entries, err := store.segments.List() + if err != nil || len(entries) != 1 || entries[0].Committed.Digest != ack.Digest { + t.Fatalf("conflicting replay left raw evidence behind: entries=%+v err=%v", entries, err) + } + batch.Sequence = 3 + if _, err := store.Ingest(ctx, token, batch, now); err == nil { + t.Fatal("expected sequence gap rejection") + } +} + +func TestProjectorReusesProjectionHandleAndRejectsPathReplacement(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + ingest := func(sequence uint64) error { + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: sequence, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}} + _, ingestErr := store.Ingest(ctx, token, batch, now) + return ingestErr + } + if err = ingest(1); err != nil { + t.Fatal(err) + } + projectAll(t, store) + store.projectionMu.Lock() + first := store.projections[scope.OrganizationID] + store.projectionMu.Unlock() + if first.db == nil { + t.Fatal("projection handle was not retained") + } + if err = ingest(2); err != nil { + t.Fatal(err) + } + projectAll(t, store) + store.projectionMu.Lock() + second := store.projections[scope.OrganizationID] + store.projectionMu.Unlock() + if second.db != first.db || second.device != first.device || second.inode != first.inode { + t.Fatal("projection handle was reopened for an unchanged organization") + } + + projection := filepath.Join(store.root, "organizations", scope.OrganizationID, "projection.sqlite") + replaced := projection + ".replaced" + if err = os.Rename(projection, replaced); err != nil { + t.Fatal(err) + } + if err = ingest(3); err != nil { + t.Fatalf("durable ingestion depended on projection path: %v", err) + } + if _, err = store.ProjectPending(ctx); err == nil { + t.Fatal("replaced projection path was accepted by projector") + } + if err = os.Rename(replaced, projection); err != nil { + t.Fatal(err) + } + if err = store.Recover(ctx); err != nil { + t.Fatal(err) + } + var projected int + if err = second.db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&projected); err != nil { + t.Fatal(err) + } + if projected != 3 { + t.Fatalf("projected=%d", projected) + } +} + +func TestCredentialScopeCannotBeOverridden(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "org-a", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"}) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source-b", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}} + if _, err := store.Ingest(ctx, token, batch, now); err == nil { + t.Fatal("expected source mismatch rejection") + } +} + +func TestRecoveryIndexesRawSegmentMissingFromControlDatabase(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "org-a", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"}) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source-a", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}} + if _, err := store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + if _, err := store.control.Exec(`DELETE FROM streams; DELETE FROM segments`); err != nil { + t.Fatal(err) + } + if err := store.Recover(ctx); err != nil { + t.Fatal(err) + } + var segments, projected int + if err := store.control.QueryRow(`SELECT COUNT(*), COUNT(projected_at) FROM segments`).Scan(&segments, &projected); err != nil { + t.Fatal(err) + } + if segments != 1 || projected != 1 { + t.Fatalf("segments=%d projected=%d", segments, projected) + } +} + +func TestRecoveryRejectsCataloguedRawMetadataMismatchWithoutDecoding(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + scope := model.Scope{OrganizationID: "org-a", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source-a", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}} + if _, err = store.Ingest(ctx, token, batch, now); err != nil { + t.Fatal(err) + } + var original string + if err = store.control.QueryRow(`SELECT path FROM segments`).Scan(&original); err != nil { + t.Fatal(err) + } + destination := filepath.Join(store.root, "raw", "org-a", "source-b", "access", filepath.Base(original)) + if err = os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + t.Fatal(err) + } + if err = os.Rename(original, destination); err != nil { + t.Fatal(err) + } + if err = store.Recover(ctx); err == nil { + t.Fatal("catalogued segment identity mismatch was accepted") + } +} + +func TestRecoveryDoesNotDecodeAlreadyCataloguedRawSegments(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + scope := model.Scope{OrganizationID: "org-a", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"} + token, err := store.CreateSource(ctx, "source-a", scope) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{Version: 1, SourceID: "source-a", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}} + ack, err := store.Ingest(ctx, token, batch, now) + if err != nil { + t.Fatal(err) + } + projectAll(t, store) + var path string + if err = store.control.QueryRow(`SELECT path FROM segments WHERE digest=?`, ack.Digest).Scan(&path); err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + body[0] ^= 0xff + if err = os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + if err = store.Recover(ctx); err != nil { + t.Fatalf("startup decoded catalogued evidence: %v", err) + } + if _, err = store.segments.Read(path, ack.Digest); err == nil { + t.Fatal("explicit forensic read accepted corrupt evidence") + } +} + +func TestRecoveryProcessesUnprojectedSegmentsInBoundedPages(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + if _, err := store.CreateSource(ctx, "source-a", scope); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + const count = recoveryPageSize + 3 + for sequence := uint64(1); sequence <= count; sequence++ { + batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: sequence, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}} + committed, err := store.segments.Commit(scope, batch) + if err != nil { + t.Fatal(err) + } + if err = store.recordCommitted(ctx, scope, batch, committed); err != nil { + t.Fatal(err) + } + } + if err := store.Recover(ctx); err != nil { + t.Fatal(err) + } + var segments, projected int + if err := store.control.QueryRow(`SELECT COUNT(*), COUNT(projected_at) FROM segments`).Scan(&segments, &projected); err != nil { + t.Fatal(err) + } + if segments != count || projected != count { + t.Fatalf("segments=%d projected=%d", segments, projected) + } +} + +func TestCommittedAdmissionRejectsInvalidTimeWithoutControlState(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + if _, err := store.CreateSource(ctx, "source-a", scope); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + batch := model.Batch{ + Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, + ObservedAt: now, Signal: model.SignalLogs, + Records: []model.Observation{{Timestamp: now, Name: "application.request", Attributes: map[string]string{"workshop.unknown": "value"}}}, + } + committed, err := store.segments.Commit(scope, batch) + if err != nil { + t.Fatal(err) + } + if err = store.admitCommitted(ctx, scope, batch, committed, time.Time{}); err == nil { + t.Fatal("invalid committed time was accepted") + } + var segments, streams, proposals int + if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&segments); err != nil { + t.Fatal(err) + } + if err = store.control.QueryRow(`SELECT COUNT(*) FROM streams`).Scan(&streams); err != nil { + t.Fatal(err) + } + if err = store.control.QueryRow(`SELECT COUNT(*) FROM descriptor_proposals`).Scan(&proposals); err != nil { + t.Fatal(err) + } + if segments != 0 || streams != 0 || proposals != 0 { + t.Fatalf("partial durable control state: segments=%d streams=%d proposals=%d", segments, streams, proposals) + } + if err = store.Recover(ctx); err != nil { + t.Fatal(err) + } + var projected int + if err = store.control.QueryRow(`SELECT COUNT(*), COUNT(projected_at) FROM segments`).Scan(&segments, &projected); err != nil { + t.Fatal(err) + } + if err = store.control.QueryRow(`SELECT COUNT(*) FROM streams`).Scan(&streams); err != nil { + t.Fatal(err) + } + if err = store.control.QueryRow(`SELECT COUNT(*) FROM descriptor_proposals`).Scan(&proposals); err != nil { + t.Fatal(err) + } + if segments != 1 || projected != 1 || streams != 1 || proposals != 1 { + t.Fatalf("recovery did not complete durable projection state: segments=%d projected=%d streams=%d proposals=%d", segments, projected, streams, proposals) + } +} + +func TestEnrollmentIsScopedExpiringAndSingleUse(t *testing.T) { + ctx := context.Background() + store := testStore(t) + defer store.Close() + now := time.Date(2026, 8, 17, 2, 0, 0, 0, time.UTC) + scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"} + token, enrollment, err := store.CreateEnrollment(ctx, "source-a", scope, "operator-a", 15*time.Minute, now) + if err != nil || token == "" || enrollment.Scope != scope { + t.Fatalf("token_present=%t enrollment=%+v err=%v", token != "", enrollment, err) + } + got, credential, err := store.RedeemEnrollment(ctx, token, now.Add(time.Minute)) + if err != nil || got.SourceID != "source-a" || credential == "" { + t.Fatalf("enrollment=%+v credential_present=%t err=%v", got, credential != "", err) + } + source, err := store.Authenticate(ctx, credential) + if err != nil || source.Scope != scope { + t.Fatalf("source=%+v err=%v", source, err) + } + if _, _, err = store.RedeemEnrollment(ctx, token, now.Add(2*time.Minute)); err == nil { + t.Fatal("single-use enrollment redeemed twice") + } + expired, _, err := store.CreateEnrollment(ctx, "source-b", scope, "operator-a", 5*time.Minute, now) + if err != nil { + t.Fatal(err) + } + if _, _, err = store.RedeemEnrollment(ctx, expired, now.Add(5*time.Minute)); err == nil { + t.Fatal("expired enrollment accepted") + } +} diff --git a/internal/tailer/tailer.go b/internal/tailer/tailer.go new file mode 100644 index 0000000..1ac6b83 --- /dev/null +++ b/internal/tailer/tailer.go @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package tailer + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "time" + + "gamertan.com/observatory/internal/agentstate" + "gamertan.com/observatory/internal/collector" + "gamertan.com/observatory/internal/config" + "gamertan.com/observatory/internal/model" +) + +const maxDirectoryEntries = 4096 +const maxReadBytesPerCycle = 4 << 20 + +type Result struct { + Signal model.Signal + Observations []model.Observation + Cursor agentstate.Cursor +} + +type fileIdentity struct{ device, inode uint64 } + +func Read(source config.AgentSource, cursor agentstate.Cursor, maximum int, now time.Time) (Result, error) { + if maximum < 1 || maximum > model.MaxRecords { + return Result{}, errors.New("tail batch limit is invalid") + } + configured, configuredIdentity, err := openRegular(source.Path) + if errors.Is(err, os.ErrNotExist) { + return Result{Cursor: cursor}, nil + } + if err != nil { + return Result{}, err + } + defer configured.Close() + + active, activeIdentity, rotated := configured, configuredIdentity, false + if cursor.Device != 0 && (cursor.Device != configuredIdentity.device || cursor.Inode != configuredIdentity.inode) { + previous, previousIdentity, findErr := findIdentity(filepath.Dir(source.Path), fileIdentity{cursor.Device, cursor.Inode}) + if findErr != nil { + return Result{}, findErr + } + if previous != nil { + defer previous.Close() + active, activeIdentity, rotated = previous, previousIdentity, true + } else { + cursor.Discontinuities++ + cursor.Device, cursor.Inode, cursor.Offset, cursor.DiscardingLine = configuredIdentity.device, configuredIdentity.inode, 0, false + } + } + if cursor.Device == 0 { + cursor.Device, cursor.Inode = activeIdentity.device, activeIdentity.inode + } + + result, atEOF, partial, err := readOpen(active, source, cursor, maximum, now) + if err != nil { + return Result{}, err + } + if !rotated || len(result.Observations) == maximum || !atEOF { + return result, nil + } + if result.Cursor.DiscardingLine { + result.Cursor.DroppedRecords++ + result.Cursor.DiscardingLine = false + } else if partial > 0 { + result.Cursor.Offset += int64(partial) + result.Cursor.DroppedRecords++ + } + result.Cursor.Device, result.Cursor.Inode, result.Cursor.Offset, result.Cursor.DiscardingLine = configuredIdentity.device, configuredIdentity.inode, 0, false + remaining := maximum - len(result.Observations) + next, _, _, err := readOpen(configured, source, result.Cursor, remaining, now) + if err != nil { + return Result{}, err + } + if result.Signal == "" { + result.Signal = next.Signal + } + result.Observations = append(result.Observations, next.Observations...) + result.Cursor = next.Cursor + return result, nil +} + +func readOpen(file *os.File, source config.AgentSource, cursor agentstate.Cursor, maximum int, now time.Time) (Result, bool, int, error) { + info, err := file.Stat() + if err != nil { + return Result{}, false, 0, err + } + identity, err := identity(info) + if err != nil { + return Result{}, false, 0, err + } + if cursor.Device != identity.device || cursor.Inode != identity.inode { + cursor.Device, cursor.Inode, cursor.Offset, cursor.DiscardingLine = identity.device, identity.inode, 0, false + } + if info.Size() < cursor.Offset { + cursor.Offset = 0 + cursor.DiscardingLine = false + cursor.Discontinuities++ + } + if _, err = file.Seek(cursor.Offset, io.SeekStart); err != nil { + return Result{}, false, 0, err + } + reader := bufio.NewReaderSize(file, 64<<10) + result := Result{Cursor: cursor} + discarding := cursor.DiscardingLine + partial := 0 + readBytes := 0 + pendingBytes := 0 + lineBuffer := make([]byte, 0, 64<<10) + for len(result.Observations) < maximum { + fragment, readErr := reader.ReadSlice('\n') + readBytes += len(fragment) + switch { + case readErr == nil: + if discarding { + result.Cursor.Offset += int64(len(fragment)) + result.Cursor.DroppedRecords++ + discarding = false + result.Cursor.DiscardingLine = false + } else { + pendingBytes += len(fragment) + if pendingBytes > collector.MaxLineBytes+1 { + result.Cursor.Offset += int64(pendingBytes) + result.Cursor.DroppedRecords++ + } else { + lineBuffer = append(lineBuffer, fragment...) + result.Cursor.Offset += int64(pendingBytes) + line := lineBuffer[:len(lineBuffer)-1] + if len(line) > 0 && line[len(line)-1] == '\r' { + line = line[:len(line)-1] + } + signal, observation, parseErr := collector.Parse(source.Kind, line, now, source.SensitiveFields...) + if parseErr != nil { + result.Cursor.DroppedRecords++ + } else { + if result.Signal != "" && signal != result.Signal { + return Result{}, false, 0, errors.New("collector changed signal inside one stream") + } + result.Signal = signal + result.Observations = append(result.Observations, observation) + } + } + pendingBytes = 0 + lineBuffer = lineBuffer[:0] + } + case errors.Is(readErr, bufio.ErrBufferFull): + if discarding { + result.Cursor.Offset += int64(len(fragment)) + } else { + pendingBytes += len(fragment) + if pendingBytes > collector.MaxLineBytes { + result.Cursor.Offset += int64(pendingBytes) + pendingBytes = 0 + lineBuffer = lineBuffer[:0] + discarding = true + result.Cursor.DiscardingLine = true + } else { + lineBuffer = append(lineBuffer, fragment...) + } + } + case errors.Is(readErr, io.EOF): + if discarding { + result.Cursor.Offset += int64(len(fragment)) + result.Cursor.DiscardingLine = true + partial = 0 + } else if pendingBytes+len(fragment) > collector.MaxLineBytes { + result.Cursor.Offset += int64(pendingBytes + len(fragment)) + result.Cursor.DiscardingLine = true + partial = 0 + } else { + partial = pendingBytes + len(fragment) + } + return result, true, partial, nil + default: + return Result{}, false, 0, readErr + } + if readBytes >= maxReadBytesPerCycle { + return result, false, 0, nil + } + } + return result, false, 0, nil +} + +func openRegular(path string) (*os.File, fileIdentity, error) { + file, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) + if err != nil { + return nil, fileIdentity{}, err + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, fileIdentity{}, err + } + if !info.Mode().IsRegular() { + file.Close() + return nil, fileIdentity{}, errors.New("collector source must be a regular file") + } + value, err := identity(info) + if err != nil { + file.Close() + return nil, fileIdentity{}, err + } + return file, value, nil +} + +func findIdentity(directory string, wanted fileIdentity) (*os.File, fileIdentity, error) { + entries, err := os.ReadDir(directory) + if errors.Is(err, os.ErrNotExist) { + return nil, fileIdentity{}, nil + } + if err != nil { + return nil, fileIdentity{}, err + } + if len(entries) > maxDirectoryEntries { + return nil, fileIdentity{}, errors.New("collector directory exceeds safe entry limit") + } + for _, entry := range entries { + if entry.Type()&os.ModeSymlink != 0 || entry.IsDir() { + continue + } + file, found, openErr := openRegular(filepath.Join(directory, entry.Name())) + if openErr != nil { + continue + } + if found == wanted { + return file, found, nil + } + file.Close() + } + return nil, fileIdentity{}, nil +} + +func identity(info os.FileInfo) (fileIdentity, error) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Dev == 0 || stat.Ino == 0 { + return fileIdentity{}, fmt.Errorf("collector file identity is unavailable") + } + return fileIdentity{device: uint64(stat.Dev), inode: uint64(stat.Ino)}, nil +} diff --git a/internal/tailer/tailer_test.go b/internal/tailer/tailer_test.go new file mode 100644 index 0000000..1be13cf --- /dev/null +++ b/internal/tailer/tailer_test.go @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package tailer + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/observatory/internal/agentstate" + "gamertan.com/observatory/internal/collector" + "gamertan.com/observatory/internal/config" + "gamertan.com/observatory/internal/model" +) + +func TestTailerPreservesPartialLineAndRecoversRotation(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "request.jsonl") + first := requestLine("one") + "\n" + requestLine("two") + "\n" + partial := `{"timestamp":"2026-08-17T01:02:03Z","method":"GET"` + if err := os.WriteFile(path, []byte(first+partial), 0o640); err != nil { + t.Fatal(err) + } + source := config.AgentSource{Kind: "requestlog_jsonl", Path: path, StreamID: "request"} + now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC) + result, err := Read(source, agentstate.Cursor{}, 10, now) + if err != nil || len(result.Observations) != 2 || result.Cursor.Offset != int64(len(first)) { + t.Fatalf("result=%+v err=%v", result, err) + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err = file.WriteString(`,"route":"/three","status":200}` + "\n"); err != nil { + t.Fatal(err) + } + file.Close() + result, err = Read(source, result.Cursor, 10, now) + if err != nil || len(result.Observations) != 1 || result.Observations[0].Attributes["http.route"] != "/three" { + t.Fatalf("partial result=%+v err=%v", result, err) + } + cursor := result.Cursor + file, err = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err = file.WriteString(requestLine("before-rotate") + "\n"); err != nil { + t.Fatal(err) + } + file.Close() + if err = os.Rename(path, path+".1"); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, []byte(requestLine("new-file")+"\n"), 0o640); err != nil { + t.Fatal(err) + } + result, err = Read(source, cursor, 10, now) + if err != nil || len(result.Observations) != 2 || result.Observations[0].Attributes["http.route"] != "/before-rotate" || result.Observations[1].Attributes["http.route"] != "/new-file" { + t.Fatalf("rotation result=%+v err=%v", result, err) + } +} + +func TestTailerBoundsOversizedLinesAndDetectsTruncation(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "request.jsonl") + if err := os.WriteFile(path, []byte(strings.Repeat("x", collector.MaxLineBytes+100)), 0o640); err != nil { + t.Fatal(err) + } + source := config.AgentSource{Kind: "requestlog_jsonl", Path: path, StreamID: "request"} + now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC) + result, err := Read(source, agentstate.Cursor{}, 10, now) + if err != nil || !result.Cursor.DiscardingLine || result.Cursor.Offset != collector.MaxLineBytes+100 { + t.Fatalf("oversized result=%+v err=%v", result, err) + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err = file.WriteString("\n" + requestLine("after-large") + "\n"); err != nil { + t.Fatal(err) + } + file.Close() + result, err = Read(source, result.Cursor, 10, now) + if err != nil || result.Cursor.DroppedRecords != 1 || len(result.Observations) != 1 { + t.Fatalf("discard result=%+v err=%v", result, err) + } + if err = os.Truncate(path, 0); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, []byte(requestLine("after-truncate")+"\n"), 0o640); err != nil { + t.Fatal(err) + } + result, err = Read(source, result.Cursor, 10, now) + if err != nil || result.Cursor.Discontinuities != 1 || len(result.Observations) != 1 { + t.Fatalf("truncate result=%+v err=%v", result, err) + } +} + +func TestTailerAcceptsBoundedLineLargerThanReaderBuffer(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "request.jsonl") + line := `{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/large","status":200,"padding":"` + strings.Repeat("x", 128<<10) + `"}` + if err := os.WriteFile(path, []byte(line+"\n"), 0o640); err != nil { + t.Fatal(err) + } + result, err := Read(config.AgentSource{Kind: "requestlog_jsonl", Path: path, StreamID: "request"}, agentstate.Cursor{}, 10, time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)) + if err != nil || len(result.Observations) != 1 || result.Cursor.DroppedRecords != 0 { + t.Fatalf("result=%+v err=%v", result, err) + } +} + +func TestTailerBoundsSourceBytesPerCycle(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "request.jsonl") + line := `{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/bounded","status":200,"padding":"` + strings.Repeat("x", 256<<10) + `"}` + "\n" + const lines = 20 + if err := os.WriteFile(path, []byte(strings.Repeat(line, lines)), 0o640); err != nil { + t.Fatal(err) + } + result, err := Read(config.AgentSource{Kind: "requestlog_jsonl", Path: path, StreamID: "request"}, agentstate.Cursor{}, model.MaxRecords, time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)) + if err != nil || len(result.Observations) == 0 || len(result.Observations) >= lines || result.Cursor.Offset < maxReadBytesPerCycle-int64(len(line)) || result.Cursor.Offset > maxReadBytesPerCycle+int64(len(line)) { + t.Fatalf("observations=%d offset=%d err=%v", len(result.Observations), result.Cursor.Offset, err) + } +} + +func TestTailerAppliesExplicitSensitiveFieldSelection(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "request.jsonl") + line := `{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/items","status":200,"client_ip":"192.0.2.25","query":"view=full"}` + if err := os.WriteFile(path, []byte(line+"\n"), 0o640); err != nil { + t.Fatal(err) + } + source := config.AgentSource{Kind: "requestlog_jsonl", Path: path, StreamID: "request", SensitiveFields: []string{"client_ip", "query"}} + result, err := Read(source, agentstate.Cursor{}, 10, time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)) + if err != nil || len(result.Observations) != 1 || result.Observations[0].Attributes["client.address"] != "192.0.2.25" || result.Observations[0].Attributes["url.query"] != "view=full" { + t.Fatalf("result=%+v err=%v", result, err) + } +} + +func TestTailerRejectsSymlinkSource(t *testing.T) { + dir := t.TempDir() + real := filepath.Join(dir, "real") + link := filepath.Join(dir, "link") + if err := os.WriteFile(real, []byte(requestLine("one")+"\n"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.Symlink(real, link); err != nil { + t.Skip(err) + } + _, err := Read(config.AgentSource{Kind: "requestlog_jsonl", Path: link, StreamID: "request"}, agentstate.Cursor{}, 10, time.Now().UTC()) + if err == nil { + t.Fatal("symlink source accepted") + } +} + +func requestLine(route string) string { + return `{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/` + route + `","status":200,"bytes":12,"duration_ns":1000,"request_id":"request-1"}` +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..9521b4b --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package version + +import "runtime" + +var ( + Version = "0.1.0-dev" + Commit = "unknown" + Date = "unknown" +) + +type Info struct { + Version string `json:"version"` + Commit string `json:"commit"` + Date string `json:"date"` + Go string `json:"go"` + Features []string `json:"features"` +} + +func Current() Info { + return Info{Version: Version, Commit: Commit, Date: Date, Go: runtime.Version(), Features: []string{"native-ingest-v1", "native-ingest-v2", "raw-zstd-segments", "sqlite-projections", "typed-query-v1", "organization-access-v1", "agent-tail-v1", "agent-enrollment-v1", "retention-v1", "cold-archive-v1", "metric-rollup-5m-v1", "tend-candidate-v1"}} +} diff --git a/internal/webpush/notifier.go b/internal/webpush/notifier.go new file mode 100644 index 0000000..a262b4a --- /dev/null +++ b/internal/webpush/notifier.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package webpush + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "gamertan.com/observatory/internal/identity" + "gamertan.com/observatory/internal/storage" + "gamertan.com/web/access" +) + +type subscriptionStore interface { + PushSubscriptions(context.Context, string) ([]storage.PushSubscription, error) + RecordPushResult(context.Context, string, string, string, time.Time) error + DeletePushSubscription(context.Context, string, string, string) (bool, error) +} + +type authorizer interface { + Authorize(context.Context, string, access.Scope, string) (access.Decision, error) +} + +type notificationSender interface { + Send(context.Context, Subscription) error +} + +type Notifier struct { + store subscriptionStore + authorizer authorizer + sender notificationSender + queue chan string + now func() time.Time + pendingMu sync.Mutex + pending map[string]struct{} + enqueued atomic.Uint64 + delivered atomic.Uint64 + failed atomic.Uint64 + dropped atomic.Uint64 +} + +type NotifierStats struct { + Enqueued uint64 + Delivered uint64 + Failed uint64 + Dropped uint64 +} + +func NewNotifier(store subscriptionStore, authorizer authorizer, sender notificationSender, capacity int) (*Notifier, error) { + if store == nil || authorizer == nil || sender == nil || capacity < 1 || capacity > 1024 { + return nil, errors.New("web push notifier options are invalid") + } + return &Notifier{store: store, authorizer: authorizer, sender: sender, queue: make(chan string, capacity), now: func() time.Time { return time.Now().UTC() }, pending: map[string]struct{}{}}, nil +} + +// Enqueue records only an opaque organization identity in a bounded in-memory +// queue. A full queue or duplicate pending organization is deliberately +// nonblocking: alert evaluation and incident persistence remain authoritative. +func (n *Notifier) Enqueue(organizationID string) bool { + n.pendingMu.Lock() + if _, exists := n.pending[organizationID]; exists { + n.pendingMu.Unlock() + return true + } + n.pending[organizationID] = struct{}{} + n.pendingMu.Unlock() + select { + case n.queue <- organizationID: + n.enqueued.Add(1) + return true + default: + n.pendingMu.Lock() + delete(n.pending, organizationID) + n.pendingMu.Unlock() + n.dropped.Add(1) + return false + } +} + +func (n *Notifier) Run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case organizationID := <-n.queue: + n.deliver(ctx, organizationID) + n.pendingMu.Lock() + delete(n.pending, organizationID) + n.pendingMu.Unlock() + } + } +} + +func (n *Notifier) Stats() NotifierStats { + return NotifierStats{Enqueued: n.enqueued.Load(), Delivered: n.delivered.Load(), Failed: n.failed.Load(), Dropped: n.dropped.Load()} +} + +func (n *Notifier) deliver(ctx context.Context, organizationID string) { + subscriptions, err := n.store.PushSubscriptions(ctx, organizationID) + if err != nil { + n.failed.Add(1) + return + } + for _, subscription := range subscriptions { + decision, authErr := n.authorizer.Authorize(ctx, subscription.UserID, access.Scope{OrganizationID: organizationID}, identity.PermissionIncidentsRead) + if authErr != nil { + n.failed.Add(1) + continue + } + if !decision.Allowed { + _, _ = n.store.DeletePushSubscription(ctx, organizationID, subscription.UserID, subscription.Endpoint) + continue + } + deliveryErr := n.sender.Send(ctx, Subscription{Endpoint: subscription.Endpoint, P256DH: subscription.P256DH, Auth: subscription.Auth}) + outcome := "sent" + if errors.Is(deliveryErr, ErrSubscriptionGone) { + outcome = "gone" + } else if deliveryErr != nil { + outcome = "failed" + } + if recordErr := n.store.RecordPushResult(ctx, organizationID, subscription.ID, outcome, n.now()); recordErr != nil || deliveryErr != nil { + n.failed.Add(1) + continue + } + n.delivered.Add(1) + } +} diff --git a/internal/webpush/notifier_test.go b/internal/webpush/notifier_test.go new file mode 100644 index 0000000..4d97b0a --- /dev/null +++ b/internal/webpush/notifier_test.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package webpush + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + "gamertan.com/observatory/internal/storage" + "gamertan.com/web/access" +) + +type fakePushStore struct { + subscriptions []storage.PushSubscription + mu sync.Mutex + results []string + deleted []string +} + +func (store *fakePushStore) DeletePushSubscription(_ context.Context, organizationID, userID, endpoint string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + store.deleted = append(store.deleted, organizationID+"/"+userID+"/"+endpoint) + return false, nil +} + +func (store *fakePushStore) PushSubscriptions(context.Context, string) ([]storage.PushSubscription, error) { + return append([]storage.PushSubscription(nil), store.subscriptions...), nil +} + +func (store *fakePushStore) RecordPushResult(_ context.Context, _, _ string, result string, _ time.Time) error { + store.mu.Lock() + defer store.mu.Unlock() + store.results = append(store.results, result) + return nil +} + +type fakeAuthorizer struct{ allowed map[string]bool } + +func (authorizer fakeAuthorizer) Authorize(_ context.Context, userID string, _ access.Scope, permission string) (access.Decision, error) { + if permission != "incidents.read" { + return access.Decision{}, errors.New("unexpected permission") + } + return access.Decision{Allowed: authorizer.allowed[userID]}, nil +} + +type fakeNotificationSender struct { + mu sync.Mutex + requests []Subscription + err error +} + +func (sender *fakeNotificationSender) Send(_ context.Context, subscription Subscription) error { + sender.mu.Lock() + defer sender.mu.Unlock() + sender.requests = append(sender.requests, subscription) + return sender.err +} + +func TestNotifierIsBoundedDeduplicatedAndAuthorizationAware(t *testing.T) { + store := &fakePushStore{subscriptions: []storage.PushSubscription{ + {OrganizationID: "organization-a", ID: "push-a", UserID: "user-a", Endpoint: "https://push.example.test/a", P256DH: make([]byte, 65), Auth: make([]byte, 16)}, + {OrganizationID: "organization-a", ID: "push-b", UserID: "user-b", Endpoint: "https://push.example.test/b", P256DH: make([]byte, 65), Auth: make([]byte, 16)}, + }} + sender := &fakeNotificationSender{} + notifier, err := NewNotifier(store, fakeAuthorizer{allowed: map[string]bool{"user-a": true}}, sender, 1) + if err != nil { + t.Fatal(err) + } + if !notifier.Enqueue("organization-a") || !notifier.Enqueue("organization-a") { + t.Fatal("duplicate pending organization was not accepted as already queued") + } + if notifier.Enqueue("organization-b") { + t.Fatal("full queue accepted a second organization") + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { notifier.Run(ctx); close(done) }() + deadline := time.Now().Add(time.Second) + for notifier.Stats().Delivered != 1 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + cancel() + <-done + if len(sender.requests) != 1 || sender.requests[0].Endpoint != "https://push.example.test/a" { + t.Fatalf("requests=%+v", sender.requests) + } + store.mu.Lock() + defer store.mu.Unlock() + if len(store.results) != 1 || store.results[0] != "sent" || len(store.deleted) != 1 || !strings.Contains(store.deleted[0], "/user-b/") { + t.Fatalf("results=%v deleted=%v", store.results, store.deleted) + } + stats := notifier.Stats() + if stats.Enqueued != 1 || stats.Delivered != 1 || stats.Dropped != 1 || stats.Failed != 0 { + t.Fatalf("stats=%+v", stats) + } +} + +func TestNotifierDeliveryFailureNeverBlocksEnqueue(t *testing.T) { + store := &fakePushStore{subscriptions: []storage.PushSubscription{{OrganizationID: "organization-a", ID: "push-a", UserID: "user-a", Endpoint: "https://push.example.test/a", P256DH: make([]byte, 65), Auth: make([]byte, 16)}}} + notifier, err := NewNotifier(store, fakeAuthorizer{allowed: map[string]bool{"user-a": true}}, &fakeNotificationSender{err: errors.New("offline")}, 1) + if err != nil { + t.Fatal(err) + } + if !notifier.Enqueue("organization-a") { + t.Fatal("enqueue failed") + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { notifier.Run(ctx); close(done) }() + deadline := time.Now().Add(time.Second) + for notifier.Stats().Failed == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + cancel() + <-done + if notifier.Stats().Failed != 1 || len(store.results) != 1 || store.results[0] != "failed" { + t.Fatalf("stats=%+v results=%v", notifier.Stats(), store.results) + } +} diff --git a/internal/webpush/sender.go b/internal/webpush/sender.go new file mode 100644 index 0000000..bd71a2e --- /dev/null +++ b/internal/webpush/sender.go @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package webpush implements the deliberately narrow Web Push boundary used +// by Observatory. It accepts only validated HTTPS push-service endpoints and +// encrypts one fixed, generic notification payload. +package webpush + +import ( + "bytes" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/ecdh" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net" + "net/http" + "net/netip" + "net/url" + "strings" + "time" +) + +const ( + GenericMessage = "Gamertan Observatory needs your attention." + maxEndpointBytes = 2048 + maxResponseBodyBytes = 4096 +) + +var ErrSubscriptionGone = errors.New("web push subscription is gone") + +type Subscription struct { + Endpoint string + P256DH []byte + Auth []byte +} + +type Options struct { + PrivateKey []byte + Subject string + Timeout time.Duration + Client *http.Client + Now func() time.Time + Random io.Reader +} + +type Sender struct { + private *ecdh.PrivateKey + subject string + timeout time.Duration + client *http.Client + now func() time.Time + random io.Reader +} + +func New(options Options) (*Sender, error) { + private, err := ecdh.P256().NewPrivateKey(options.PrivateKey) + if err != nil { + return nil, errors.New("web push private key is invalid") + } + if err = validateSubject(options.Subject); err != nil { + return nil, err + } + if options.Timeout < time.Second || options.Timeout > 30*time.Second { + return nil, errors.New("web push timeout must be between 1s and 30s") + } + if options.Now == nil { + options.Now = func() time.Time { return time.Now().UTC() } + } + if options.Random == nil { + options.Random = rand.Reader + } + if options.Client == nil { + options.Client = safeClient(options.Timeout) + } + return &Sender{private: private, subject: options.Subject, timeout: options.Timeout, client: options.Client, now: options.Now, random: options.Random}, nil +} + +func (s *Sender) PublicKey() string { + return base64.RawURLEncoding.EncodeToString(s.private.PublicKey().Bytes()) +} + +func (s *Sender) Send(ctx context.Context, subscription Subscription) error { + endpoint, err := ValidateEndpoint(subscription.Endpoint) + if err != nil { + return err + } + body, _, err := encrypt([]byte(GenericMessage), subscription, s.random) + if err != nil { + return err + } + jwt, err := s.vapid(endpoint, s.now(), s.random) + if err != nil { + return err + } + requestContext, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + request, err := http.NewRequestWithContext(requestContext, http.MethodPost, endpoint.String(), bytes.NewReader(body)) + if err != nil { + return errors.New("create web push request") + } + request.Header.Set("Authorization", "vapid t="+jwt+", k="+s.PublicKey()) + request.Header.Set("Content-Encoding", "aes128gcm") + request.Header.Set("Content-Type", "application/octet-stream") + request.Header.Set("TTL", "300") + request.Header.Set("Urgency", "high") + response, err := s.client.Do(request) + if err != nil { + return errors.New("deliver web push notification") + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, maxResponseBodyBytes)) + if response.StatusCode == http.StatusGone || response.StatusCode == http.StatusNotFound { + return ErrSubscriptionGone + } + if response.StatusCode < 200 || response.StatusCode > 299 { + return fmt.Errorf("web push service returned status %d", response.StatusCode) + } + return nil +} + +func ValidateEndpoint(raw string) (*url.URL, error) { + if len(raw) < len("https://a.b/x") || len(raw) > maxEndpointBytes || !strings.HasPrefix(raw, "https://") { + return nil, errors.New("web push endpoint must be a bounded HTTPS URL") + } + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" { + return nil, errors.New("web push endpoint is invalid") + } + if parsed.Port() != "" && parsed.Port() != "443" { + return nil, errors.New("web push endpoint port is not permitted") + } + host := parsed.Hostname() + if net.ParseIP(host) != nil || !validDNSName(host) { + return nil, errors.New("web push endpoint host is invalid") + } + if parsed.Path == "" || parsed.Path[0] != '/' { + return nil, errors.New("web push endpoint path is invalid") + } + return parsed, nil +} + +func validateSubject(subject string) error { + if len(subject) < 8 || len(subject) > 512 || strings.ContainsAny(subject, "\r\n\t ") { + return errors.New("web push subject is invalid") + } + parsed, err := url.Parse(subject) + if err != nil || parsed.Fragment != "" || parsed.RawQuery != "" { + return errors.New("web push subject is invalid") + } + if parsed.Scheme == "mailto" && parsed.Opaque != "" && strings.Contains(parsed.Opaque, "@") { + return nil + } + if parsed.Scheme == "https" && parsed.Host != "" && parsed.User == nil { + return nil + } + return errors.New("web push subject must be a mailto address or HTTPS URL") +} + +func validDNSName(host string) bool { + if host == "" || len(host) > 253 || strings.HasSuffix(host, ".") || strings.EqualFold(host, "localhost") { + return false + } + labels := strings.Split(host, ".") + if len(labels) < 2 { + return false + } + for _, label := range labels { + if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for _, character := range label { + if character > 127 || !(character == '-' || character >= '0' && character <= '9' || character >= 'A' && character <= 'Z' || character >= 'a' && character <= 'z') { + return false + } + } + } + return true +} + +func safeClient(timeout time.Duration) *http.Client { + dialer := &net.Dialer{Timeout: timeout, KeepAlive: 30 * time.Second} + transport := &http.Transport{ + Proxy: nil, + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil || port != "443" || !validDNSName(host) { + return nil, errors.New("web push dial target is invalid") + } + addresses, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil || len(addresses) == 0 { + return nil, errors.New("resolve web push endpoint") + } + var last error + for _, candidate := range addresses { + if !publicIP(candidate.IP) { + return nil, errors.New("web push endpoint resolved to a non-public address") + } + connection, dialErr := dialer.DialContext(ctx, network, net.JoinHostPort(candidate.IP.String(), port)) + if dialErr == nil { + return connection, nil + } + last = dialErr + } + if last != nil { + return nil, errors.New("connect to web push endpoint") + } + return nil, errors.New("web push endpoint has no usable address") + }, + ForceAttemptHTTP2: true, + TLSHandshakeTimeout: timeout, + ResponseHeaderTimeout: timeout, + MaxIdleConns: 16, + MaxIdleConnsPerHost: 4, + IdleConnTimeout: 30 * time.Second, + } + return &http.Client{Transport: transport, Timeout: timeout, CheckRedirect: func(*http.Request, []*http.Request) error { + return errors.New("web push redirects are disabled") + }} +} + +func publicIP(ip net.IP) bool { + if ip == nil || ip.IsUnspecified() || ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() { + return false + } + address, ok := netip.AddrFromSlice(ip) + if !ok { + return false + } + address = address.Unmap() + for _, prefix := range deniedPushPrefixes { + if prefix.Contains(address) { + return false + } + } + return true +} + +var deniedPushPrefixes = []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/8"), + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("192.0.0.0/24"), + netip.MustParsePrefix("192.0.2.0/24"), + netip.MustParsePrefix("198.18.0.0/15"), + netip.MustParsePrefix("198.51.100.0/24"), + netip.MustParsePrefix("203.0.113.0/24"), + netip.MustParsePrefix("240.0.0.0/4"), + netip.MustParsePrefix("2001:db8::/32"), +} + +func encrypt(payload []byte, subscription Subscription, random io.Reader) ([]byte, []byte, error) { + if len(payload) == 0 || len(payload) > 2048 || len(subscription.P256DH) != 65 || subscription.P256DH[0] != 4 || len(subscription.Auth) != 16 { + return nil, nil, errors.New("web push subscription keys are invalid") + } + serverPrivate, err := ecdh.P256().GenerateKey(random) + if err != nil { + return nil, nil, errors.New("generate web push content key") + } + salt := make([]byte, 16) + if _, err = io.ReadFull(random, salt); err != nil { + return nil, nil, errors.New("generate web push salt") + } + return encryptWithMaterial(payload, subscription, serverPrivate, salt) +} + +func encryptWithMaterial(payload []byte, subscription Subscription, serverPrivate *ecdh.PrivateKey, salt []byte) ([]byte, []byte, error) { + if len(payload) == 0 || len(payload) > 2048 || len(subscription.P256DH) != 65 || subscription.P256DH[0] != 4 || len(subscription.Auth) != 16 || serverPrivate == nil || len(salt) != 16 { + return nil, nil, errors.New("web push subscription keys are invalid") + } + clientPublic, err := ecdh.P256().NewPublicKey(subscription.P256DH) + if err != nil { + return nil, nil, errors.New("web push subscription public key is invalid") + } + shared, err := serverPrivate.ECDH(clientPublic) + if err != nil { + return nil, nil, errors.New("derive web push content key") + } + serverPublic := serverPrivate.PublicKey().Bytes() + keyInfo := append([]byte("WebPush: info\x00"), subscription.P256DH...) + keyInfo = append(keyInfo, serverPublic...) + prkKey := hkdfExtract(subscription.Auth, shared) + ikm := hkdfExpand(prkKey, keyInfo, 32) + prk := hkdfExtract(salt, ikm) + contentKey := hkdfExpand(prk, []byte("Content-Encoding: aes128gcm\x00"), 16) + nonce := hkdfExpand(prk, []byte("Content-Encoding: nonce\x00"), 12) + block, err := aes.NewCipher(contentKey) + if err != nil { + return nil, nil, errors.New("create web push cipher") + } + aead, err := cipher.NewGCM(block) + if err != nil { + return nil, nil, errors.New("create web push authenticated cipher") + } + plaintext := append(append([]byte(nil), payload...), 2) + ciphertext := aead.Seal(nil, nonce, plaintext, nil) + recordSize := uint32(4096) + body := make([]byte, 0, 16+4+1+len(serverPublic)+len(ciphertext)) + body = append(body, salt...) + record := make([]byte, 4) + binary.BigEndian.PutUint32(record, recordSize) + body = append(body, record...) + body = append(body, byte(len(serverPublic))) + body = append(body, serverPublic...) + body = append(body, ciphertext...) + return body, serverPublic, nil +} + +func (s *Sender) vapid(endpoint *url.URL, now time.Time, random io.Reader) (string, error) { + header, _ := json.Marshal(struct { + Type string `json:"typ"` + Algorithm string `json:"alg"` + }{"JWT", "ES256"}) + audience := endpoint.Scheme + "://" + strings.ToLower(endpoint.Hostname()) + payload, _ := json.Marshal(struct { + Audience string `json:"aud"` + Expiry int64 `json:"exp"` + Subject string `json:"sub"` + }{audience, now.UTC().Add(12 * time.Hour).Unix(), s.subject}) + unsigned := base64.RawURLEncoding.EncodeToString(header) + "." + base64.RawURLEncoding.EncodeToString(payload) + digest := sha256.Sum256([]byte(unsigned)) + d := new(big.Int).SetBytes(s.private.Bytes()) + curve := elliptic.P256() + x, y := curve.ScalarBaseMult(s.private.Bytes()) + private := &ecdsa.PrivateKey{PublicKey: ecdsa.PublicKey{Curve: curve, X: x, Y: y}, D: d} + r, signatureS, err := ecdsa.Sign(random, private, digest[:]) + if err != nil { + return "", errors.New("sign VAPID token") + } + signature := make([]byte, 64) + r.FillBytes(signature[:32]) + signatureS.FillBytes(signature[32:]) + return unsigned + "." + base64.RawURLEncoding.EncodeToString(signature), nil +} + +func hkdfExtract(salt, input []byte) []byte { + mac := hmac.New(sha256.New, salt) + _, _ = mac.Write(input) + return mac.Sum(nil) +} + +func hkdfExpand(key, info []byte, length int) []byte { + var output, previous []byte + for counter := byte(1); len(output) < length; counter++ { + mac := hmac.New(sha256.New, key) + _, _ = mac.Write(previous) + _, _ = mac.Write(info) + _, _ = mac.Write([]byte{counter}) + previous = mac.Sum(nil) + output = append(output, previous...) + } + return output[:length] +} diff --git a/internal/webpush/sender_test.go b/internal/webpush/sender_test.go new file mode 100644 index 0000000..e6f0ca3 --- /dev/null +++ b/internal/webpush/sender_test.go @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package webpush + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/ecdh" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "io" + "math/big" + "net" + "net/http" + "strings" + "testing" + "time" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (function roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return function(request) +} + +func TestSenderUsesEncryptedGenericPayloadAndValidVAPID(t *testing.T) { + serverKey, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + clientKey, err := ecdh.P256().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + auth := make([]byte, 16) + if _, err = rand.Read(auth); err != nil { + t.Fatal(err) + } + var captured *http.Request + var encrypted []byte + client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + captured = request.Clone(context.Background()) + encrypted, err = io.ReadAll(request.Body) + if err != nil { + return nil, err + } + return &http.Response{StatusCode: http.StatusCreated, Body: io.NopCloser(strings.NewReader("accepted")), Header: make(http.Header)}, nil + })} + fixed := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC) + sender, err := New(Options{PrivateKey: serverKey.Bytes(), Subject: "mailto:security@sandwichhime.com", Timeout: 5 * time.Second, Client: client, Now: func() time.Time { return fixed }}) + if err != nil { + t.Fatal(err) + } + subscription := Subscription{Endpoint: "https://push.example.test/send/opaque-token", P256DH: clientKey.PublicKey().Bytes(), Auth: auth} + if err = sender.Send(context.Background(), subscription); err != nil { + t.Fatal(err) + } + if captured == nil || captured.Method != http.MethodPost || captured.URL.String() != subscription.Endpoint { + t.Fatalf("request=%v", captured) + } + if captured.Header.Get("Content-Encoding") != "aes128gcm" || captured.Header.Get("TTL") != "300" || captured.Header.Get("Urgency") != "high" { + t.Fatalf("headers=%v", captured.Header) + } + if len(encrypted) < 100 || strings.Contains(string(encrypted), GenericMessage) { + t.Fatalf("payload length=%d plaintext=%t", len(encrypted), strings.Contains(string(encrypted), GenericMessage)) + } + if decrypted := decryptTestPayload(t, encrypted, clientKey, auth); string(decrypted) != GenericMessage { + t.Fatalf("decrypted=%q", decrypted) + } + authorization := captured.Header.Get("Authorization") + if !strings.HasPrefix(authorization, "vapid t=") || !strings.Contains(authorization, ", k="+sender.PublicKey()) { + t.Fatalf("authorization=%q", authorization) + } + token := strings.TrimPrefix(strings.Split(authorization, ", k=")[0], "vapid t=") + parts := strings.Split(token, ".") + if len(parts) != 3 { + t.Fatalf("token parts=%d", len(parts)) + } + payloadBytes, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatal(err) + } + var payload struct { + Audience string `json:"aud"` + Expiry int64 `json:"exp"` + Subject string `json:"sub"` + } + if err = json.Unmarshal(payloadBytes, &payload); err != nil { + t.Fatal(err) + } + if payload.Audience != "https://push.example.test" || payload.Subject != "mailto:security@sandwichhime.com" || payload.Expiry != fixed.Add(12*time.Hour).Unix() { + t.Fatalf("payload=%+v", payload) + } + signature, err := base64.RawURLEncoding.DecodeString(parts[2]) + if err != nil || len(signature) != 64 { + t.Fatalf("signature length=%d err=%v", len(signature), err) + } + publicBytes, err := base64.RawURLEncoding.DecodeString(sender.PublicKey()) + if err != nil { + t.Fatal(err) + } + x, y := elliptic.Unmarshal(elliptic.P256(), publicBytes) + digest := sha256.Sum256([]byte(parts[0] + "." + parts[1])) + if !ecdsa.Verify(&ecdsa.PublicKey{Curve: elliptic.P256(), X: x, Y: y}, digest[:], new(big.Int).SetBytes(signature[:32]), new(big.Int).SetBytes(signature[32:])) { + t.Fatal("VAPID signature did not verify") + } +} + +func TestEncryptionMatchesRFC8291SectionFiveVector(t *testing.T) { + decode := func(value string) []byte { + t.Helper() + decoded, err := base64.RawURLEncoding.DecodeString(value) + if err != nil { + t.Fatal(err) + } + return decoded + } + serverPrivate, err := ecdh.P256().NewPrivateKey(decode("yfWPiYE-n46HLnH0KqZOF1fJJU3MYrct3AELtAQ-oRw")) + if err != nil { + t.Fatal(err) + } + subscription := Subscription{ + P256DH: decode("BCVxsr7N_eNgVRqvHtD0zTZsEc6-VV-JvLexhqUzORcxaOzi6-AYWXvTBHm4bjyPjs7Vd8pZGH6SRpkNtoIAiw4"), + Auth: decode("BTBZMqHH6r4Tts7J_aSIgg"), + } + payload := decode("V2hlbiBJIGdyb3cgdXAsIEkgd2FudCB0byBiZSBhIHdhdGVybWVsb24") + body, _, err := encryptWithMaterial(payload, subscription, serverPrivate, decode("DGv6ra1nlYgDCS1FRnbzlw")) + if err != nil { + t.Fatal(err) + } + expected := decode("DGv6ra1nlYgDCS1FRnbzlwAAEABBBP4z9KsN6nGRTbVYI_c7VJSPQTBtkgcy27mlmlMoZIIgDll6e3vCYLocInmYWAmS6TlzAC8wEqKK6PBru3jl7A_yl95bQpu6cVPTpK4Mqgkf1CXztLVBSt2Ks3oZwbuwXPXLWyouBWLVWGNWQexSgSxsj_Qulcy4a-fN") + if string(body) != string(expected) { + t.Fatalf("RFC 8291 vector mismatch\n got: %s\nwant: %s", base64.RawURLEncoding.EncodeToString(body), base64.RawURLEncoding.EncodeToString(expected)) + } +} + +func TestSenderClassifiesGoneSubscription(t *testing.T) { + serverKey, _ := ecdh.P256().GenerateKey(rand.Reader) + clientKey, _ := ecdh.P256().GenerateKey(rand.Reader) + auth := make([]byte, 16) + _, _ = rand.Read(auth) + sender, err := New(Options{PrivateKey: serverKey.Bytes(), Subject: "https://observatory.example/security", Timeout: time.Second, Client: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusGone, Body: io.NopCloser(strings.NewReader("expired")), Header: make(http.Header)}, nil + })}}) + if err != nil { + t.Fatal(err) + } + err = sender.Send(context.Background(), Subscription{Endpoint: "https://push.example.test/s/expired", P256DH: clientKey.PublicKey().Bytes(), Auth: auth}) + if !errors.Is(err, ErrSubscriptionGone) { + t.Fatalf("err=%v", err) + } +} + +func TestEndpointAndAddressValidation(t *testing.T) { + valid := []string{"https://push.example.test/send/token", "https://push.example.test:443/send/token", "https://push.example.test/send/token?opaque=one"} + for _, candidate := range valid { + if _, err := ValidateEndpoint(candidate); err != nil { + t.Errorf("valid endpoint %q: %v", candidate, err) + } + } + invalid := []string{ + "http://push.example.test/send/token", "https://localhost/send/token", "https://127.0.0.1/send/token", + "https://push.example.test:8443/send/token", "https://user@push.example.test/send/token", "https://push.example.test", + "https://push.example.test/send/token#secret", + } + for _, candidate := range invalid { + if _, err := ValidateEndpoint(candidate); err == nil { + t.Errorf("invalid endpoint accepted: %q", candidate) + } + } + for _, candidate := range []string{"127.0.0.1", "10.0.0.1", "169.254.1.1", "224.0.0.1", "::1", "fc00::1", "fe80::1"} { + if publicIP(net.ParseIP(candidate)) { + t.Errorf("non-public address accepted: %s", candidate) + } + } + for _, candidate := range []string{"100.64.0.1", "192.0.2.1", "198.18.0.1", "203.0.113.1", "2001:db8::1"} { + if publicIP(net.ParseIP(candidate)) { + t.Errorf("special-use address accepted: %s", candidate) + } + } + for _, candidate := range []string{"8.8.8.8", "2606:4700:4700::1111"} { + if !publicIP(net.ParseIP(candidate)) { + t.Errorf("public address rejected: %s", candidate) + } + } +} + +func TestSenderRejectsInvalidInputs(t *testing.T) { + private, _ := ecdh.P256().GenerateKey(rand.Reader) + for _, test := range []Options{ + {PrivateKey: []byte("short"), Subject: "mailto:security@example.test", Timeout: time.Second}, + {PrivateKey: private.Bytes(), Subject: "javascript:alert(1)", Timeout: time.Second}, + {PrivateKey: private.Bytes(), Subject: "mailto:security@example.test", Timeout: time.Millisecond}, + } { + if _, err := New(test); err == nil { + t.Fatalf("invalid options accepted: %+v", test) + } + } +} + +func decryptTestPayload(t *testing.T, body []byte, clientPrivate *ecdh.PrivateKey, auth []byte) []byte { + t.Helper() + if len(body) < 16+4+1+65+16 || binary.BigEndian.Uint32(body[16:20]) != 4096 || body[20] != 65 { + t.Fatalf("invalid aes128gcm record length=%d", len(body)) + } + return decryptWithAuth(t, body, clientPrivate, auth) +} + +func decryptWithAuth(t *testing.T, body []byte, clientPrivate *ecdh.PrivateKey, auth []byte) []byte { + t.Helper() + salt := body[:16] + serverPublicBytes := body[21:86] + serverPublic, err := ecdh.P256().NewPublicKey(serverPublicBytes) + if err != nil { + t.Fatal(err) + } + shared, err := clientPrivate.ECDH(serverPublic) + if err != nil { + t.Fatal(err) + } + keyInfo := append([]byte("WebPush: info\x00"), clientPrivate.PublicKey().Bytes()...) + keyInfo = append(keyInfo, serverPublicBytes...) + ikm := hkdfExpand(hkdfExtract(auth, shared), keyInfo, 32) + prk := hkdfExtract(salt, ikm) + block, err := aes.NewCipher(hkdfExpand(prk, []byte("Content-Encoding: aes128gcm\x00"), 16)) + if err != nil { + t.Fatal(err) + } + var aead cipher.AEAD + aead, err = cipher.NewGCM(block) + if err != nil { + t.Fatal(err) + } + plaintext, err := aead.Open(nil, hkdfExpand(prk, []byte("Content-Encoding: nonce\x00"), 12), body[86:], nil) + if err != nil { + t.Fatal(err) + } + if len(plaintext) < 1 || plaintext[len(plaintext)-1] != 2 { + t.Fatalf("invalid record delimiter: %x", plaintext) + } + return plaintext[:len(plaintext)-1] +} diff --git a/release/Caddyfile.observatory b/release/Caddyfile.observatory new file mode 100644 index 0000000..aa1ee33 --- /dev/null +++ b/release/Caddyfile.observatory @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: AGPL-3.0-only +observatory.gamertan.com { + @responseBody method GET + encode @responseBody zstd gzip + header Strict-Transport-Security "max-age=31536000; includeSubDomains" + log observatory_access { + output file /var/log/caddy/observatory-access.jsonl { + mode 0640 + roll_size 100mb + roll_keep 10 + roll_keep_for 720h + } + format filter { + request>uri regexp \?.*$ "" + request>remote_ip delete + request>remote_port delete + request>client_ip delete + request>headers delete + resp_headers delete + user_id delete + wrap json + } + } + log_append request_id {http.response.header.X-Request-ID} + reverse_proxy 127.0.0.1:8093 +} diff --git a/release/agent.json b/release/agent.json new file mode 100644 index 0000000..abefb24 --- /dev/null +++ b/release/agent.json @@ -0,0 +1,57 @@ +{ + "schema": 1, + "server_url": "https://observatory.gamertan.com", + "credential_file": "/etc/gamertan-observatory/agent-credential.json", + "spool_dir": "/var/lib/gamertan-observatory-agent", + "state_file": "/var/lib/gamertan-observatory-agent/state.json", + "max_spool_bytes": 5368709120, + "max_spool_age": "72h", + "batch_records": 5000, + "flush_interval": "1s", + "sources": [ + { + "kind": "caddy_json", + "path": "/var/log/caddy/eqlwiki-edge.jsonl", + "stream_id": "eql-edge" + }, + { + "kind": "requestlog_jsonl", + "path": "/var/log/eqlwiki/access.jsonl", + "stream_id": "eql-application" + }, + { + "kind": "tend_events_jsonl", + "path": "/opt/gamertancom/deployment-events.jsonl", + "stream_id": "tend-gamertancom" + }, + { + "kind": "tend_events_jsonl", + "path": "/opt/sandwich-hime-site/deployment-events.jsonl", + "stream_id": "tend-sandwich-hime-site" + }, + { + "kind": "tend_events_jsonl", + "path": "/opt/gamertan-observatory/deployment-events.jsonl", + "stream_id": "tend-observatory" + }, + { + "kind": "linux_metrics", + "stream_id": "public-node-host-metrics", + "linux_metrics": { + "proc_root": "/proc", + "cgroup_root": "/sys/fs/cgroup", + "filesystems": [ + { "name": "root", "path": "/" } + ], + "cgroups": [ + { "name": "caddy", "path": "system.slice/caddy.service" }, + { "name": "eql", "path": "system.slice/system-eqlwiki.slice" }, + { "name": "gamertan", "path": "system.slice/system-gamertancom.slice" }, + { "name": "sandwich-hime", "path": "system.slice/sandwich-hime-site.service" }, + { "name": "observatory", "path": "system.slice/gamertan-observatory.service" }, + { "name": "observatory-agent", "path": "system.slice/gamertan-observatory-agent.service" } + ] + } + } + ] +} diff --git a/release/agent.json.license b/release/agent.json.license new file mode 100644 index 0000000..10e6ada --- /dev/null +++ b/release/agent.json.license @@ -0,0 +1 @@ +SPDX-License-Identifier: AGPL-3.0-only diff --git a/release/observatory-agent.service b/release/observatory-agent.service new file mode 100644 index 0000000..649fadb --- /dev/null +++ b/release/observatory-agent.service @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: AGPL-3.0-only +[Unit] +Description=Gamertan Observatory production agent +After=network-online.target gamertan-observatory.service +Wants=network-online.target +PartOf=gamertan-observatory.service + +[Service] +Type=simple +User=observatory-agent +Group=observatory-agent +SupplementaryGroups=caddy eqlwiki gamertancom +LoadCredential=agent.json:/etc/gamertan-observatory/agent.json +LoadCredential=agent-credential.json:/etc/gamertan-observatory/agent-credential.json +ExecStart=/opt/gamertan-observatory/current/observatory agent --systemd-credentials --config %d/agent.json --credential-file %d/agent-credential.json +Restart=on-failure +RestartSec=5s +TimeoutStartSec=20s +TimeoutStopSec=20s +KillMode=mixed +UMask=0077 + +StateDirectory=gamertan-observatory-agent +StateDirectoryMode=0700 +NoNewPrivileges=yes +PrivateDevices=yes +PrivateTmp=yes +ProtectClock=yes +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelLogs=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectProc=invisible +ProtectSystem=strict +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +SystemCallArchitectures=native +CapabilityBoundingSet= +AmbientCapabilities= +ReadOnlyPaths=/proc /sys/fs/cgroup /var/log/caddy /var/log/eqlwiki /opt/gamertancom/deployment-events.jsonl /opt/sandwich-hime-site/deployment-events.jsonl /opt/gamertan-observatory/deployment-events.jsonl +ReadWritePaths=/var/lib/gamertan-observatory-agent +TasksMax=64 +MemoryMax=256M +LimitNOFILE=1024 + +[Install] +WantedBy=multi-user.target diff --git a/release/observatory.env.example b/release/observatory.env.example new file mode 100644 index 0000000..40368dd --- /dev/null +++ b/release/observatory.env.example @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Intentionally empty. Secrets use systemd credentials. Tend supplies the +# candidate-only OBSERVATORY_TEND_CANDIDATE_LISTEN value itself. diff --git a/release/observatory.service b/release/observatory.service new file mode 100644 index 0000000..2995bbb --- /dev/null +++ b/release/observatory.service @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: AGPL-3.0-only +[Unit] +Description=Gamertan Observatory +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=gamertan-observatory +Group=gamertan-observatory +LoadCredential=server.json:/etc/gamertan-observatory/server.json +LoadCredential=web-push.json:/etc/gamertan-observatory/web-push.json +EnvironmentFile=/etc/tend/environment/observatory.env +ExecStart=/opt/gamertan-observatory/current/observatory server --config %d/server.json --systemd-credential-config +Restart=on-failure +RestartSec=2s +TimeoutStartSec=20s +TimeoutStopSec=20s +KillMode=mixed +UMask=0077 + +StateDirectory=gamertan-observatory +StateDirectoryMode=0700 +NoNewPrivileges=yes +PrivateDevices=yes +PrivateTmp=yes +ProtectClock=yes +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelLogs=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectProc=invisible +ProtectSystem=strict +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +SystemCallArchitectures=native +CapabilityBoundingSet= +AmbientCapabilities= +TasksMax=128 +MemoryMax=768M +LimitNOFILE=8192 + +[Install] +WantedBy=multi-user.target diff --git a/release/server.json b/release/server.json new file mode 100644 index 0000000..3376a69 --- /dev/null +++ b/release/server.json @@ -0,0 +1,30 @@ +{ + "schema": 1, + "listen": "127.0.0.1:8093", + "public_url": "https://observatory.gamertan.com", + "data_dir": "/var/lib/gamertan-observatory", + "max_body_bytes": 33554432, + "max_concurrent_ingest": 8, + "session_lifetime": "12h", + "query": { + "max_duration": "5s", + "max_rows": 10000, + "max_scanned_bytes": 268435456, + "max_memory_bytes": 134217728 + }, + "retention": { + "raw_logs_days": 30, + "raw_traces_days": 30, + "raw_metrics_days": 14, + "cold_raw_days": 400, + "delete_cold_raw": false, + "metric_rollups_days": 400, + "evidence_days": 400 + }, + "web_push": { + "private_key_file": "/run/credentials/gamertan-observatory.service/web-push.json", + "subject": "mailto:security@sandwichhime.com", + "queue_capacity": 64, + "request_timeout": "10s" + } +} diff --git a/release/server.json.license b/release/server.json.license new file mode 100644 index 0000000..10e6ada --- /dev/null +++ b/release/server.json.license @@ -0,0 +1 @@ +SPDX-License-Identifier: AGPL-3.0-only diff --git a/release/tend.json b/release/tend.json new file mode 100644 index 0000000..c70c9b5 --- /dev/null +++ b/release/tend.json @@ -0,0 +1,50 @@ +{ + "schema_version": 2, + "service": { + "name": "observatory", + "allowed_host": "observatory.gamertan.com", + "environment_file": "/etc/tend/environment/observatory.env" + }, + "build": { + "package": "./cmd/observatory", + "binary": "observatory", + "branch": "main", + "version_symbol": "gamertan.com/observatory/internal/version.Version", + "commit_symbol": "gamertan.com/observatory/internal/version.Commit", + "date_symbol": "gamertan.com/observatory/internal/version.Date" + }, + "deployment": { + "strategy": "singleton_candidate", + "root": "/opt/gamertan-observatory", + "lock_file": "/run/lock/tend-deploy.lock", + "state_file": "/opt/gamertan-observatory/tend-state.json", + "event_log": "/opt/gamertan-observatory/deployment-events.jsonl", + "health_path": "/healthz", + "readiness_path": "/readyz", + "candidate_timeout_seconds": 30, + "activation_window_seconds": 10, + "smoke": [ + { + "path": "/", + "contains": "Gamertan Observatory" + } + ], + "public_smoke": [ + { + "url": "https://observatory.gamertan.com/", + "contains": "Gamertan Observatory" + } + ], + "singleton": { + "unit": "gamertan-observatory.service", + "address": "127.0.0.1:8093", + "candidate_address": "127.0.0.1:18093", + "listen_env": "OBSERVATORY_TEND_CANDIDATE_LISTEN", + "current_link": "/opt/gamertan-observatory/current", + "previous_link": "/opt/gamertan-observatory/previous", + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/observatory.caddy", + "caddy_handler_template": "/etc/tend/caddy/observatory.template" + } + } +} diff --git a/release/tend.json.license b/release/tend.json.license new file mode 100644 index 0000000..10e6ada --- /dev/null +++ b/release/tend.json.license @@ -0,0 +1 @@ +SPDX-License-Identifier: AGPL-3.0-only diff --git a/scripts/browser-campaign.sh b/scripts/browser-campaign.sh new file mode 100755 index 0000000..fa3cea7 --- /dev/null +++ b/scripts/browser-campaign.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$root" + +test "$(node -p 'require("./test/browser/node_modules/@playwright/test/package.json").version')" = 1.62.1 + +campaign_dir=$(mktemp -d) +fixture_pid= +cleanup() { + if test -n "$fixture_pid"; then + kill -TERM "$fixture_pid" 2>/dev/null || true + wait "$fixture_pid" 2>/dev/null || true + fi + rm -rf "$campaign_dir" +} +trap cleanup EXIT + +fixture=${OBSERVATORY_BROWSER_FIXTURE:-} +if test -z "$fixture"; then + test "$(go env GOVERSION)" = go1.26.6 + go build -buildvcs=false -trimpath -tags observatory_browser_fixture -o "$campaign_dir/fixture" ./internal/browsertest + fixture="$campaign_dir/fixture" +else + case "$fixture" in /*) ;; *) exit 1;; esac + test -x "$fixture" +fi +"$fixture" >"$campaign_dir/origin" 2>"$campaign_dir/fixture.log" & +fixture_pid=$! +for _ in $(seq 1 100); do + if test "$(wc -l <"$campaign_dir/origin")" -ge 2; then break; fi + if ! kill -0 "$fixture_pid" 2>/dev/null; then + cat "$campaign_dir/fixture.log" >&2 + exit 1 + fi + sleep 0.1 +done +origin=$(head -n 1 "$campaign_dir/origin") +spki=$(sed -n '2p' "$campaign_dir/origin") +case "$origin" in https://localhost:*) ;; *) cat "$campaign_dir/fixture.log" >&2; exit 1;; esac +case "$spki" in *[!A-Za-z0-9+/=]*|'') exit 1;; esac + +OBSERVATORY_BROWSER_ORIGIN="$origin" OBSERVATORY_BROWSER_SPKI="$spki" node test/browser/campaign.mjs +if test -d .git; then + test -z "$(git status --porcelain=v1 --untracked-files=all -- . ':!test/browser/node_modules')" +fi diff --git a/scripts/capacity-campaign.sh b/scripts/capacity-campaign.sh new file mode 100755 index 0000000..e5cd2a6 --- /dev/null +++ b/scripts/capacity-campaign.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$root" +test "$(uname -s)" = Linux + +campaign_dir=$(mktemp -d) +cleanup() { rm -rf "$campaign_dir"; } +trap cleanup EXIT +fixture=${OBSERVATORY_CAPACITY_FIXTURE:-} +if test -z "$fixture"; then + test "$(go env GOVERSION)" = go1.26.6 + CGO_ENABLED=0 go build -buildvcs=false -trimpath -tags observatory_capacity_fixture -o "$campaign_dir/observatory-capacity" ./internal/capacitytest + fixture="$campaign_dir/observatory-capacity" +else + case "$fixture" in /*) ;; *) exit 2;; esac + test -x "$fixture" +fi + +case "${OBSERVATORY_CAPACITY_MODE:-development}" in + development) + "$fixture" \ + -sustain-rate 200 -sustain-duration 5s \ + -burst-rate 1000 -burst-duration 2s \ + -minimum-primary-observations 20000 -query-iterations 3 + ;; + release) + "$fixture" \ + -require-cgroup -expected-cpus 4 -expected-memory-bytes 8589934592 + ;; + *) exit 2;; +esac + +test -z "$(git status --porcelain=v1 --untracked-files=all)" diff --git a/scripts/check-licenses.sh b/scripts/check-licenses.sh new file mode 100755 index 0000000..3249e40 --- /dev/null +++ b/scripts/check-licenses.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +test -s "$root/LICENSE" +test -s "$root/COPYRIGHT" +test -s "$root/examples/LICENSE" + +while IFS= read -r file; do + case $file in + *.sando.go) + grep -Fq '// Code generated by himesan; DO NOT EDIT.' "$file" || { echo "generated provenance missing: $file" >&2; exit 1; } + source=${file%.go} + grep -Fq 'SPDX-License-Identifier: AGPL-3.0-only' "$source" || { echo "generated source licence missing: $source" >&2; exit 1; } + ;; + *) + grep -Fq 'SPDX-License-Identifier: AGPL-3.0-only' "$file" || { echo "missing AGPL SPDX identifier: $file" >&2; exit 1; } + ;; + esac +done < <(find "$root/cmd" "$root/internal" -type f \( -name '*.go' -o -name '*.sando' -o -name '*.css' -o -name '*.js' -o -name '*.svg' \) -print | LC_ALL=C sort) + +while IFS= read -r file; do + grep -Fq 'SPDX-License-Identifier: AGPL-3.0-only' "$file" || { echo "missing AGPL SPDX identifier: $file" >&2; exit 1; } +done < <(find "$root/docs" "$root/scripts" -type f \( -name '*.md' -o -name '*.sh' \) -print | LC_ALL=C sort) + +for file in Caddyfile.observatory observatory-agent.service observatory.env.example observatory.service; do + grep -Fq 'SPDX-License-Identifier: AGPL-3.0-only' "$root/release/$file" || { echo "missing AGPL SPDX identifier: release/$file" >&2; exit 1; } +done +for file in agent.json server.json tend.json; do + test "$(cat "$root/release/$file.license")" = 'SPDX-License-Identifier: AGPL-3.0-only' +done + +for file in README.md SECURITY.md LICENSES.md .gitignore .gitattributes; do + grep -Fq 'SPDX-License-Identifier: AGPL-3.0-only' "$root/$file" || { echo "missing AGPL SPDX identifier: $file" >&2; exit 1; } +done +for file in Caddyfile.sensitive-access-log README.md observatory-agent.service; do + grep -Fq 'SPDX-License-Identifier: 0BSD' "$root/examples/$file" || { echo "missing 0BSD SPDX identifier: examples/$file" >&2; exit 1; } +done +for file in agent-credential.json agent-enrollment.json agent.json server.json; do + test "$(cat "$root/examples/$file.license")" = 'SPDX-License-Identifier: 0BSD' +done +grep -Fq '"license": "AGPL-3.0-only"' "$root/test/browser/package.json" +grep -Fq '"license": "AGPL-3.0-only"' "$root/test/browser/package-lock.json" +echo "license boundaries verified" diff --git a/scripts/export-public.sh b/scripts/export-public.sh new file mode 100755 index 0000000..2eb0e75 --- /dev/null +++ b/scripts/export-public.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +usage(){ echo "Usage: export-public.sh --destination ABSOLUTE-PATH" >&2; } +destination= +while [[ $# -gt 0 ]]; do + case $1 in + --destination) destination=$2; shift 2 ;; + *) usage; exit 2 ;; + esac +done +[[ $destination == /* && $destination != / && ! -e $destination ]] || { usage; exit 2; } +destination=$(realpath -m "$destination"); root=$(realpath -e "$root") +case $destination/ in "$root"/*) echo "destination must be outside the private worktree" >&2; exit 1;; esac +git_dir_raw=$(git -C "$root" rev-parse --git-dir) +common_dir_raw=$(git -C "$root" rev-parse --git-common-dir) +[[ $git_dir_raw == /* ]] || git_dir_raw=$root/$git_dir_raw +[[ $common_dir_raw == /* ]] || common_dir_raw=$root/$common_dir_raw +git_dir=$(realpath -e "$git_dir_raw") +common_dir=$(realpath -e "$common_dir_raw") +case $destination/ in "$git_dir"/*|"$common_dir"/*) echo "destination must be outside Git metadata" >&2; exit 1;; esac +[[ -z $(git -C "$root" status --porcelain=v1 --untracked-files=all) ]] || { echo "private worktree is not clean" >&2; exit 1; } +commit=$(git -C "$root" rev-parse HEAD) +tree=$(git -C "$root" rev-parse HEAD^{tree}) +remote=$(git -C "$root" ls-remote --exit-code origin refs/heads/main | awk 'NR==1{print $1}') +[[ $remote == "$commit" ]] || { echo "private HEAD is not exact pushed origin/main" >&2; exit 1; } +allow=$root/scripts/public-snapshot.allow +LC_ALL=C sort -c "$allow" +[[ $(LC_ALL=C sort "$allow" | uniq -d | wc -l) -eq 0 ]] +mapfile -t files <"$allow" +[[ ${#files[@]} -gt 0 ]] +for file in "${files[@]}"; do + [[ -n $file && $file != /* && $file != *..* && $file != .gitea/* && $file != .github/* ]] + git -C "$root" cat-file -e "$commit:$file" +done +parent=$(dirname "$destination") +mkdir -p "$parent" +stage=$(mktemp -d "$parent/.observatory-public.XXXXXX") +trap 'rm -rf -- "$stage"' EXIT +git -C "$root" archive "$commit" -- "${files[@]}" | tar -xf - -C "$stage" +while IFS= read -r file; do + relative=${file#"$stage"/} + cmp "$file" "$root/$relative" +done < <(find "$stage" -type f -print | LC_ALL=C sort) +epoch=$(git -C "$root" show -s --format=%ct "$commit") +printf '{"schema_version":1,"source_commit":"%s","source_tree":"%s","source_date_epoch":%s,"file_count":%d}\n' "$commit" "$tree" "$epoch" "${#files[@]}" >"$stage/PUBLIC-SNAPSHOT.json" +(cd "$stage" && sha256sum PUBLIC-SNAPSHOT.json >PUBLIC-SNAPSHOT.sha256) +if (cd "$stage" && rg -n --hidden --glob '!.git/**' --glob '!PUBLIC-SNAPSHOT.json' --glob '!scripts/export-public.sh' '/home/cole|/tmp/gamertan-observatory-identity|BEGIN (RSA|OPENSSH|EC) PRIVATE KEY|gitea-api\.token|observatory-dev\.git|192\.168\.' .); then + echo "private material found" >&2 + exit 1 +fi +mv "$stage" "$destination" +trap - EXIT +printf 'destination=%s\nsource_commit=%s\nsource_tree=%s\n' "$destination" "$commit" "$tree" diff --git a/scripts/preview-gate.sh b/scripts/preview-gate.sh new file mode 100755 index 0000000..4b474c2 --- /dev/null +++ b/scripts/preview-gate.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$root" + +# Complete but bounded dogfood evidence. Release-scale capacity and extended +# assurance remain explicit milestone campaigns. +./scripts/verify.sh + +fuzz_time=${OBSERVATORY_PREVIEW_FUZZ_TIME:-5s} +go test -buildvcs=false ./internal/query -run '^$' -fuzz '^FuzzParse$' -fuzztime "$fuzz_time" +go test -buildvcs=false ./internal/nativeprotocol -run '^$' -fuzz '^FuzzParseEnvelopeHeaders$' -fuzztime "$fuzz_time" +go test -buildvcs=false ./internal/otlp -run '^$' -fuzz '^FuzzDecode$' -fuzztime "$fuzz_time" +go test -buildvcs=false ./internal/collector -run '^$' -fuzz '^FuzzTendCollector$' -fuzztime "$fuzz_time" + +OBSERVATORY_CAPACITY_MODE=development ./scripts/capacity-campaign.sh +test -z "$(git status --porcelain=v1 --untracked-files=all)" diff --git a/scripts/public-snapshot.allow b/scripts/public-snapshot.allow new file mode 100644 index 0000000..135d06b --- /dev/null +++ b/scripts/public-snapshot.allow @@ -0,0 +1,199 @@ +.gitattributes +.gitignore +COPYRIGHT +LICENSE +LICENSES.md +README.md +SECURITY.md +cmd/observatory/main.go +cmd/observatory/main_test.go +cmd/observatory/release_test.go +docs/ADAPTIVE_PROJECTIONS.md +docs/AGENT.md +docs/ARCHITECTURE.md +docs/BOOTSTRAP.md +docs/CAPACITY.md +docs/DASHBOARDS.md +docs/DEPLOYMENT.md +docs/INCIDENTS.md +docs/INTERFACE.md +docs/PERFORMANCE.md +docs/PUBLIC_SNAPSHOT.md +docs/PWA.md +docs/QUERY.md +docs/RECOVERY.md +docs/RETENTION.md +docs/ROADMAP.md +docs/SCHEMA.md +docs/SECURITY_CAMPAIGN.md +docs/TEND.md +examples/Caddyfile.sensitive-access-log +examples/LICENSE +examples/README.md +examples/agent-credential.json +examples/agent-credential.json.license +examples/agent-enrollment.json +examples/agent-enrollment.json.license +examples/agent.json +examples/agent.json.license +examples/observatory-agent.service +examples/server.json +examples/server.json.license +go.mod +go.sum +internal/agent/agent.go +internal/agent/agent_test.go +internal/agentclient/client.go +internal/agentclient/client_test.go +internal/agentstate/state.go +internal/agentstate/state_test.go +internal/browsertest/main.go +internal/capacitytest/main.go +internal/capacitytest/main_test.go +internal/collector/collector.go +internal/collector/collector_test.go +internal/config/config.go +internal/config/config_test.go +internal/edgealert/evaluator.go +internal/edgealert/evaluator_test.go +internal/hostmetrics/hostmetrics.go +internal/hostmetrics/hostmetrics_test.go +internal/httpserver/explore.go +internal/httpserver/incidents.go +internal/httpserver/manage.go +internal/httpserver/native_replay_benchmark_test.go +internal/httpserver/push.go +internal/httpserver/pwa.go +internal/httpserver/refresh.go +internal/httpserver/refresh_test.go +internal/httpserver/server.go +internal/httpserver/server_test.go +internal/httpserver/source_alerts_test.go +internal/httpserver/ui.go +internal/httpserver/ui_test.go +internal/identity/identity.go +internal/identity/identity_test.go +internal/identity/retention.go +internal/model/alert_transition.go +internal/model/alert_transition_test.go +internal/model/model.go +internal/model/model_test.go +internal/nativeprotocol/headers.go +internal/nativeprotocol/headers_test.go +internal/otlp/otlp.go +internal/otlp/otlp_test.go +internal/query/match.go +internal/query/match_test.go +internal/query/planner.go +internal/query/planner_test.go +internal/query/query.go +internal/query/query_test.go +internal/query/result.go +internal/schema/schema.go +internal/schema/schema_test.go +internal/segment/store.go +internal/segment/store_test.go +internal/site/app.sando +internal/site/app.sando.go +internal/site/assets.go +internal/site/assets/observatory.svg +internal/site/assets/site.css +internal/site/assets/site.js +internal/site/assets_test.go +internal/site/dashboard.sando +internal/site/dashboard.sando.go +internal/site/explore.sando +internal/site/explore.sando.go +internal/site/head.sando +internal/site/head.sando.go +internal/site/incidents.sando +internal/site/incidents.sando.go +internal/site/landing.sando +internal/site/landing.sando.go +internal/site/login.sando +internal/site/login.sando.go +internal/site/offline.sando +internal/site/offline.sando.go +internal/site/offline_incidents.sando +internal/site/offline_incidents.sando.go +internal/site/password.sando +internal/site/password.sando.go +internal/site/pwa.go +internal/site/pwa_test.go +internal/site/result_table.sando +internal/site/result_table.sando.go +internal/site/site_footer.sando +internal/site/site_footer.sando.go +internal/site/site_header.sando +internal/site/site_header.sando.go +internal/site/views.go +internal/spool/spool.go +internal/spool/spool_test.go +internal/storage/activation.go +internal/storage/activation_test.go +internal/storage/base_indexes.go +internal/storage/base_indexes_test.go +internal/storage/batch_envelope.go +internal/storage/batch_metadata.go +internal/storage/batch_metadata_test.go +internal/storage/cold_query.go +internal/storage/dashboards.go +internal/storage/dashboards_test.go +internal/storage/descriptors.go +internal/storage/descriptors_test.go +internal/storage/incidents.go +internal/storage/incidents_test.go +internal/storage/log_rollup.go +internal/storage/log_summary_query.go +internal/storage/log_summary_query_test.go +internal/storage/migration_test.go +internal/storage/processlock.go +internal/storage/processlock_test.go +internal/storage/projector.go +internal/storage/projector_test.go +internal/storage/push.go +internal/storage/push_test.go +internal/storage/query.go +internal/storage/query_test.go +internal/storage/raw_query_candidate.go +internal/storage/raw_query_candidate_test.go +internal/storage/rebuild.go +internal/storage/rebuild_test.go +internal/storage/retention.go +internal/storage/retention_test.go +internal/storage/rollup.go +internal/storage/rollup_query.go +internal/storage/rollup_query_test.go +internal/storage/source_alerts.go +internal/storage/source_alerts_test.go +internal/storage/storage.go +internal/storage/storage_test.go +internal/tailer/tailer.go +internal/tailer/tailer_test.go +internal/version/version.go +internal/webpush/notifier.go +internal/webpush/notifier_test.go +internal/webpush/sender.go +internal/webpush/sender_test.go +release/Caddyfile.observatory +release/agent.json +release/agent.json.license +release/observatory-agent.service +release/observatory.env.example +release/observatory.service +release/server.json +release/server.json.license +release/tend.json +release/tend.json.license +scripts/browser-campaign.sh +scripts/capacity-campaign.sh +scripts/check-licenses.sh +scripts/export-public.sh +scripts/preview-gate.sh +scripts/public-snapshot.allow +scripts/security-campaign.sh +scripts/test-public-snapshot.sh +scripts/verify.sh +test/browser/campaign.mjs +test/browser/package-lock.json +test/browser/package.json diff --git a/scripts/security-campaign.sh b/scripts/security-campaign.sh new file mode 100755 index 0000000..49644ce --- /dev/null +++ b/scripts/security-campaign.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$root" + +test "$(go env GOVERSION)" = go1.26.6 +fuzz_time=${OBSERVATORY_FUZZ_TIME:-30s} + +go test -buildvcs=false ./internal/query -run '^$' -fuzz '^FuzzParse$' -fuzztime "$fuzz_time" +go test -buildvcs=false ./internal/otlp -run '^$' -fuzz '^FuzzDecode$' -fuzztime "$fuzz_time" +go test -buildvcs=false ./internal/collector -run '^$' -fuzz '^FuzzTendCollector$' -fuzztime "$fuzz_time" + +go test -buildvcs=false -race -count=1 \ + ./cmd/observatory \ + ./internal/config \ + ./internal/httpserver \ + ./internal/identity \ + ./internal/model \ + ./internal/query \ + ./internal/segment \ + ./internal/spool \ + ./internal/storage \ + ./internal/tailer \ + ./internal/webpush + +git diff --check diff --git a/scripts/test-public-snapshot.sh b/scripts/test-public-snapshot.sh new file mode 100755 index 0000000..1f6e2cf --- /dev/null +++ b/scripts/test-public-snapshot.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf -- "$work"' EXIT +destination=$work/public +"$root/scripts/export-public.sh" --destination "$destination" >/dev/null +test -f "$destination/PUBLIC-SNAPSHOT.json" +test -f "$destination/PUBLIC-SNAPSHOT.sha256" +(cd "$destination" && sha256sum -c PUBLIC-SNAPSHOT.sha256) +test ! -e "$destination/.git" +test ! -e "$destination/.gitea" +test ! -e "$destination/.github" +while IFS= read -r file; do + cmp "$root/$file" "$destination/$file" +done <"$root/scripts/public-snapshot.allow" +(cd "$destination" && GOWORK=off go test -buildvcs=false ./...) +(cd "$destination" && GOWORK=off go vet -buildvcs=false ./...) +echo "public snapshot isolation verified" diff --git a/scripts/verify.sh b/scripts/verify.sh new file mode 100755 index 0000000..dc8577c --- /dev/null +++ b/scripts/verify.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$root" +./scripts/check-licenses.sh +test "$(go env GOVERSION)" = go1.26.6 +test "$(himesan version --json)" = '{"compiler":"v1.0.0-beta.2","runtime_abi":"sando.v1","go":"go1.26.6","features":["lsp-stdio"]}' +test "$(go list -m -f '{{.Version}}' gamertan.com/tend)" = v0.2.0-preview.2 +go tool tend version | grep -Fq '"version": "v0.2.0-preview.2"' +go tool tend check --config "$root/release/tend.json" >/dev/null +himesan check -json . >/dev/null + +verify_dir=$(mktemp -d) +trap 'rm -rf "$verify_dir"' EXIT +find internal -type f -name '*.sando.go' -print | LC_ALL=C sort >"$verify_dir/generated" +test -s "$verify_dir/generated" +while IFS= read -r file; do sha256sum "$file"; stat -c '%y %n' "$file"; done <"$verify_dir/generated" >"$verify_dir/before" +himesan generate -json . >/dev/null +himesan generate -json . >/dev/null +while IFS= read -r file; do sha256sum "$file"; stat -c '%y %n' "$file"; done <"$verify_dir/generated" >"$verify_dir/after" +cmp "$verify_dir/before" "$verify_dir/after" + +test -z "$(gofmt -l .)" +go test -buildvcs=false ./... +go test -buildvcs=false -tags observatory_browser_fixture ./internal/browsertest +go test -buildvcs=false -tags observatory_capacity_fixture ./internal/capacitytest +go test -buildvcs=false -race ./... +go vet -buildvcs=false ./... +go build -buildvcs=false -trimpath -o "$verify_dir/observatory" ./cmd/observatory +test "$(go list -buildvcs=false -deps ./cmd/observatory | grep '^gamertan.com/sandwich-hime' || true)" = 'gamertan.com/sandwich-hime/sando' +git diff --check diff --git a/test/browser/campaign.mjs b/test/browser/campaign.mjs new file mode 100644 index 0000000..2170ed7 --- /dev/null +++ b/test/browser/campaign.mjs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +import assert from "node:assert/strict"; +import { chromium } from "playwright"; + +const origin = process.env.OBSERVATORY_BROWSER_ORIGIN; +const certificateSPKI = process.env.OBSERVATORY_BROWSER_SPKI; +assert.match(origin || "", /^https:\/\/localhost:\d+$/); +assert.match(certificateSPKI || "", /^[A-Za-z0-9+/]{43}=$/); + +const evidence = { + version: 1, + browser: "chromium", + installable: false, + offlineShell: false, + privateInbox: false, + privateCacheCleared: false, + sseReconnected: false, + genericNotification: false, + notificationActivated: false, + badge: false, + accessibility: false, + externalRequests: 0 +}; + +const campaignTimeout = setTimeout(() => { + console.error("stage=campaign-timeout"); + process.exit(124); +}, 120000); +const browser = await chromium.launch({headless: true, args: [`--ignore-certificate-errors-spki-list=${certificateSPKI}`]}); +const context = await browser.newContext({ignoreHTTPSErrors: true, viewport: {width: 320, height: 800}}); +let intentionallyOffline = false; +const pageErrors = []; +const consoleErrors = []; +const externalRequests = []; +context.on("request", request => { + const url = new URL(request.url()); + if (url.protocol === "http:" || url.protocol === "https:") { + if (url.origin !== origin) externalRequests.push(url.origin + url.pathname); + } +}); +context.on("page", candidate => { + candidate.on("pageerror", error => pageErrors.push(error.message)); + candidate.on("console", message => { + if (message.type() === "error" && !intentionallyOffline) consoleErrors.push(message.text()); + }); +}); +await context.addInitScript(() => { + Object.defineProperty(navigator, "setAppBadge", {configurable: true, value: async value => { globalThis.__observatoryBadge = value; }}); + Object.defineProperty(navigator, "clearAppBadge", {configurable: true, value: async () => { globalThis.__observatoryBadge = 0; }}); + globalThis.__observatoryPermissionRequests = 0; + const requestPermission = Notification.requestPermission.bind(Notification); + Notification.requestPermission = (...arguments_) => { + globalThis.__observatoryPermissionRequests++; + return requestPermission(...arguments_); + }; +}); + +const page = await context.newPage(); +page.setDefaultTimeout(10000); +page.setDefaultNavigationTimeout(10000); +await page.goto(origin + "/", {waitUntil: "networkidle"}); +console.error("stage=landing"); +assert.equal(await page.locator("h1").textContent(), "Keep the evidence close."); +assert.equal(await page.evaluate(() => globalThis.__observatoryPermissionRequests), 0); +assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= document.documentElement.clientWidth + 1), true); +await page.keyboard.press("Tab"); +assert.equal((await page.locator(":focus").textContent()).trim(), "Skip to content"); +await page.emulateMedia({forcedColors: "active", reducedMotion: "reduce"}); +assert.equal(await page.evaluate(() => matchMedia("(forced-colors: active)").matches && matchMedia("(prefers-reduced-motion: reduce)").matches), true); +assert.equal(await page.locator("main").count(), 1); +assert.equal(await page.locator('nav[aria-label="Primary"]').count(), 1); +evidence.accessibility = true; + +const cdp = await context.newCDPSession(page); +await cdp.send("Page.enable"); +console.error("stage=manifest-request"); +const manifest = await Promise.race([ + cdp.send("Page.getAppManifest"), + new Promise((_, reject) => setTimeout(() => reject(new Error("manifest inspection timed out")), 5000)) +]); +console.error("stage=manifest-response"); +assert.equal(manifest.url, origin + "/manifest.webmanifest"); +assert.equal((manifest.errors || []).length, 0); +const manifestBody = JSON.parse(manifest.data); +assert.equal(manifestBody.start_url, "/app/"); +assert.equal(manifestBody.display, "standalone"); +const workerProbe = await page.evaluate(async () => { + try { + const value = await navigator.serviceWorker.register("/service-worker.js", {scope: "/"}); + return {secure: isSecureContext, scope: value.scope, installing: value.installing?.state || "", waiting: value.waiting?.state || "", active: value.active?.state || ""}; + } catch (error) { + return {secure: isSecureContext, error: `${error.name}: ${error.message}`}; + } +}); +console.error(`stage=service-worker-probe ${JSON.stringify(workerProbe)}`); +assert.equal(workerProbe.secure, true); +assert.equal(workerProbe.error, undefined); +const registration = await page.evaluate(async () => { + const ready = await Promise.race([ + navigator.serviceWorker.ready, + new Promise((_, reject) => setTimeout(() => reject(new Error("service worker readiness timed out")), 5000)) + ]); + return {scope: ready.scope, controlled: Boolean(navigator.serviceWorker.controller)}; +}); +assert.equal(registration.scope, origin + "/"); +assert.equal(registration.controlled, true); +evidence.installable = true; +console.error("stage=installable"); + +await page.goto(origin + "/login/"); +await page.locator("#identifier").fill("browser-operator"); +await page.locator("#password").fill("browser-fixture-password"); +const [loginResponse] = await Promise.all([ + page.waitForResponse(response => new URL(response.url()).pathname === "/login/" && response.request().method() === "POST"), + page.getByRole("button", {name: "Sign in"}).click() +]); +const loginOrigin = await loginResponse.request().headerValue("origin"); +const loginFetchSite = await loginResponse.request().headerValue("sec-fetch-site"); +const loginFailure = loginResponse.status() === 303 ? "" : (await loginResponse.text()).slice(0, 256); +console.error(`stage=login-response status=${loginResponse.status()} origin=${loginOrigin || "missing"} fetch_site=${loginFetchSite || "missing"} body=${JSON.stringify(loginFailure)}`); +assert.equal(loginOrigin, "null"); +assert.equal(loginFetchSite, "same-origin"); +assert.equal(loginResponse.status(), 303); +await page.waitForURL(origin + "/app/"); +await page.waitForLoadState("load"); +assert.equal(await page.locator("h1").count(), 1); +console.error("stage=authenticated"); + +const overviewURL = page.url(); +const exploreLink = page.getByRole("link", {name: "Explore", exact: true}); +assert.match(await exploreLink.getAttribute("href"), /^\/app\/explore\/\?organization=/); +await exploreLink.click(); +await page.getByRole("heading", {name: "Follow the evidence."}).waitFor(); +assert.equal(await page.locator('nav[aria-label="Primary"] a[aria-current="page"]').textContent(), "Explore"); +assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= document.documentElement.clientWidth + 1), true); +await page.locator("#explore-query").fill("logs | window 1h | limit 10"); +const [queryResponse] = await Promise.all([ + page.waitForResponse(response => new URL(response.url()).pathname === "/app/explore/" && response.request().method() === "POST"), + page.getByRole("button", {name: "Run query"}).click() +]); +assert.equal(queryResponse.status(), 200); +assert.equal(new URL(page.url()).searchParams.has("query"), false); +await page.getByRole("heading", {name: "Query results"}).waitFor(); +assert.equal(await page.locator(".query-stats dt").allTextContents().then(values => values.join("|")), "Scanned rows|Matched rows|Scanned bytes|Execution"); +assert.ok(await page.locator(".query-results table tbody tr").count() >= 1); +await page.goto(overviewURL); +await page.locator("main h1").waitFor(); +assert.equal(new URL(page.url()).pathname, "/app/"); +console.error("stage=explore"); + +const inboxLink = page.getByRole("link", {name: "Open incident inbox"}); +const inboxPath = await inboxLink.getAttribute("href"); +assert.match(inboxPath, /^\/app\/incidents\/\?organization=/); +let eventRequests = 0; +page.on("request", request => { + if (new URL(request.url()).pathname === "/app/events") eventRequests++; +}); +await inboxLink.click(); +await page.waitForURL(origin + inboxPath); +await page.getByRole("heading", {name: "What needs attention?"}).waitFor(); +await page.locator('.incident-card[data-state="firing"] h3', {hasText: "Browser fixture incident"}).waitFor(); +await page.waitForFunction(() => globalThis.__observatoryBadge === 1); +assert.equal(await page.evaluate(() => globalThis.__observatoryBadge), 1); +evidence.badge = true; +await page.waitForFunction(() => !document.querySelector("[data-cache-inbox]").disabled); +assert.equal(await page.evaluate(() => caches.has("observatory-private-v1")), false); +const cacheButton = page.locator("[data-cache-inbox]"); +const cacheTarget = await cacheButton.getAttribute("data-offline-target"); +await cacheButton.click(); +await page.getByText("This incident inbox is available offline on this browser.", {exact: true}).waitFor(); +const pushButton = page.locator("[data-push-toggle]"); +await page.waitForFunction(() => !document.querySelector("[data-push-toggle]").disabled); +assert.equal(await page.evaluate(() => globalThis.__observatoryPermissionRequests), 0); +await pushButton.click(); +await page.getByText("The browser push setting could not be changed.", {exact: true}).waitFor(); +assert.equal(await page.evaluate(() => globalThis.__observatoryPermissionRequests), 1); +const cached = await page.evaluate(async target => { + const cache = await caches.open("observatory-private-v1"); + const response = await cache.match(target); + return {keys: (await cache.keys()).map(request => request.url), text: response ? await response.text() : ""}; +}, cacheTarget); +assert.deepEqual(cached.keys, [origin + cacheTarget]); +assert.match(cached.text, /Browser fixture incident/); +for (const forbidden of ["csrf_token", "logs | window", "browser-project", "browser-service", "incident_"]) { + assert.equal(cached.text.includes(forbidden), false, `private snapshot contained ${forbidden}`); +} + +intentionallyOffline = true; +await context.setOffline(true); +await page.goto(origin + cacheTarget, {waitUntil: "domcontentloaded"}); +await page.getByRole("heading", {name: "Saved incident inbox"}).waitFor(); +await page.goto(origin + "/network-unavailable/", {waitUntil: "domcontentloaded"}); +await page.getByRole("heading", {name: "The evidence is still safe."}).waitFor(); +await context.setOffline(false); +intentionallyOffline = false; +evidence.offlineShell = true; +evidence.privateInbox = true; +console.error("stage=offline"); + +await page.goto(origin + inboxPath); +await page.waitForFunction(() => !document.querySelector(".live-status").hidden, null, {timeout: 5000}); +const requestsBeforeDisconnect = eventRequests; +await page.evaluate(() => { document.querySelector(".live-status").hidden = true; }); +const reconnectDeadline = Date.now() + 10000; +while (eventRequests <= requestsBeforeDisconnect && Date.now() < reconnectDeadline) { + await page.waitForTimeout(250); +} +assert.ok(eventRequests > requestsBeforeDisconnect, `SSE requests did not reconnect: ${eventRequests} <= ${requestsBeforeDisconnect}`); +await page.waitForFunction(() => !document.querySelector(".live-status").hidden, null, {timeout: 5000}); +evidence.sseReconnected = true; +console.error("stage=sse"); + +const workers = context.serviceWorkers(); +assert.equal(workers.length, 1); +const worker = workers[0]; +const notification = await worker.evaluate(async () => { + let shown; + const original = self.registration.showNotification; + self.registration.showNotification = async (title, options) => { shown = {title, options}; }; + let completion; + const event = new Event("push"); + Object.defineProperty(event, "data", {value: "telemetry that must be ignored"}); + Object.defineProperty(event, "waitUntil", {value: value => { completion = Promise.resolve(value); }}); + self.dispatchEvent(event); + await completion; + self.registration.showNotification = original; + return shown; +}); +assert.equal(notification.title, "Gamertan Observatory needs your attention."); +assert.equal(notification.options.tag, "observatory-attention"); +assert.equal(notification.options.renotify, true); +assert.deepEqual(notification.options.data, {url: "/app/"}); +assert.equal(notification.options.icon, notification.options.badge); +assert.match(notification.options.icon, /^\/assets\/observatory-[a-f0-9]{16}\.svg$/); +evidence.genericNotification = true; +const activation = await worker.evaluate(async () => { + let navigated = ""; + let focused = false; + const originalMatchAll = self.clients.matchAll; + self.clients.matchAll = async () => [{ + url: self.location.origin + "/offline/", + navigate: async target => { navigated = target; }, + focus: async () => { focused = true; } + }]; + let completion; + const event = new Event("notificationclick"); + Object.defineProperty(event, "notification", {value: {close() {}}}); + Object.defineProperty(event, "waitUntil", {value: value => { completion = Promise.resolve(value); }}); + self.dispatchEvent(event); + await completion; + self.clients.matchAll = originalMatchAll; + return {navigated, focused}; +}); +assert.deepEqual(activation, {navigated: "/app/", focused: true}); +evidence.notificationActivated = true; +console.error("stage=notification"); + +await page.goto(origin + "/app/"); +await page.getByRole("button", {name: "Sign out"}).click(); +await page.waitForURL(origin + "/login/"); +assert.equal(await page.evaluate(() => caches.has("observatory-private-v1")), false); +evidence.privateCacheCleared = true; + +assert.deepEqual(pageErrors, []); +assert.deepEqual(consoleErrors, []); +assert.deepEqual(externalRequests, []); +evidence.externalRequests = externalRequests.length; +console.log(JSON.stringify(evidence)); +clearTimeout(campaignTimeout); +await browser.close(); diff --git a/test/browser/package-lock.json b/test/browser/package-lock.json new file mode 100644 index 0000000..019c3f3 --- /dev/null +++ b/test/browser/package-lock.json @@ -0,0 +1,79 @@ +{ + "name": "@gamertan/observatory-browser-campaign", + "version": "0.0.0-private", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@gamertan/observatory-browser-campaign", + "version": "0.0.0-private", + "license": "AGPL-3.0-only", + "devDependencies": { + "@playwright/test": "1.62.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/test/browser/package.json b/test/browser/package.json new file mode 100644 index 0000000..eb0a1b7 --- /dev/null +++ b/test/browser/package.json @@ -0,0 +1,13 @@ +{ + "name": "@gamertan/observatory-browser-campaign", + "version": "0.0.0-private", + "private": true, + "license": "AGPL-3.0-only", + "type": "module", + "scripts": { + "test": "node campaign.mjs" + }, + "devDependencies": { + "@playwright/test": "1.62.1" + } +}