docs: publish Preview 19 dogfood evidence
Export the reviewed allowlisted snapshot from private source commit 05928cebd01b586cf9e9d4b8c8537a7605a6068c. This records the exact candidate, bounded capacity result, stateful migration scratch requirement, authenticated batch identity proof, and immediate live acceptance evidence. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
This commit is contained in:
@@ -0,0 +1,306 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/agentstate"
|
||||
"gamertan.com/observatory/internal/config"
|
||||
"gamertan.com/observatory/internal/hostmetrics"
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/spool"
|
||||
"gamertan.com/observatory/internal/storage"
|
||||
)
|
||||
|
||||
type fakeSender struct {
|
||||
fail bool
|
||||
failAlerts bool
|
||||
batchDigests map[uint64]string
|
||||
sent []uint64
|
||||
alerts []model.AlertTransition
|
||||
}
|
||||
|
||||
func (sender *fakeSender) SendAlertTransition(_ context.Context, transition model.AlertTransition) (storage.SourceAlertTransitionAck, error) {
|
||||
if sender.failAlerts {
|
||||
return storage.SourceAlertTransitionAck{}, errors.New("alert endpoint unavailable")
|
||||
}
|
||||
sender.alerts = append(sender.alerts, transition)
|
||||
digest, err := transition.Digest()
|
||||
if err != nil {
|
||||
return storage.SourceAlertTransitionAck{}, err
|
||||
}
|
||||
return storage.SourceAlertTransitionAck{SourceID: "source", RuleID: transition.RuleID, RuleRevision: transition.RuleRevision, AgentEpoch: transition.AgentEpoch, Sequence: transition.Sequence, Digest: digest}, nil
|
||||
}
|
||||
|
||||
func (sender *fakeSender) Send(_ context.Context, batch model.Batch) (storage.Ack, error) {
|
||||
if sender.fail {
|
||||
return storage.Ack{}, errors.New("server unavailable")
|
||||
}
|
||||
sender.sent = append(sender.sent, batch.Sequence)
|
||||
batchDigest, err := batch.Digest()
|
||||
if err != nil {
|
||||
return storage.Ack{}, err
|
||||
}
|
||||
if override := sender.batchDigests[batch.Sequence]; override != "" {
|
||||
batchDigest = override
|
||||
}
|
||||
return storage.Ack{SourceID: batch.SourceID, StreamID: batch.StreamID, Sequence: batch.Sequence, Digest: strings.Repeat("d", 64), BatchDigest: batchDigest}, nil
|
||||
}
|
||||
|
||||
func TestRunnerCollectsWhileOfflineAndRecoversCheckpoint(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
logPath := filepath.Join(root, "request.jsonl")
|
||||
if err := os.WriteFile(logPath, []byte(requestLine("one")+"\n"), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spoolRoot := filepath.Join(root, "spool")
|
||||
configuration := config.Agent{BatchRecords: 1, MaxSpoolBytes: 1 << 20, MaxSpoolAge: 72 * time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}}
|
||||
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sender := &fakeSender{fail: true}
|
||||
runner, err := New(configuration, "source", stateStore, state, queue, sender)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
|
||||
if err = runner.RunOnce(context.Background(), now); err == nil {
|
||||
t.Fatal("offline delivery unexpectedly succeeded")
|
||||
}
|
||||
file, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = file.WriteString(requestLine("two") + "\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file.Close()
|
||||
if err = runner.RunOnce(context.Background(), now.Add(time.Second)); err == nil {
|
||||
t.Fatal("offline delivery unexpectedly succeeded")
|
||||
}
|
||||
entries, err := queue.List(now.Add(time.Second))
|
||||
if err != nil || len(entries) != 2 || entries[0].Sequence != 1 || entries[1].Sequence != 2 {
|
||||
t.Fatalf("entries=%+v err=%v", entries, err)
|
||||
}
|
||||
// Simulate a crash after both durable spool commits but before either cursor
|
||||
// checkpoint reached the state file. Recovery must advance state from the
|
||||
// spool envelopes before it reads the source again.
|
||||
empty := agentstate.State{Version: agentstate.Version, Streams: map[string]agentstate.Cursor{}}
|
||||
if err = stateStore.Save(empty); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, recoveredState, err := agentstate.Open(configuration.StateFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sender.fail = false
|
||||
restarted, err := New(configuration, "source", stateStore, recoveredState, queue, sender)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = restarted.RunOnce(context.Background(), now.Add(2*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sender.sent) != 2 || sender.sent[0] != 1 || sender.sent[1] != 2 {
|
||||
t.Fatalf("sent=%v", sender.sent)
|
||||
}
|
||||
if entries, err = queue.List(now.Add(2 * time.Second)); err != nil || len(entries) != 0 {
|
||||
t.Fatalf("remaining=%+v err=%v", entries, err)
|
||||
}
|
||||
_, finalState, err := agentstate.Open(configuration.StateFile)
|
||||
if err != nil || finalState.Streams["request"].Sequence != 2 {
|
||||
t.Fatalf("state=%+v err=%v", finalState, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPreservesSpoolOnMismatchedAcknowledgement(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
logPath := filepath.Join(root, "request.jsonl")
|
||||
if err := os.WriteFile(logPath, []byte(requestLine("one")+"\n"), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spoolRoot := filepath.Join(root, "spool")
|
||||
configuration := config.Agent{BatchRecords: 1, MaxSpoolBytes: 1 << 20, MaxSpoolAge: 72 * time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}}
|
||||
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sender := &fakeSender{batchDigests: map[uint64]string{1: strings.Repeat("a", 64)}}
|
||||
runner, err := New(configuration, "source", stateStore, state, queue, sender)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
|
||||
if err = runner.RunOnce(context.Background(), now); err == nil || !strings.Contains(err.Error(), "acknowledgement does not match exact batch") {
|
||||
t.Fatalf("expected exact acknowledgement rejection, got %v", err)
|
||||
}
|
||||
entries, err := queue.List(now)
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("entries=%+v err=%v", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerDeliversLocallyEvaluatedAlertAfterRawBatchAcknowledgement(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
logPath := filepath.Join(root, "request.jsonl")
|
||||
if err := os.WriteFile(logPath, []byte(requestLineWithStatus("failed", 503)+"\n"), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spoolRoot := filepath.Join(root, "spool")
|
||||
ast, err := query.Parse("logs | where status >= 500 | limit 10", model.MaxRecords)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configuration := config.Agent{BatchRecords: 10, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}, AlertRules: []config.AgentAlertRule{{Version: 1, ID: "http-failures", Revision: 1, StreamID: "request", Query: "logs | where status >= 500 | limit 10", MinimumMatches: 1, AST: ast}}}
|
||||
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sender := &fakeSender{}
|
||||
runner, err := New(configuration, "source", stateStore, state, queue, sender)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
|
||||
if err = runner.RunOnce(context.Background(), now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sender.sent) != 1 || len(sender.alerts) != 1 || sender.alerts[0].State != "matched" || sender.alerts[0].Sequence != 1 || sender.alerts[0].SegmentDigest != strings.Repeat("d", 64) || len(sender.alerts[0].AgentEpoch) != 32 {
|
||||
t.Fatalf("sent=%v alerts=%+v", sender.sent, sender.alerts)
|
||||
}
|
||||
if entries, listErr := queue.List(now); listErr != nil || len(entries) != 0 {
|
||||
t.Fatalf("entries=%+v err=%v", entries, listErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRetainsRawBatchUntilAlertTransitionIsAcknowledged(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
logPath := filepath.Join(root, "request.jsonl")
|
||||
if err := os.WriteFile(logPath, []byte(requestLineWithStatus("failed", 503)+"\n"), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spoolRoot := filepath.Join(root, "spool")
|
||||
ast, err := query.Parse("logs | where status >= 500 | limit 10", model.MaxRecords)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configuration := config.Agent{BatchRecords: 10, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}, AlertRules: []config.AgentAlertRule{{Version: 1, ID: "http-failures", Revision: 1, StreamID: "request", Query: "logs | where status >= 500 | limit 10", MinimumMatches: 1, AST: ast}}}
|
||||
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sender := &fakeSender{failAlerts: true}
|
||||
runner, err := New(configuration, "source", stateStore, state, queue, sender)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
|
||||
if err = runner.RunOnce(context.Background(), now); err == nil || !strings.Contains(err.Error(), "alert endpoint unavailable") {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if entries, listErr := queue.List(now); listErr != nil || len(entries) != 1 {
|
||||
t.Fatalf("entries=%+v err=%v", entries, listErr)
|
||||
}
|
||||
sender.failAlerts = false
|
||||
if err = runner.RunOnce(context.Background(), now.Add(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sender.sent) != 2 || len(sender.alerts) != 1 {
|
||||
t.Fatalf("sent=%v alerts=%+v", sender.sent, sender.alerts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceIDParsingKeepsDotsInsideIdentifier(t *testing.T) {
|
||||
id, err := sourceIDFromCredential("obs1.node.example.abcdef")
|
||||
if err != nil || id != "node.example" {
|
||||
t.Fatalf("id=%q err=%v", id, err)
|
||||
}
|
||||
for _, invalid := range []string{"token", "obs1..secret", "obs1.bad value.secret"} {
|
||||
if _, err = sourceIDFromCredential(invalid); err == nil {
|
||||
t.Fatalf("accepted %q", invalid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerSpoolsLinuxMetricsWhileServerIsOffline(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
proc := filepath.Join(root, "proc")
|
||||
if err := os.MkdirAll(filepath.Join(proc, "net"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files := map[string]string{
|
||||
"stat": "cpu 1 2 3 4\n", "uptime": "10 5\n",
|
||||
"meminfo": "MemTotal: 10 kB\nMemAvailable: 8 kB\nSwapTotal: 2 kB\nSwapFree: 1 kB\n",
|
||||
"loadavg": "0.1 0.2 0.3 1/1 1\n", filepath.Join("net", "dev"): "Inter-| Receive | Transmit\n",
|
||||
}
|
||||
for name, body := range files {
|
||||
if err := os.WriteFile(filepath.Join(proc, name), []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
spoolRoot := filepath.Join(root, "spool")
|
||||
configuration := config.Agent{BatchRecords: 3, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "linux_metrics", StreamID: "host-metrics", LinuxMetrics: &hostmetrics.Config{ProcRoot: proc}}}}
|
||||
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner, err := New(configuration, "source", stateStore, state, queue, &fakeSender{fail: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 17, 6, 0, 0, 0, time.UTC)
|
||||
if err = runner.RunOnce(context.Background(), now); err == nil {
|
||||
t.Fatal("offline delivery unexpectedly succeeded")
|
||||
}
|
||||
entries, err := queue.List(now)
|
||||
if err != nil || len(entries) < 2 {
|
||||
t.Fatalf("entries=%+v err=%v", entries, err)
|
||||
}
|
||||
for index, entry := range entries {
|
||||
batch, _, readErr := queue.ReadWithCheckpoint(entry)
|
||||
if readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
if batch.Signal != model.SignalMetrics || batch.StreamID != "host-metrics" || len(batch.Records) == 0 || len(batch.Records) > configuration.BatchRecords || batch.Sequence != uint64(index+1) {
|
||||
t.Fatalf("batch=%+v", batch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func requestLine(route string) string {
|
||||
return `{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/` + route + `","status":200,"bytes":12,"duration_ns":1000,"request_id":"request-1"}`
|
||||
}
|
||||
|
||||
func requestLineWithStatus(route string, status int) string {
|
||||
return fmt.Sprintf(`{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/%s","status":%d,"bytes":12,"duration_ns":1000,"request_id":"request-1"}`, route, status)
|
||||
}
|
||||
Reference in New Issue
Block a user