gamertan
92a66db3df
Export the reviewed allowlisted snapshot from private source commit 05928cebd01b586cf9e9d4b8c8537a7605a6068c. This records the exact candidate, bounded capacity result, stateful migration scratch requirement, authenticated batch identity proof, and immediate live acceptance evidence. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
307 lines
12 KiB
Go
307 lines
12 KiB
Go
// SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
package agent
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gamertan.com/observatory/internal/agentstate"
|
|
"gamertan.com/observatory/internal/config"
|
|
"gamertan.com/observatory/internal/hostmetrics"
|
|
"gamertan.com/observatory/internal/model"
|
|
"gamertan.com/observatory/internal/query"
|
|
"gamertan.com/observatory/internal/spool"
|
|
"gamertan.com/observatory/internal/storage"
|
|
)
|
|
|
|
type fakeSender struct {
|
|
fail bool
|
|
failAlerts bool
|
|
batchDigests map[uint64]string
|
|
sent []uint64
|
|
alerts []model.AlertTransition
|
|
}
|
|
|
|
func (sender *fakeSender) SendAlertTransition(_ context.Context, transition model.AlertTransition) (storage.SourceAlertTransitionAck, error) {
|
|
if sender.failAlerts {
|
|
return storage.SourceAlertTransitionAck{}, errors.New("alert endpoint unavailable")
|
|
}
|
|
sender.alerts = append(sender.alerts, transition)
|
|
digest, err := transition.Digest()
|
|
if err != nil {
|
|
return storage.SourceAlertTransitionAck{}, err
|
|
}
|
|
return storage.SourceAlertTransitionAck{SourceID: "source", RuleID: transition.RuleID, RuleRevision: transition.RuleRevision, AgentEpoch: transition.AgentEpoch, Sequence: transition.Sequence, Digest: digest}, nil
|
|
}
|
|
|
|
func (sender *fakeSender) Send(_ context.Context, batch model.Batch) (storage.Ack, error) {
|
|
if sender.fail {
|
|
return storage.Ack{}, errors.New("server unavailable")
|
|
}
|
|
sender.sent = append(sender.sent, batch.Sequence)
|
|
batchDigest, err := batch.Digest()
|
|
if err != nil {
|
|
return storage.Ack{}, err
|
|
}
|
|
if override := sender.batchDigests[batch.Sequence]; override != "" {
|
|
batchDigest = override
|
|
}
|
|
return storage.Ack{SourceID: batch.SourceID, StreamID: batch.StreamID, Sequence: batch.Sequence, Digest: strings.Repeat("d", 64), BatchDigest: batchDigest}, nil
|
|
}
|
|
|
|
func TestRunnerCollectsWhileOfflineAndRecoversCheckpoint(t *testing.T) {
|
|
root := t.TempDir()
|
|
logPath := filepath.Join(root, "request.jsonl")
|
|
if err := os.WriteFile(logPath, []byte(requestLine("one")+"\n"), 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spoolRoot := filepath.Join(root, "spool")
|
|
configuration := config.Agent{BatchRecords: 1, MaxSpoolBytes: 1 << 20, MaxSpoolAge: 72 * time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}}
|
|
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sender := &fakeSender{fail: true}
|
|
runner, err := New(configuration, "source", stateStore, state, queue, sender)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
|
|
if err = runner.RunOnce(context.Background(), now); err == nil {
|
|
t.Fatal("offline delivery unexpectedly succeeded")
|
|
}
|
|
file, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = file.WriteString(requestLine("two") + "\n"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
file.Close()
|
|
if err = runner.RunOnce(context.Background(), now.Add(time.Second)); err == nil {
|
|
t.Fatal("offline delivery unexpectedly succeeded")
|
|
}
|
|
entries, err := queue.List(now.Add(time.Second))
|
|
if err != nil || len(entries) != 2 || entries[0].Sequence != 1 || entries[1].Sequence != 2 {
|
|
t.Fatalf("entries=%+v err=%v", entries, err)
|
|
}
|
|
// Simulate a crash after both durable spool commits but before either cursor
|
|
// checkpoint reached the state file. Recovery must advance state from the
|
|
// spool envelopes before it reads the source again.
|
|
empty := agentstate.State{Version: agentstate.Version, Streams: map[string]agentstate.Cursor{}}
|
|
if err = stateStore.Save(empty); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, recoveredState, err := agentstate.Open(configuration.StateFile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sender.fail = false
|
|
restarted, err := New(configuration, "source", stateStore, recoveredState, queue, sender)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = restarted.RunOnce(context.Background(), now.Add(2*time.Second)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(sender.sent) != 2 || sender.sent[0] != 1 || sender.sent[1] != 2 {
|
|
t.Fatalf("sent=%v", sender.sent)
|
|
}
|
|
if entries, err = queue.List(now.Add(2 * time.Second)); err != nil || len(entries) != 0 {
|
|
t.Fatalf("remaining=%+v err=%v", entries, err)
|
|
}
|
|
_, finalState, err := agentstate.Open(configuration.StateFile)
|
|
if err != nil || finalState.Streams["request"].Sequence != 2 {
|
|
t.Fatalf("state=%+v err=%v", finalState, err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerPreservesSpoolOnMismatchedAcknowledgement(t *testing.T) {
|
|
root := t.TempDir()
|
|
logPath := filepath.Join(root, "request.jsonl")
|
|
if err := os.WriteFile(logPath, []byte(requestLine("one")+"\n"), 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spoolRoot := filepath.Join(root, "spool")
|
|
configuration := config.Agent{BatchRecords: 1, MaxSpoolBytes: 1 << 20, MaxSpoolAge: 72 * time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}}
|
|
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sender := &fakeSender{batchDigests: map[uint64]string{1: strings.Repeat("a", 64)}}
|
|
runner, err := New(configuration, "source", stateStore, state, queue, sender)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
|
|
if err = runner.RunOnce(context.Background(), now); err == nil || !strings.Contains(err.Error(), "acknowledgement does not match exact batch") {
|
|
t.Fatalf("expected exact acknowledgement rejection, got %v", err)
|
|
}
|
|
entries, err := queue.List(now)
|
|
if err != nil || len(entries) != 1 {
|
|
t.Fatalf("entries=%+v err=%v", entries, err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerDeliversLocallyEvaluatedAlertAfterRawBatchAcknowledgement(t *testing.T) {
|
|
root := t.TempDir()
|
|
logPath := filepath.Join(root, "request.jsonl")
|
|
if err := os.WriteFile(logPath, []byte(requestLineWithStatus("failed", 503)+"\n"), 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spoolRoot := filepath.Join(root, "spool")
|
|
ast, err := query.Parse("logs | where status >= 500 | limit 10", model.MaxRecords)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configuration := config.Agent{BatchRecords: 10, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}, AlertRules: []config.AgentAlertRule{{Version: 1, ID: "http-failures", Revision: 1, StreamID: "request", Query: "logs | where status >= 500 | limit 10", MinimumMatches: 1, AST: ast}}}
|
|
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sender := &fakeSender{}
|
|
runner, err := New(configuration, "source", stateStore, state, queue, sender)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
|
|
if err = runner.RunOnce(context.Background(), now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(sender.sent) != 1 || len(sender.alerts) != 1 || sender.alerts[0].State != "matched" || sender.alerts[0].Sequence != 1 || sender.alerts[0].SegmentDigest != strings.Repeat("d", 64) || len(sender.alerts[0].AgentEpoch) != 32 {
|
|
t.Fatalf("sent=%v alerts=%+v", sender.sent, sender.alerts)
|
|
}
|
|
if entries, listErr := queue.List(now); listErr != nil || len(entries) != 0 {
|
|
t.Fatalf("entries=%+v err=%v", entries, listErr)
|
|
}
|
|
}
|
|
|
|
func TestRunnerRetainsRawBatchUntilAlertTransitionIsAcknowledged(t *testing.T) {
|
|
root := t.TempDir()
|
|
logPath := filepath.Join(root, "request.jsonl")
|
|
if err := os.WriteFile(logPath, []byte(requestLineWithStatus("failed", 503)+"\n"), 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spoolRoot := filepath.Join(root, "spool")
|
|
ast, err := query.Parse("logs | where status >= 500 | limit 10", model.MaxRecords)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configuration := config.Agent{BatchRecords: 10, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}, AlertRules: []config.AgentAlertRule{{Version: 1, ID: "http-failures", Revision: 1, StreamID: "request", Query: "logs | where status >= 500 | limit 10", MinimumMatches: 1, AST: ast}}}
|
|
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sender := &fakeSender{failAlerts: true}
|
|
runner, err := New(configuration, "source", stateStore, state, queue, sender)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
|
|
if err = runner.RunOnce(context.Background(), now); err == nil || !strings.Contains(err.Error(), "alert endpoint unavailable") {
|
|
t.Fatalf("err=%v", err)
|
|
}
|
|
if entries, listErr := queue.List(now); listErr != nil || len(entries) != 1 {
|
|
t.Fatalf("entries=%+v err=%v", entries, listErr)
|
|
}
|
|
sender.failAlerts = false
|
|
if err = runner.RunOnce(context.Background(), now.Add(time.Second)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(sender.sent) != 2 || len(sender.alerts) != 1 {
|
|
t.Fatalf("sent=%v alerts=%+v", sender.sent, sender.alerts)
|
|
}
|
|
}
|
|
|
|
func TestSourceIDParsingKeepsDotsInsideIdentifier(t *testing.T) {
|
|
id, err := sourceIDFromCredential("obs1.node.example.abcdef")
|
|
if err != nil || id != "node.example" {
|
|
t.Fatalf("id=%q err=%v", id, err)
|
|
}
|
|
for _, invalid := range []string{"token", "obs1..secret", "obs1.bad value.secret"} {
|
|
if _, err = sourceIDFromCredential(invalid); err == nil {
|
|
t.Fatalf("accepted %q", invalid)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunnerSpoolsLinuxMetricsWhileServerIsOffline(t *testing.T) {
|
|
root := t.TempDir()
|
|
proc := filepath.Join(root, "proc")
|
|
if err := os.MkdirAll(filepath.Join(proc, "net"), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
files := map[string]string{
|
|
"stat": "cpu 1 2 3 4\n", "uptime": "10 5\n",
|
|
"meminfo": "MemTotal: 10 kB\nMemAvailable: 8 kB\nSwapTotal: 2 kB\nSwapFree: 1 kB\n",
|
|
"loadavg": "0.1 0.2 0.3 1/1 1\n", filepath.Join("net", "dev"): "Inter-| Receive | Transmit\n",
|
|
}
|
|
for name, body := range files {
|
|
if err := os.WriteFile(filepath.Join(proc, name), []byte(body), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
spoolRoot := filepath.Join(root, "spool")
|
|
configuration := config.Agent{BatchRecords: 3, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "linux_metrics", StreamID: "host-metrics", LinuxMetrics: &hostmetrics.Config{ProcRoot: proc}}}}
|
|
stateStore, state, err := agentstate.Open(configuration.StateFile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner, err := New(configuration, "source", stateStore, state, queue, &fakeSender{fail: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Date(2026, 8, 17, 6, 0, 0, 0, time.UTC)
|
|
if err = runner.RunOnce(context.Background(), now); err == nil {
|
|
t.Fatal("offline delivery unexpectedly succeeded")
|
|
}
|
|
entries, err := queue.List(now)
|
|
if err != nil || len(entries) < 2 {
|
|
t.Fatalf("entries=%+v err=%v", entries, err)
|
|
}
|
|
for index, entry := range entries {
|
|
batch, _, readErr := queue.ReadWithCheckpoint(entry)
|
|
if readErr != nil {
|
|
t.Fatal(readErr)
|
|
}
|
|
if batch.Signal != model.SignalMetrics || batch.StreamID != "host-metrics" || len(batch.Records) == 0 || len(batch.Records) > configuration.BatchRecords || batch.Sequence != uint64(index+1) {
|
|
t.Fatalf("batch=%+v", batch)
|
|
}
|
|
}
|
|
}
|
|
|
|
func requestLine(route string) string {
|
|
return `{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/` + route + `","status":200,"bytes":12,"duration_ns":1000,"request_id":"request-1"}`
|
|
}
|
|
|
|
func requestLineWithStatus(route string, status int) string {
|
|
return fmt.Sprintf(`{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/%s","status":%d,"bytes":12,"duration_ns":1000,"request_id":"request-1"}`, route, status)
|
|
}
|