docs: publish Preview 19 dogfood evidence

Export the reviewed allowlisted snapshot from private source commit 05928cebd01b586cf9e9d4b8c8537a7605a6068c. This records the exact candidate, bounded capacity result, stateful migration scratch requirement, authenticated batch identity proof, and immediate live acceptance evidence. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
2026-08-18 21:47:08 -04:00
commit 92a66db3df
201 changed files with 38227 additions and 0 deletions
@@ -0,0 +1,303 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package agent
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"gamertan.com/observatory/internal/agentclient"
+	"gamertan.com/observatory/internal/agentstate"
+	"gamertan.com/observatory/internal/config"
+	"gamertan.com/observatory/internal/edgealert"
+	"gamertan.com/observatory/internal/hostmetrics"
+	"gamertan.com/observatory/internal/model"
+	"gamertan.com/observatory/internal/spool"
+	"gamertan.com/observatory/internal/storage"
+	"gamertan.com/observatory/internal/tailer"
+)
+
+type Sender interface {
+	Send(context.Context, model.Batch) (storage.Ack, error)
+	SendAlertTransition(context.Context, model.AlertTransition) (storage.SourceAlertTransitionAck, error)
+}
+
+type Runner struct {
+	configuration config.Agent
+	sourceID      string
+	stateStore    *agentstate.Store
+	state         agentstate.State
+	spool         *spool.Spool
+	sender        Sender
+	agentEpoch    string
+}
+
+func Open(configuration config.Agent, credential string, transport http.RoundTripper) (*Runner, error) {
+	sourceID, err := sourceIDFromCredential(credential)
+	if err != nil {
+		return nil, err
+	}
+	stateStore, state, err := agentstate.Open(configuration.StateFile)
+	if err != nil {
+		return nil, err
+	}
+	queue, err := spool.Open(configuration.SpoolDir, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
+	if err != nil {
+		return nil, err
+	}
+	client, err := agentclient.New(configuration.ServerURL, credential, transport)
+	if err != nil {
+		return nil, err
+	}
+	return newRunner(configuration, sourceID, stateStore, state, queue, client, agentEpoch(credential))
+}
+
+func New(configuration config.Agent, sourceID string, stateStore *agentstate.Store, state agentstate.State, queue *spool.Spool, sender Sender) (*Runner, error) {
+	return newRunner(configuration, sourceID, stateStore, state, queue, sender, agentEpoch(sourceID))
+}
+
+func newRunner(configuration config.Agent, sourceID string, stateStore *agentstate.Store, state agentstate.State, queue *spool.Spool, sender Sender, epoch string) (*Runner, error) {
+	if err := model.ValidateSourceID(sourceID); err != nil || stateStore == nil || queue == nil || sender == nil {
+		return nil, errors.New("agent runtime dependencies are invalid")
+	}
+	if err := state.Validate(); err != nil {
+		return nil, err
+	}
+	if len(epoch) != 32 {
+		return nil, errors.New("agent epoch is invalid")
+	}
+	return &Runner{configuration: configuration, sourceID: sourceID, stateStore: stateStore, state: state, spool: queue, sender: sender, agentEpoch: epoch}, nil
+}
+
+func (runner *Runner) RunOnce(ctx context.Context, now time.Time) error {
+	if now.IsZero() {
+		return errors.New("agent cycle time is required")
+	}
+	var cycleErrors []error
+	if err := runner.recoverCheckpoints(now); err != nil {
+		return err
+	}
+	deliveryFailed := false
+	if err := runner.deliver(ctx, now); err != nil {
+		cycleErrors = append(cycleErrors, err)
+		deliveryFailed = true
+	}
+	for _, source := range runner.configuration.Sources {
+		cursor := runner.state.Streams[source.StreamID]
+		if source.Kind == "linux_metrics" {
+			observations, collectErr := hostmetrics.Collect(*source.LinuxMetrics, now)
+			if len(observations) == 0 {
+				if collectErr != nil {
+					cycleErrors = append(cycleErrors, fmt.Errorf("collect stream %s: %w", source.StreamID, collectErr))
+				}
+				continue
+			}
+			for start := 0; start < len(observations); start += runner.configuration.BatchRecords {
+				end := min(start+runner.configuration.BatchRecords, len(observations))
+				cursor.Sequence++
+				spooled, err := runner.spoolObservations(source.StreamID, cursor, model.SignalMetrics, observations[start:end], now)
+				if err != nil {
+					if spooled {
+						return err
+					}
+					cycleErrors = append(cycleErrors, fmt.Errorf("spool stream %s: %w", source.StreamID, err))
+					break
+				}
+			}
+			if collectErr != nil {
+				cycleErrors = append(cycleErrors, fmt.Errorf("collect stream %s: %w", source.StreamID, collectErr))
+			}
+			continue
+		}
+		result, err := tailer.Read(source, cursor, runner.configuration.BatchRecords, now)
+		if err != nil {
+			cycleErrors = append(cycleErrors, fmt.Errorf("collect stream %s: %w", source.StreamID, err))
+			continue
+		}
+		if len(result.Observations) == 0 {
+			if result.Cursor != cursor {
+				runner.state.Streams[source.StreamID] = result.Cursor
+				if err = runner.stateStore.Save(runner.state); err != nil {
+					return err
+				}
+			}
+			continue
+		}
+		result.Cursor.Sequence = cursor.Sequence + 1
+		spooled, err := runner.spoolObservations(source.StreamID, result.Cursor, result.Signal, result.Observations, now)
+		if err != nil {
+			if spooled {
+				return err
+			}
+			cycleErrors = append(cycleErrors, fmt.Errorf("spool stream %s: %w", source.StreamID, err))
+			continue
+		}
+	}
+	if !deliveryFailed {
+		if err := runner.deliver(ctx, now); err != nil {
+			cycleErrors = append(cycleErrors, err)
+		}
+	}
+	return errors.Join(cycleErrors...)
+}
+
+func (runner *Runner) spoolObservations(streamID string, cursor agentstate.Cursor, signal model.Signal, observations []model.Observation, now time.Time) (bool, error) {
+	batch := model.Batch{Version: model.BatchVersion, SourceID: runner.sourceID, StreamID: streamID, Sequence: cursor.Sequence, ObservedAt: now.UTC(), Signal: signal, Records: observations}
+	checkpoint, err := json.Marshal(cursor)
+	if err != nil {
+		return false, err
+	}
+	if _, err = runner.spool.PutWithCheckpoint(batch, checkpoint, now); err != nil {
+		return false, err
+	}
+	runner.state.Streams[streamID] = cursor
+	return true, runner.stateStore.Save(runner.state)
+}
+
+func (runner *Runner) recoverCheckpoints(now time.Time) error {
+	entries, err := runner.spool.List(now)
+	if err != nil {
+		return err
+	}
+	changed := false
+	for _, entry := range entries {
+		_, checkpoint, err := runner.spool.ReadWithCheckpoint(entry)
+		if err != nil {
+			return err
+		}
+		if len(checkpoint) == 0 {
+			return errors.New("pending spool batch lacks a cursor checkpoint")
+		}
+		cursor, err := decodeCursor(checkpoint)
+		if err != nil || cursor.Sequence != entry.Sequence {
+			return errors.New("pending spool checkpoint is invalid")
+		}
+		current := runner.state.Streams[entry.StreamID]
+		if cursor.Sequence < current.Sequence {
+			continue
+		}
+		if cursor.Sequence == current.Sequence {
+			if cursor != current {
+				return errors.New("pending spool checkpoint conflicts with agent state")
+			}
+			continue
+		}
+		if cursor.Sequence != current.Sequence+1 {
+			return errors.New("pending spool checkpoint has a sequence gap")
+		}
+		runner.state.Streams[entry.StreamID] = cursor
+		changed = true
+	}
+	if changed {
+		return runner.stateStore.Save(runner.state)
+	}
+	return nil
+}
+
+func (runner *Runner) deliver(ctx context.Context, now time.Time) error {
+	entries, err := runner.spool.List(now)
+	if err != nil {
+		return err
+	}
+	blocked := map[string]bool{}
+	var deliveryErrors []error
+	for _, entry := range entries {
+		if blocked[entry.StreamID] {
+			continue
+		}
+		batch, _, err := runner.spool.ReadWithCheckpoint(entry)
+		if err != nil {
+			return err
+		}
+		ack, sendErr := runner.sender.Send(ctx, batch)
+		if sendErr != nil {
+			blocked[entry.StreamID] = true
+			deliveryErrors = append(deliveryErrors, fmt.Errorf("deliver stream %s sequence %d: %w", entry.StreamID, entry.Sequence, sendErr))
+			continue
+		}
+		batchDigest, digestErr := batch.Digest()
+		if digestErr != nil || ack.SourceID != batch.SourceID || ack.StreamID != batch.StreamID || ack.Sequence != batch.Sequence || ack.BatchDigest != batchDigest {
+			blocked[entry.StreamID] = true
+			deliveryErrors = append(deliveryErrors, fmt.Errorf("deliver stream %s sequence %d: acknowledgement does not match exact batch", entry.StreamID, entry.Sequence))
+			continue
+		}
+		transitionFailed := false
+		for _, rule := range runner.configuration.AlertRules {
+			if rule.StreamID != batch.StreamID {
+				continue
+			}
+			evaluation, evaluationErr := edgealert.Evaluate(rule, batch)
+			if evaluationErr != nil {
+				blocked[entry.StreamID] = true
+				deliveryErrors = append(deliveryErrors, fmt.Errorf("evaluate alert rule %s for stream %s sequence %d: %w", rule.ID, entry.StreamID, entry.Sequence, evaluationErr))
+				transitionFailed = true
+				break
+			}
+			transition := model.AlertTransition{Version: model.AlertTransitionVersion, RuleID: rule.ID, RuleRevision: rule.Revision, AgentEpoch: runner.agentEpoch, Sequence: batch.Sequence, StreamID: batch.StreamID, BatchSequence: batch.Sequence, SegmentDigest: ack.Digest, WindowStart: evaluation.WindowStart, WindowEnd: evaluation.WindowEnd, State: evaluation.State, ObservedAt: evaluation.ObservedAt}
+			transitionAck, transitionErr := runner.sender.SendAlertTransition(ctx, transition)
+			expectedDigest, digestErr := transition.Digest()
+			if transitionErr != nil || digestErr != nil || transitionAck.SourceID != batch.SourceID || transitionAck.RuleID != transition.RuleID || transitionAck.RuleRevision != transition.RuleRevision || transitionAck.AgentEpoch != transition.AgentEpoch || transitionAck.Sequence != transition.Sequence || transitionAck.Digest != expectedDigest {
+				blocked[entry.StreamID] = true
+				if transitionErr == nil {
+					transitionErr = errors.New("acknowledgement does not match exact transition")
+				}
+				deliveryErrors = append(deliveryErrors, fmt.Errorf("deliver alert rule %s for stream %s sequence %d: %w", rule.ID, entry.StreamID, entry.Sequence, transitionErr))
+				transitionFailed = true
+				break
+			}
+		}
+		if transitionFailed {
+			continue
+		}
+		if err = runner.spool.Acknowledge(entry, entry.Digest); err != nil {
+			return err
+		}
+	}
+	return errors.Join(deliveryErrors...)
+}
+
+func agentEpoch(sourceID string) string {
+	digest := sha256.Sum256([]byte("observatory-agent-epoch-v1\x00" + sourceID))
+	return hex.EncodeToString(digest[:16])
+}
+
+func decodeCursor(body []byte) (agentstate.Cursor, error) {
+	decoder := json.NewDecoder(bytes.NewReader(body))
+	decoder.DisallowUnknownFields()
+	var cursor agentstate.Cursor
+	if err := decoder.Decode(&cursor); err != nil {
+		return agentstate.Cursor{}, err
+	}
+	if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
+		return agentstate.Cursor{}, errors.New("cursor checkpoint has trailing data")
+	}
+	if cursor.Offset < 0 || (cursor.Device == 0) != (cursor.Inode == 0) || cursor.Sequence == 0 {
+		return agentstate.Cursor{}, errors.New("cursor checkpoint is invalid")
+	}
+	return cursor, nil
+}
+
+func sourceIDFromCredential(credential string) (string, error) {
+	if !strings.HasPrefix(credential, "obs1.") || strings.ContainsAny(credential, " \t\r\n") {
+		return "", errors.New("source credential is invalid")
+	}
+	remainder := strings.TrimPrefix(credential, "obs1.")
+	separator := strings.LastIndexByte(remainder, '.')
+	if separator < 1 || separator == len(remainder)-1 {
+		return "", errors.New("source credential is invalid")
+	}
+	sourceID := remainder[:separator]
+	if err := model.ValidateSourceID(sourceID); err != nil {
+		return "", errors.New("source credential is invalid")
+	}
+	return sourceID, nil
+}
@@ -0,0 +1,306 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package agent
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gamertan.com/observatory/internal/agentstate"
+	"gamertan.com/observatory/internal/config"
+	"gamertan.com/observatory/internal/hostmetrics"
+	"gamertan.com/observatory/internal/model"
+	"gamertan.com/observatory/internal/query"
+	"gamertan.com/observatory/internal/spool"
+	"gamertan.com/observatory/internal/storage"
+)
+
+type fakeSender struct {
+	fail         bool
+	failAlerts   bool
+	batchDigests map[uint64]string
+	sent         []uint64
+	alerts       []model.AlertTransition
+}
+
+func (sender *fakeSender) SendAlertTransition(_ context.Context, transition model.AlertTransition) (storage.SourceAlertTransitionAck, error) {
+	if sender.failAlerts {
+		return storage.SourceAlertTransitionAck{}, errors.New("alert endpoint unavailable")
+	}
+	sender.alerts = append(sender.alerts, transition)
+	digest, err := transition.Digest()
+	if err != nil {
+		return storage.SourceAlertTransitionAck{}, err
+	}
+	return storage.SourceAlertTransitionAck{SourceID: "source", RuleID: transition.RuleID, RuleRevision: transition.RuleRevision, AgentEpoch: transition.AgentEpoch, Sequence: transition.Sequence, Digest: digest}, nil
+}
+
+func (sender *fakeSender) Send(_ context.Context, batch model.Batch) (storage.Ack, error) {
+	if sender.fail {
+		return storage.Ack{}, errors.New("server unavailable")
+	}
+	sender.sent = append(sender.sent, batch.Sequence)
+	batchDigest, err := batch.Digest()
+	if err != nil {
+		return storage.Ack{}, err
+	}
+	if override := sender.batchDigests[batch.Sequence]; override != "" {
+		batchDigest = override
+	}
+	return storage.Ack{SourceID: batch.SourceID, StreamID: batch.StreamID, Sequence: batch.Sequence, Digest: strings.Repeat("d", 64), BatchDigest: batchDigest}, nil
+}
+
+func TestRunnerCollectsWhileOfflineAndRecoversCheckpoint(t *testing.T) {
+	root := t.TempDir()
+	logPath := filepath.Join(root, "request.jsonl")
+	if err := os.WriteFile(logPath, []byte(requestLine("one")+"\n"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	spoolRoot := filepath.Join(root, "spool")
+	configuration := config.Agent{BatchRecords: 1, MaxSpoolBytes: 1 << 20, MaxSpoolAge: 72 * time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}}
+	stateStore, state, err := agentstate.Open(configuration.StateFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sender := &fakeSender{fail: true}
+	runner, err := New(configuration, "source", stateStore, state, queue, sender)
+	if err != nil {
+		t.Fatal(err)
+	}
+	now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
+	if err = runner.RunOnce(context.Background(), now); err == nil {
+		t.Fatal("offline delivery unexpectedly succeeded")
+	}
+	file, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err = file.WriteString(requestLine("two") + "\n"); err != nil {
+		t.Fatal(err)
+	}
+	file.Close()
+	if err = runner.RunOnce(context.Background(), now.Add(time.Second)); err == nil {
+		t.Fatal("offline delivery unexpectedly succeeded")
+	}
+	entries, err := queue.List(now.Add(time.Second))
+	if err != nil || len(entries) != 2 || entries[0].Sequence != 1 || entries[1].Sequence != 2 {
+		t.Fatalf("entries=%+v err=%v", entries, err)
+	}
+	// Simulate a crash after both durable spool commits but before either cursor
+	// checkpoint reached the state file. Recovery must advance state from the
+	// spool envelopes before it reads the source again.
+	empty := agentstate.State{Version: agentstate.Version, Streams: map[string]agentstate.Cursor{}}
+	if err = stateStore.Save(empty); err != nil {
+		t.Fatal(err)
+	}
+	_, recoveredState, err := agentstate.Open(configuration.StateFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sender.fail = false
+	restarted, err := New(configuration, "source", stateStore, recoveredState, queue, sender)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = restarted.RunOnce(context.Background(), now.Add(2*time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	if len(sender.sent) != 2 || sender.sent[0] != 1 || sender.sent[1] != 2 {
+		t.Fatalf("sent=%v", sender.sent)
+	}
+	if entries, err = queue.List(now.Add(2 * time.Second)); err != nil || len(entries) != 0 {
+		t.Fatalf("remaining=%+v err=%v", entries, err)
+	}
+	_, finalState, err := agentstate.Open(configuration.StateFile)
+	if err != nil || finalState.Streams["request"].Sequence != 2 {
+		t.Fatalf("state=%+v err=%v", finalState, err)
+	}
+}
+
+func TestRunnerPreservesSpoolOnMismatchedAcknowledgement(t *testing.T) {
+	root := t.TempDir()
+	logPath := filepath.Join(root, "request.jsonl")
+	if err := os.WriteFile(logPath, []byte(requestLine("one")+"\n"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	spoolRoot := filepath.Join(root, "spool")
+	configuration := config.Agent{BatchRecords: 1, MaxSpoolBytes: 1 << 20, MaxSpoolAge: 72 * time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}}
+	stateStore, state, err := agentstate.Open(configuration.StateFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sender := &fakeSender{batchDigests: map[uint64]string{1: strings.Repeat("a", 64)}}
+	runner, err := New(configuration, "source", stateStore, state, queue, sender)
+	if err != nil {
+		t.Fatal(err)
+	}
+	now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
+	if err = runner.RunOnce(context.Background(), now); err == nil || !strings.Contains(err.Error(), "acknowledgement does not match exact batch") {
+		t.Fatalf("expected exact acknowledgement rejection, got %v", err)
+	}
+	entries, err := queue.List(now)
+	if err != nil || len(entries) != 1 {
+		t.Fatalf("entries=%+v err=%v", entries, err)
+	}
+}
+
+func TestRunnerDeliversLocallyEvaluatedAlertAfterRawBatchAcknowledgement(t *testing.T) {
+	root := t.TempDir()
+	logPath := filepath.Join(root, "request.jsonl")
+	if err := os.WriteFile(logPath, []byte(requestLineWithStatus("failed", 503)+"\n"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	spoolRoot := filepath.Join(root, "spool")
+	ast, err := query.Parse("logs | where status >= 500 | limit 10", model.MaxRecords)
+	if err != nil {
+		t.Fatal(err)
+	}
+	configuration := config.Agent{BatchRecords: 10, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}, AlertRules: []config.AgentAlertRule{{Version: 1, ID: "http-failures", Revision: 1, StreamID: "request", Query: "logs | where status >= 500 | limit 10", MinimumMatches: 1, AST: ast}}}
+	stateStore, state, err := agentstate.Open(configuration.StateFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sender := &fakeSender{}
+	runner, err := New(configuration, "source", stateStore, state, queue, sender)
+	if err != nil {
+		t.Fatal(err)
+	}
+	now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
+	if err = runner.RunOnce(context.Background(), now); err != nil {
+		t.Fatal(err)
+	}
+	if len(sender.sent) != 1 || len(sender.alerts) != 1 || sender.alerts[0].State != "matched" || sender.alerts[0].Sequence != 1 || sender.alerts[0].SegmentDigest != strings.Repeat("d", 64) || len(sender.alerts[0].AgentEpoch) != 32 {
+		t.Fatalf("sent=%v alerts=%+v", sender.sent, sender.alerts)
+	}
+	if entries, listErr := queue.List(now); listErr != nil || len(entries) != 0 {
+		t.Fatalf("entries=%+v err=%v", entries, listErr)
+	}
+}
+
+func TestRunnerRetainsRawBatchUntilAlertTransitionIsAcknowledged(t *testing.T) {
+	root := t.TempDir()
+	logPath := filepath.Join(root, "request.jsonl")
+	if err := os.WriteFile(logPath, []byte(requestLineWithStatus("failed", 503)+"\n"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	spoolRoot := filepath.Join(root, "spool")
+	ast, err := query.Parse("logs | where status >= 500 | limit 10", model.MaxRecords)
+	if err != nil {
+		t.Fatal(err)
+	}
+	configuration := config.Agent{BatchRecords: 10, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "requestlog_jsonl", Path: logPath, StreamID: "request"}}, AlertRules: []config.AgentAlertRule{{Version: 1, ID: "http-failures", Revision: 1, StreamID: "request", Query: "logs | where status >= 500 | limit 10", MinimumMatches: 1, AST: ast}}}
+	stateStore, state, err := agentstate.Open(configuration.StateFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sender := &fakeSender{failAlerts: true}
+	runner, err := New(configuration, "source", stateStore, state, queue, sender)
+	if err != nil {
+		t.Fatal(err)
+	}
+	now := time.Date(2026, 8, 17, 1, 2, 4, 0, time.UTC)
+	if err = runner.RunOnce(context.Background(), now); err == nil || !strings.Contains(err.Error(), "alert endpoint unavailable") {
+		t.Fatalf("err=%v", err)
+	}
+	if entries, listErr := queue.List(now); listErr != nil || len(entries) != 1 {
+		t.Fatalf("entries=%+v err=%v", entries, listErr)
+	}
+	sender.failAlerts = false
+	if err = runner.RunOnce(context.Background(), now.Add(time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	if len(sender.sent) != 2 || len(sender.alerts) != 1 {
+		t.Fatalf("sent=%v alerts=%+v", sender.sent, sender.alerts)
+	}
+}
+
+func TestSourceIDParsingKeepsDotsInsideIdentifier(t *testing.T) {
+	id, err := sourceIDFromCredential("obs1.node.example.abcdef")
+	if err != nil || id != "node.example" {
+		t.Fatalf("id=%q err=%v", id, err)
+	}
+	for _, invalid := range []string{"token", "obs1..secret", "obs1.bad value.secret"} {
+		if _, err = sourceIDFromCredential(invalid); err == nil {
+			t.Fatalf("accepted %q", invalid)
+		}
+	}
+}
+
+func TestRunnerSpoolsLinuxMetricsWhileServerIsOffline(t *testing.T) {
+	root := t.TempDir()
+	proc := filepath.Join(root, "proc")
+	if err := os.MkdirAll(filepath.Join(proc, "net"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	files := map[string]string{
+		"stat": "cpu 1 2 3 4\n", "uptime": "10 5\n",
+		"meminfo": "MemTotal: 10 kB\nMemAvailable: 8 kB\nSwapTotal: 2 kB\nSwapFree: 1 kB\n",
+		"loadavg": "0.1 0.2 0.3 1/1 1\n", filepath.Join("net", "dev"): "Inter-| Receive | Transmit\n",
+	}
+	for name, body := range files {
+		if err := os.WriteFile(filepath.Join(proc, name), []byte(body), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	spoolRoot := filepath.Join(root, "spool")
+	configuration := config.Agent{BatchRecords: 3, MaxSpoolBytes: 1 << 20, MaxSpoolAge: time.Hour, SpoolDir: spoolRoot, StateFile: filepath.Join(spoolRoot, "state.json"), Sources: []config.AgentSource{{Kind: "linux_metrics", StreamID: "host-metrics", LinuxMetrics: &hostmetrics.Config{ProcRoot: proc}}}}
+	stateStore, state, err := agentstate.Open(configuration.StateFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	queue, err := spool.Open(spoolRoot, configuration.MaxSpoolBytes, configuration.MaxSpoolAge)
+	if err != nil {
+		t.Fatal(err)
+	}
+	runner, err := New(configuration, "source", stateStore, state, queue, &fakeSender{fail: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	now := time.Date(2026, 8, 17, 6, 0, 0, 0, time.UTC)
+	if err = runner.RunOnce(context.Background(), now); err == nil {
+		t.Fatal("offline delivery unexpectedly succeeded")
+	}
+	entries, err := queue.List(now)
+	if err != nil || len(entries) < 2 {
+		t.Fatalf("entries=%+v err=%v", entries, err)
+	}
+	for index, entry := range entries {
+		batch, _, readErr := queue.ReadWithCheckpoint(entry)
+		if readErr != nil {
+			t.Fatal(readErr)
+		}
+		if batch.Signal != model.SignalMetrics || batch.StreamID != "host-metrics" || len(batch.Records) == 0 || len(batch.Records) > configuration.BatchRecords || batch.Sequence != uint64(index+1) {
+			t.Fatalf("batch=%+v", batch)
+		}
+	}
+}
+
+func requestLine(route string) string {
+	return `{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/` + route + `","status":200,"bytes":12,"duration_ns":1000,"request_id":"request-1"}`
+}
+
+func requestLineWithStatus(route string, status int) string {
+	return fmt.Sprintf(`{"timestamp":"2026-08-17T01:02:03Z","method":"GET","route":"/%s","status":%d,"bytes":12,"duration_ns":1000,"request_id":"request-1"}`, route, status)
+}