docs: publish Preview 19 dogfood evidence
Export the reviewed allowlisted snapshot from private source commit 05928cebd01b586cf9e9d4b8c8537a7605a6068c. This records the exact candidate, bounded capacity result, stateful migration scratch requirement, authenticated batch identity proof, and immediate live acceptance evidence. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
This commit is contained in:
@@ -0,0 +1,247 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package tailer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/agentstate"
|
||||
"gamertan.com/observatory/internal/collector"
|
||||
"gamertan.com/observatory/internal/config"
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
const maxDirectoryEntries = 4096
|
||||
const maxReadBytesPerCycle = 4 << 20
|
||||
|
||||
type Result struct {
|
||||
Signal model.Signal
|
||||
Observations []model.Observation
|
||||
Cursor agentstate.Cursor
|
||||
}
|
||||
|
||||
type fileIdentity struct{ device, inode uint64 }
|
||||
|
||||
func Read(source config.AgentSource, cursor agentstate.Cursor, maximum int, now time.Time) (Result, error) {
|
||||
if maximum < 1 || maximum > model.MaxRecords {
|
||||
return Result{}, errors.New("tail batch limit is invalid")
|
||||
}
|
||||
configured, configuredIdentity, err := openRegular(source.Path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return Result{Cursor: cursor}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
defer configured.Close()
|
||||
|
||||
active, activeIdentity, rotated := configured, configuredIdentity, false
|
||||
if cursor.Device != 0 && (cursor.Device != configuredIdentity.device || cursor.Inode != configuredIdentity.inode) {
|
||||
previous, previousIdentity, findErr := findIdentity(filepath.Dir(source.Path), fileIdentity{cursor.Device, cursor.Inode})
|
||||
if findErr != nil {
|
||||
return Result{}, findErr
|
||||
}
|
||||
if previous != nil {
|
||||
defer previous.Close()
|
||||
active, activeIdentity, rotated = previous, previousIdentity, true
|
||||
} else {
|
||||
cursor.Discontinuities++
|
||||
cursor.Device, cursor.Inode, cursor.Offset, cursor.DiscardingLine = configuredIdentity.device, configuredIdentity.inode, 0, false
|
||||
}
|
||||
}
|
||||
if cursor.Device == 0 {
|
||||
cursor.Device, cursor.Inode = activeIdentity.device, activeIdentity.inode
|
||||
}
|
||||
|
||||
result, atEOF, partial, err := readOpen(active, source, cursor, maximum, now)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if !rotated || len(result.Observations) == maximum || !atEOF {
|
||||
return result, nil
|
||||
}
|
||||
if result.Cursor.DiscardingLine {
|
||||
result.Cursor.DroppedRecords++
|
||||
result.Cursor.DiscardingLine = false
|
||||
} else if partial > 0 {
|
||||
result.Cursor.Offset += int64(partial)
|
||||
result.Cursor.DroppedRecords++
|
||||
}
|
||||
result.Cursor.Device, result.Cursor.Inode, result.Cursor.Offset, result.Cursor.DiscardingLine = configuredIdentity.device, configuredIdentity.inode, 0, false
|
||||
remaining := maximum - len(result.Observations)
|
||||
next, _, _, err := readOpen(configured, source, result.Cursor, remaining, now)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if result.Signal == "" {
|
||||
result.Signal = next.Signal
|
||||
}
|
||||
result.Observations = append(result.Observations, next.Observations...)
|
||||
result.Cursor = next.Cursor
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func readOpen(file *os.File, source config.AgentSource, cursor agentstate.Cursor, maximum int, now time.Time) (Result, bool, int, error) {
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return Result{}, false, 0, err
|
||||
}
|
||||
identity, err := identity(info)
|
||||
if err != nil {
|
||||
return Result{}, false, 0, err
|
||||
}
|
||||
if cursor.Device != identity.device || cursor.Inode != identity.inode {
|
||||
cursor.Device, cursor.Inode, cursor.Offset, cursor.DiscardingLine = identity.device, identity.inode, 0, false
|
||||
}
|
||||
if info.Size() < cursor.Offset {
|
||||
cursor.Offset = 0
|
||||
cursor.DiscardingLine = false
|
||||
cursor.Discontinuities++
|
||||
}
|
||||
if _, err = file.Seek(cursor.Offset, io.SeekStart); err != nil {
|
||||
return Result{}, false, 0, err
|
||||
}
|
||||
reader := bufio.NewReaderSize(file, 64<<10)
|
||||
result := Result{Cursor: cursor}
|
||||
discarding := cursor.DiscardingLine
|
||||
partial := 0
|
||||
readBytes := 0
|
||||
pendingBytes := 0
|
||||
lineBuffer := make([]byte, 0, 64<<10)
|
||||
for len(result.Observations) < maximum {
|
||||
fragment, readErr := reader.ReadSlice('\n')
|
||||
readBytes += len(fragment)
|
||||
switch {
|
||||
case readErr == nil:
|
||||
if discarding {
|
||||
result.Cursor.Offset += int64(len(fragment))
|
||||
result.Cursor.DroppedRecords++
|
||||
discarding = false
|
||||
result.Cursor.DiscardingLine = false
|
||||
} else {
|
||||
pendingBytes += len(fragment)
|
||||
if pendingBytes > collector.MaxLineBytes+1 {
|
||||
result.Cursor.Offset += int64(pendingBytes)
|
||||
result.Cursor.DroppedRecords++
|
||||
} else {
|
||||
lineBuffer = append(lineBuffer, fragment...)
|
||||
result.Cursor.Offset += int64(pendingBytes)
|
||||
line := lineBuffer[:len(lineBuffer)-1]
|
||||
if len(line) > 0 && line[len(line)-1] == '\r' {
|
||||
line = line[:len(line)-1]
|
||||
}
|
||||
signal, observation, parseErr := collector.Parse(source.Kind, line, now, source.SensitiveFields...)
|
||||
if parseErr != nil {
|
||||
result.Cursor.DroppedRecords++
|
||||
} else {
|
||||
if result.Signal != "" && signal != result.Signal {
|
||||
return Result{}, false, 0, errors.New("collector changed signal inside one stream")
|
||||
}
|
||||
result.Signal = signal
|
||||
result.Observations = append(result.Observations, observation)
|
||||
}
|
||||
}
|
||||
pendingBytes = 0
|
||||
lineBuffer = lineBuffer[:0]
|
||||
}
|
||||
case errors.Is(readErr, bufio.ErrBufferFull):
|
||||
if discarding {
|
||||
result.Cursor.Offset += int64(len(fragment))
|
||||
} else {
|
||||
pendingBytes += len(fragment)
|
||||
if pendingBytes > collector.MaxLineBytes {
|
||||
result.Cursor.Offset += int64(pendingBytes)
|
||||
pendingBytes = 0
|
||||
lineBuffer = lineBuffer[:0]
|
||||
discarding = true
|
||||
result.Cursor.DiscardingLine = true
|
||||
} else {
|
||||
lineBuffer = append(lineBuffer, fragment...)
|
||||
}
|
||||
}
|
||||
case errors.Is(readErr, io.EOF):
|
||||
if discarding {
|
||||
result.Cursor.Offset += int64(len(fragment))
|
||||
result.Cursor.DiscardingLine = true
|
||||
partial = 0
|
||||
} else if pendingBytes+len(fragment) > collector.MaxLineBytes {
|
||||
result.Cursor.Offset += int64(pendingBytes + len(fragment))
|
||||
result.Cursor.DiscardingLine = true
|
||||
partial = 0
|
||||
} else {
|
||||
partial = pendingBytes + len(fragment)
|
||||
}
|
||||
return result, true, partial, nil
|
||||
default:
|
||||
return Result{}, false, 0, readErr
|
||||
}
|
||||
if readBytes >= maxReadBytesPerCycle {
|
||||
return result, false, 0, nil
|
||||
}
|
||||
}
|
||||
return result, false, 0, nil
|
||||
}
|
||||
|
||||
func openRegular(path string) (*os.File, fileIdentity, error) {
|
||||
file, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
||||
if err != nil {
|
||||
return nil, fileIdentity{}, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, fileIdentity{}, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
file.Close()
|
||||
return nil, fileIdentity{}, errors.New("collector source must be a regular file")
|
||||
}
|
||||
value, err := identity(info)
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, fileIdentity{}, err
|
||||
}
|
||||
return file, value, nil
|
||||
}
|
||||
|
||||
func findIdentity(directory string, wanted fileIdentity) (*os.File, fileIdentity, error) {
|
||||
entries, err := os.ReadDir(directory)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fileIdentity{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fileIdentity{}, err
|
||||
}
|
||||
if len(entries) > maxDirectoryEntries {
|
||||
return nil, fileIdentity{}, errors.New("collector directory exceeds safe entry limit")
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.Type()&os.ModeSymlink != 0 || entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
file, found, openErr := openRegular(filepath.Join(directory, entry.Name()))
|
||||
if openErr != nil {
|
||||
continue
|
||||
}
|
||||
if found == wanted {
|
||||
return file, found, nil
|
||||
}
|
||||
file.Close()
|
||||
}
|
||||
return nil, fileIdentity{}, nil
|
||||
}
|
||||
|
||||
func identity(info os.FileInfo) (fileIdentity, error) {
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok || stat.Dev == 0 || stat.Ino == 0 {
|
||||
return fileIdentity{}, fmt.Errorf("collector file identity is unavailable")
|
||||
}
|
||||
return fileIdentity{device: uint64(stat.Dev), inode: uint64(stat.Ino)}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user