docs: publish Preview 19 dogfood evidence
Export the reviewed allowlisted snapshot from private source commit 05928cebd01b586cf9e9d4b8c8537a7605a6068c. This records the exact candidate, bounded capacity result, stateful migration scratch requirement, authenticated batch identity proof, and immediate live acceptance evidence. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
This commit is contained in:
@@ -0,0 +1,445 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
const maxProjectionVersion = 999999
|
||||
const indexedTimeFormat = "2006-01-02T15:04:05.000000000Z"
|
||||
|
||||
type DescriptorActivation struct {
|
||||
OrganizationID string `json:"organization_id"`
|
||||
Signal model.Signal `json:"signal"`
|
||||
Field string `json:"field"`
|
||||
Previous int `json:"previous_version"`
|
||||
Active int `json:"active_version"`
|
||||
IndexedRows int64 `json:"indexed_rows"`
|
||||
Descriptor schema.Descriptor `json:"descriptor"`
|
||||
}
|
||||
|
||||
func (s *Store) namedLock(name string) *sync.Mutex {
|
||||
value, _ := s.locks.LoadOrStore(name, &sync.Mutex{})
|
||||
return value.(*sync.Mutex)
|
||||
}
|
||||
|
||||
func ensureProjectionMetadata(ctx context.Context, db sqlExecutor) error {
|
||||
statements := []string{
|
||||
`CREATE TABLE IF NOT EXISTS projection_versions (
|
||||
version INTEGER PRIMARY KEY CHECK(version >= 1 AND version <= 999999),
|
||||
created_at TEXT NOT NULL,
|
||||
activated_at TEXT NOT NULL
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS projection_state (
|
||||
id INTEGER PRIMARY KEY CHECK(id=1),
|
||||
active_version INTEGER NOT NULL REFERENCES projection_versions(version)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS projection_descriptors (
|
||||
version INTEGER NOT NULL REFERENCES projection_versions(version),
|
||||
signal TEXT NOT NULL,
|
||||
field TEXT NOT NULL,
|
||||
descriptor_json TEXT NOT NULL,
|
||||
PRIMARY KEY(version,signal,field)
|
||||
)`,
|
||||
`INSERT OR IGNORE INTO projection_versions(version,created_at,activated_at) VALUES(1,'1970-01-01T00:00:00Z','1970-01-01T00:00:00Z')`,
|
||||
`INSERT OR IGNORE INTO projection_state(id,active_version) VALUES(1,1)`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := db.ExecContext(ctx, statement); err != nil {
|
||||
return fmt.Errorf("migrate projection metadata: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type sqlExecutor interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
func activeProjection(ctx context.Context, db interface {
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
QueryContext(context.Context, string, ...any) (*sql.Rows, error)
|
||||
}) (int, query.MapRegistry, []schema.Descriptor, error) {
|
||||
var version int
|
||||
if err := db.QueryRowContext(ctx, `SELECT active_version FROM projection_state WHERE id=1`).Scan(&version); err != nil {
|
||||
return 0, nil, nil, errors.New("read active projection version")
|
||||
}
|
||||
if version < 1 || version > maxProjectionVersion {
|
||||
return 0, nil, nil, errors.New("active projection version is invalid")
|
||||
}
|
||||
rows, err := db.QueryContext(ctx, `SELECT descriptor_json FROM projection_descriptors WHERE version=? ORDER BY signal,field`, version)
|
||||
if err != nil {
|
||||
return 0, nil, nil, errors.New("read active projection descriptors")
|
||||
}
|
||||
defer rows.Close()
|
||||
registry := query.MapRegistry{}
|
||||
var descriptors []schema.Descriptor
|
||||
for rows.Next() {
|
||||
if len(descriptors) >= model.MaxDistinctFields {
|
||||
return 0, nil, nil, errors.New("active projection descriptor limit exceeded")
|
||||
}
|
||||
var encoded string
|
||||
if err = rows.Scan(&encoded); err != nil {
|
||||
return 0, nil, nil, errors.New("read active projection descriptor")
|
||||
}
|
||||
var descriptor schema.Descriptor
|
||||
if err = json.Unmarshal([]byte(encoded), &descriptor); err != nil || descriptor.Validate() != nil || descriptor.ProjectionVersion != version {
|
||||
return 0, nil, nil, errors.New("active projection descriptor is invalid")
|
||||
}
|
||||
key := string(descriptor.Signal) + ":" + query.CanonicalField(descriptor.Field)
|
||||
if _, exists := registry[key]; exists {
|
||||
return 0, nil, nil, errors.New("active projection descriptor is duplicated")
|
||||
}
|
||||
registry[key] = descriptor
|
||||
descriptors = append(descriptors, descriptor)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return 0, nil, nil, errors.New("read active projection descriptors")
|
||||
}
|
||||
return version, registry, descriptors, nil
|
||||
}
|
||||
|
||||
func projectionIndexTable(version int) (string, error) {
|
||||
if version < 2 || version > maxProjectionVersion {
|
||||
return "", errors.New("indexed projection version is invalid")
|
||||
}
|
||||
return fmt.Sprintf("indexed_fields_v%06d", version), nil
|
||||
}
|
||||
|
||||
func (s *Store) ActiveDescriptors(ctx context.Context, organizationID string) (query.MapRegistry, int, error) {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil {
|
||||
return nil, 0, errors.New("invalid organization identifier")
|
||||
}
|
||||
path := filepath.Join(s.root, "organizations", organizationID, "projection.sqlite")
|
||||
info, err := os.Lstat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return query.MapRegistry{}, 1, nil
|
||||
}
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return nil, 0, errors.New("organization projection is unavailable")
|
||||
}
|
||||
dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String()
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, 0, errors.New("open organization projection")
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
var metadataTables int
|
||||
if err = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='projection_state'`).Scan(&metadataTables); err != nil {
|
||||
return nil, 0, errors.New("inspect organization projection metadata")
|
||||
}
|
||||
if metadataTables == 0 {
|
||||
return query.MapRegistry{}, 1, nil
|
||||
}
|
||||
version, registry, _, err := activeProjection(ctx, db)
|
||||
return registry, version, err
|
||||
}
|
||||
|
||||
func (s *Store) ActivateDescriptor(ctx context.Context, organizationID string, reviewed schema.Descriptor, now time.Time) (DescriptorActivation, error) {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil || reviewed.Validate() != nil || reviewed.ProjectionVersion != 1 || now.IsZero() {
|
||||
return DescriptorActivation{}, errors.New("descriptor activation input is invalid")
|
||||
}
|
||||
proposal, err := s.descriptorProposal(ctx, organizationID, reviewed.Signal, reviewed.Field)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, err
|
||||
}
|
||||
if proposal.Status == "rejected" {
|
||||
return DescriptorActivation{}, errors.New("rejected descriptor proposal cannot be activated")
|
||||
}
|
||||
if proposal.Proposal.Descriptor.Signal != reviewed.Signal || proposal.Proposal.Descriptor.Field != reviewed.Field {
|
||||
return DescriptorActivation{}, errors.New("reviewed descriptor does not match proposal")
|
||||
}
|
||||
|
||||
lock := s.namedLock("organization:" + organizationID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
path := filepath.Join(s.root, "organizations", organizationID, "projection.sqlite")
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return DescriptorActivation{}, errors.New("organization projection is unavailable")
|
||||
}
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, errors.New("open organization projection")
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
for _, statement := range []string{`PRAGMA journal_mode=WAL`, `PRAGMA synchronous=FULL`, `PRAGMA busy_timeout=5000`, `PRAGMA foreign_keys=ON`} {
|
||||
if _, err = db.ExecContext(ctx, statement); err != nil {
|
||||
return DescriptorActivation{}, errors.New("configure organization projection")
|
||||
}
|
||||
}
|
||||
if err = ensureProjectionMetadata(ctx, db); err != nil {
|
||||
return DescriptorActivation{}, err
|
||||
}
|
||||
currentVersion, _, current, err := activeProjection(ctx, db)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, err
|
||||
}
|
||||
for _, descriptor := range current {
|
||||
if descriptor.Signal == reviewed.Signal && descriptor.Field == reviewed.Field {
|
||||
if sameDescriptorIgnoringProjection(descriptor, reviewed) {
|
||||
if err = s.markProposalActivated(ctx, organizationID, descriptor); err != nil {
|
||||
return DescriptorActivation{}, err
|
||||
}
|
||||
return DescriptorActivation{OrganizationID: organizationID, Signal: descriptor.Signal, Field: descriptor.Field, Previous: currentVersion, Active: currentVersion, Descriptor: descriptor}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if proposal.Status == "activated" {
|
||||
return DescriptorActivation{}, errors.New("activated descriptor revision requires a new review proposal")
|
||||
}
|
||||
controlTx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, errors.New("begin descriptor proposal claim")
|
||||
}
|
||||
defer controlTx.Rollback()
|
||||
claim, err := controlTx.ExecContext(ctx, `UPDATE descriptor_proposals SET status=status WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, organizationID, reviewed.Signal, reviewed.Field)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, errors.New("claim descriptor proposal")
|
||||
}
|
||||
if changed, _ := claim.RowsAffected(); changed != 1 {
|
||||
return DescriptorActivation{}, errors.New("descriptor proposal is no longer pending")
|
||||
}
|
||||
if currentVersion >= maxProjectionVersion {
|
||||
return DescriptorActivation{}, errors.New("projection version space exhausted")
|
||||
}
|
||||
nextVersion := currentVersion + 1
|
||||
reviewed.ProjectionVersion = nextVersion
|
||||
if err = reviewed.Validate(); err != nil {
|
||||
return DescriptorActivation{}, errors.New("reviewed descriptor is invalid")
|
||||
}
|
||||
byKey := map[string]schema.Descriptor{}
|
||||
for _, descriptor := range current {
|
||||
descriptor.ProjectionVersion = nextVersion
|
||||
byKey[string(descriptor.Signal)+":"+descriptor.Field] = descriptor
|
||||
}
|
||||
byKey[string(reviewed.Signal)+":"+reviewed.Field] = reviewed
|
||||
if len(byKey) > model.MaxDistinctFields {
|
||||
return DescriptorActivation{}, errors.New("active projection descriptor limit exceeded")
|
||||
}
|
||||
next := make([]schema.Descriptor, 0, len(byKey))
|
||||
for _, descriptor := range byKey {
|
||||
next = append(next, descriptor)
|
||||
}
|
||||
sort.Slice(next, func(i, j int) bool {
|
||||
if next[i].Signal == next[j].Signal {
|
||||
return next[i].Field < next[j].Field
|
||||
}
|
||||
return next[i].Signal < next[j].Signal
|
||||
})
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, errors.New("begin descriptor activation")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
timestamp := now.UTC().Format(time.RFC3339Nano)
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO projection_versions(version,created_at,activated_at) VALUES(?,?,?)`, nextVersion, timestamp, timestamp); err != nil {
|
||||
return DescriptorActivation{}, errors.New("create projection version")
|
||||
}
|
||||
for _, descriptor := range next {
|
||||
encoded, marshalErr := json.Marshal(descriptor)
|
||||
if marshalErr != nil {
|
||||
return DescriptorActivation{}, errors.New("encode active descriptor")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO projection_descriptors(version,signal,field,descriptor_json) VALUES(?,?,?,?)`, nextVersion, descriptor.Signal, descriptor.Field, string(encoded)); err != nil {
|
||||
return DescriptorActivation{}, errors.New("store active descriptor")
|
||||
}
|
||||
}
|
||||
indexedRows, err := buildProjectionIndex(ctx, tx, nextVersion, next)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, err
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE projection_state SET active_version=? WHERE id=1 AND active_version=?`, nextVersion, currentVersion)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, errors.New("activate projection version")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return DescriptorActivation{}, errors.New("active projection changed during activation")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return DescriptorActivation{}, errors.New("commit descriptor activation")
|
||||
}
|
||||
encoded, err := json.Marshal(reviewed)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, errors.New("encode activated descriptor")
|
||||
}
|
||||
result, err = controlTx.ExecContext(ctx, `UPDATE descriptor_proposals SET descriptor_json=?,status='activated' WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, string(encoded), organizationID, reviewed.Signal, reviewed.Field)
|
||||
if err != nil {
|
||||
return DescriptorActivation{}, errors.New("acknowledge activated descriptor")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return DescriptorActivation{}, errors.New("descriptor proposal changed during activation")
|
||||
}
|
||||
if err = controlTx.Commit(); err != nil {
|
||||
return DescriptorActivation{}, errors.New("commit activated descriptor acknowledgement")
|
||||
}
|
||||
return DescriptorActivation{OrganizationID: organizationID, Signal: reviewed.Signal, Field: reviewed.Field, Previous: currentVersion, Active: nextVersion, IndexedRows: indexedRows, Descriptor: reviewed}, nil
|
||||
}
|
||||
|
||||
func buildProjectionIndex(ctx context.Context, tx *sql.Tx, version int, descriptors []schema.Descriptor) (int64, error) {
|
||||
table, err := projectionIndexTable(version)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `CREATE TABLE `+table+` (
|
||||
signal TEXT NOT NULL, field TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL, stream_id TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL, record_index INTEGER NOT NULL,
|
||||
timestamp TEXT NOT NULL, value_text TEXT NOT NULL, value_number REAL,
|
||||
PRIMARY KEY(signal,field,source_id,stream_id,sequence,record_index)
|
||||
) WITHOUT ROWID`); err != nil {
|
||||
return 0, errors.New("create projection index")
|
||||
}
|
||||
indexed := map[string]schema.Descriptor{}
|
||||
for _, descriptor := range descriptors {
|
||||
if descriptor.Index != schema.IndexNone {
|
||||
indexed[string(descriptor.Signal)+":"+descriptor.Field] = descriptor
|
||||
}
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `SELECT source_id,stream_id,sequence,record_index,signal,timestamp,attributes_json FROM observations ORDER BY source_id,stream_id,sequence,record_index`)
|
||||
if err != nil {
|
||||
return 0, errors.New("scan projection for index build")
|
||||
}
|
||||
statement, err := tx.PrepareContext(ctx, `INSERT INTO `+table+`(signal,field,source_id,stream_id,sequence,record_index,timestamp,value_text,value_number) VALUES(?,?,?,?,?,?,?,?,?)`)
|
||||
if err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, errors.New("prepare projection index build")
|
||||
}
|
||||
defer statement.Close()
|
||||
var count int64
|
||||
for rows.Next() {
|
||||
var sourceID, streamID, signalText, timestamp, attributesJSON string
|
||||
var sequence uint64
|
||||
var recordIndex int
|
||||
if err = rows.Scan(&sourceID, &streamID, &sequence, &recordIndex, &signalText, ×tamp, &attributesJSON); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, errors.New("read projection index source")
|
||||
}
|
||||
var attributes map[string]string
|
||||
if err = json.Unmarshal([]byte(attributesJSON), &attributes); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, errors.New("decode projection index source")
|
||||
}
|
||||
for field, raw := range attributes {
|
||||
descriptor, ok := indexed[signalText+":"+query.CanonicalField(field)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
text, number, ok := indexValue(raw, descriptor.Type)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, err = statement.ExecContext(ctx, signalText, descriptor.Field, sourceID, streamID, sequence, recordIndex, timestamp, text, number); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, errors.New("build projection index")
|
||||
}
|
||||
if count == math.MaxInt64 {
|
||||
_ = rows.Close()
|
||||
return 0, errors.New("projection index row count overflow")
|
||||
}
|
||||
count++
|
||||
}
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return 0, errors.New("close projection index source")
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return 0, errors.New("scan projection for index build")
|
||||
}
|
||||
for _, suffix := range []struct{ name, columns string }{
|
||||
{"exact", "signal,field,value_text,timestamp"},
|
||||
{"number", "signal,field,value_number,timestamp"},
|
||||
{"record", "source_id,stream_id,sequence,record_index,signal,field"},
|
||||
} {
|
||||
if _, err = tx.ExecContext(ctx, `CREATE INDEX `+table+`_`+suffix.name+` ON `+table+`(`+suffix.columns+`)`); err != nil {
|
||||
return 0, errors.New("index activated projection")
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func indexValue(raw string, valueType schema.Type) (string, any, bool) {
|
||||
switch valueType {
|
||||
case schema.TypeInteger:
|
||||
value, err := strconv.ParseInt(raw, 10, 64)
|
||||
return raw, value, err == nil
|
||||
case schema.TypeFloat, schema.TypeDuration:
|
||||
value, err := strconv.ParseFloat(raw, 64)
|
||||
return raw, value, err == nil && !math.IsNaN(value) && !math.IsInf(value, 0)
|
||||
case schema.TypeBoolean:
|
||||
value, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return "", nil, false
|
||||
}
|
||||
return strconv.FormatBool(value), nil, true
|
||||
case schema.TypeTime:
|
||||
value, err := time.Parse(time.RFC3339Nano, raw)
|
||||
if err != nil {
|
||||
return "", nil, false
|
||||
}
|
||||
return value.UTC().Format(indexedTimeFormat), nil, true
|
||||
case schema.TypeString:
|
||||
return raw, nil, true
|
||||
default:
|
||||
return "", nil, false
|
||||
}
|
||||
}
|
||||
|
||||
func indexProjectedObservation(ctx context.Context, tx *sql.Tx, version int, descriptors []schema.Descriptor, batch model.Batch, recordIndex int, observation model.Observation) error {
|
||||
if version == 1 {
|
||||
return nil
|
||||
}
|
||||
table, err := projectionIndexTable(version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
indexed := map[string]schema.Descriptor{}
|
||||
for _, descriptor := range descriptors {
|
||||
if descriptor.Signal == batch.Signal && descriptor.Index != schema.IndexNone {
|
||||
indexed[descriptor.Field] = descriptor
|
||||
}
|
||||
}
|
||||
for field, raw := range observation.Attributes {
|
||||
descriptor, ok := indexed[query.CanonicalField(field)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
text, number, ok := indexValue(raw, descriptor.Type)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT OR IGNORE INTO `+table+`(signal,field,source_id,stream_id,sequence,record_index,timestamp,value_text,value_number) VALUES(?,?,?,?,?,?,?,?,?)`, batch.Signal, descriptor.Field, batch.SourceID, batch.StreamID, batch.Sequence, recordIndex, observation.Timestamp.UTC().Format(time.RFC3339Nano), text, number)
|
||||
if err != nil {
|
||||
return errors.New("index projected observation")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sameDescriptorIgnoringProjection(left, right schema.Descriptor) bool {
|
||||
left.ProjectionVersion = 1
|
||||
right.ProjectionVersion = 1
|
||||
leftJSON, _ := json.Marshal(left)
|
||||
rightJSON, _ := json.Marshal(right)
|
||||
return string(leftJSON) == string(rightJSON)
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
func TestIndexedTimeValuesUseFixedChronologicalUTCText(t *testing.T) {
|
||||
earlier, _, ok := indexValue("2026-08-17T09:00:00Z", schema.TypeTime)
|
||||
if !ok {
|
||||
t.Fatal("earlier time rejected")
|
||||
}
|
||||
later, _, ok := indexValue("2026-08-17T09:00:00.1Z", schema.TypeTime)
|
||||
if !ok {
|
||||
t.Fatal("later time rejected")
|
||||
}
|
||||
if len(earlier) != len(later) || strings.Compare(earlier, later) >= 0 || earlier != "2026-08-17T09:00:00.000000000Z" {
|
||||
t.Fatalf("earlier=%q later=%q", earlier, later)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptorActivationAndIngestionSerializeAcrossStoreInstances(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
second, err := Open(store.root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer second.Close()
|
||||
now := time.Date(2026, 8, 17, 9, 0, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: now, Name: "queue.depth", Value: floatPointer(1), Attributes: map[string]string{"workshop.queue_depth": "1"}}}}
|
||||
if _, err = store.Ingest(ctx, token, first, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
reviewed := schema.Descriptor{Version: schema.DescriptorVersion, Signal: model.SignalMetrics, Field: "workshop.queue_depth", Type: schema.TypeInteger, Meaning: "Number of work items waiting in the selected service queue.", Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, Index: schema.IndexRange, Retention: schema.RetentionRaw, ProjectionVersion: 1}
|
||||
secondBatch := first
|
||||
secondBatch.Sequence = 2
|
||||
secondBatch.ObservedAt = now.Add(time.Second)
|
||||
secondBatch.Records = []model.Observation{{Timestamp: secondBatch.ObservedAt, Name: "queue.depth", Value: floatPointer(2), Attributes: map[string]string{"workshop.queue_depth": "2"}}}
|
||||
start := make(chan struct{})
|
||||
errorsFound := make(chan error, 2)
|
||||
go func() {
|
||||
<-start
|
||||
_, activateErr := store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(2*time.Second))
|
||||
errorsFound <- activateErr
|
||||
}()
|
||||
go func() {
|
||||
<-start
|
||||
_, ingestErr := second.Ingest(ctx, token, secondBatch, now.Add(2*time.Second))
|
||||
errorsFound <- ingestErr
|
||||
}()
|
||||
close(start)
|
||||
for range 2 {
|
||||
if runErr := <-errorsFound; runErr != nil {
|
||||
t.Fatal(runErr)
|
||||
}
|
||||
}
|
||||
projectAll(t, store)
|
||||
path := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite")
|
||||
db := openTestProjection(t, path)
|
||||
defer db.Close()
|
||||
var indexed int
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM indexed_fields_v000002`).Scan(&indexed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if indexed != 2 {
|
||||
t.Fatalf("indexed=%d", indexed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptorActivationBuildsBesideCurrentAndFeedsQueriesAndIngestion(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 9, 0, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{
|
||||
{Timestamp: now, Name: "queue.depth", Value: floatPointer(10), Attributes: map[string]string{"workshop.queue_depth": "10", "workshop.mode": "fast"}},
|
||||
{Timestamp: now.Add(time.Second), Name: "queue.depth", Value: floatPointer(11), Attributes: map[string]string{"workshop.queue_depth": "not-an-integer", "workshop.mode": "slow"}},
|
||||
}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now.Add(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
ast, err := query.Parse(`metrics | where workshop.queue_depth >= 9 | limit 50`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now.Add(time.Minute)); !errors.Is(err, query.ErrSensitivePermissionRequired) {
|
||||
t.Fatalf("unreviewed query err=%v", err)
|
||||
}
|
||||
reviewed := schema.Descriptor{
|
||||
Version: schema.DescriptorVersion, Signal: model.SignalMetrics, Field: "workshop.queue_depth",
|
||||
Type: schema.TypeInteger, Meaning: "Number of work items waiting in the selected service queue.",
|
||||
Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow,
|
||||
Index: schema.IndexRange, Retention: schema.RetentionRaw, ProjectionVersion: 1,
|
||||
}
|
||||
activation, err := store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(2*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if activation.Previous != 1 || activation.Active != 2 || activation.IndexedRows != 1 || activation.Descriptor.ProjectionVersion != 2 {
|
||||
t.Fatalf("activation=%+v", activation)
|
||||
}
|
||||
registry, version, err := store.ActiveDescriptors(ctx, "organization-a")
|
||||
if err != nil || version != 2 {
|
||||
t.Fatalf("version=%d registry=%+v err=%v", version, registry, err)
|
||||
}
|
||||
descriptor, ok := registry.Lookup(model.SignalMetrics, "workshop.queue_depth")
|
||||
if !ok || descriptor.ProjectionVersion != 2 || descriptor.Sensitivity != schema.SensitivityInternal || descriptor.Index != schema.IndexRange {
|
||||
t.Fatalf("descriptor=%+v ok=%t", descriptor, ok)
|
||||
}
|
||||
result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now.Add(3*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Rows) != 1 || len(result.Explain.Fields) != 1 || !result.Explain.Fields[0].Indexed || result.Explain.Fields[0].Unknown {
|
||||
t.Fatalf("result=%+v", result)
|
||||
}
|
||||
|
||||
batch.Sequence = 2
|
||||
batch.ObservedAt = now.Add(4 * time.Minute)
|
||||
batch.Records = []model.Observation{{Timestamp: batch.ObservedAt, Name: "queue.depth", Value: floatPointer(12), Attributes: map[string]string{"workshop.queue_depth": "12", "workshop.mode": "fast"}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, batch.ObservedAt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
result, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now.Add(5*time.Minute))
|
||||
if err != nil || len(result.Rows) != 2 {
|
||||
t.Fatalf("rows=%d err=%v", len(result.Rows), err)
|
||||
}
|
||||
|
||||
path := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite")
|
||||
db := openTestProjection(t, path)
|
||||
defer db.Close()
|
||||
var active, indexed int
|
||||
if err = db.QueryRow(`SELECT active_version FROM projection_state WHERE id=1`).Scan(&active); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM indexed_fields_v000002`).Scan(&indexed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if active != 2 || indexed != 2 {
|
||||
t.Fatalf("active=%d indexed=%d", active, indexed)
|
||||
}
|
||||
statement, arguments, err := projectionSelection(ast, query.Scope{OrganizationID: "organization-a"}, registry, version, now.Add(5*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planRows, err := db.Query(`EXPLAIN QUERY PLAN `+statement, arguments...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var plan strings.Builder
|
||||
for planRows.Next() {
|
||||
var id, parent, unused int
|
||||
var detail string
|
||||
if err = planRows.Scan(&id, &parent, &unused, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan.WriteString(detail)
|
||||
plan.WriteByte('\n')
|
||||
}
|
||||
if err = planRows.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(plan.String(), "indexed_fields_v000002_number") {
|
||||
t.Fatalf("custom range index absent from query plan:\n%s", plan.String())
|
||||
}
|
||||
|
||||
preActivationDescriptor, err := json.Marshal(reviewed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.control.Exec(`UPDATE descriptor_proposals SET descriptor_json=?,status='pending' WHERE organization_id='organization-a' AND signal='metrics' AND field='workshop.queue_depth'`, string(preActivationDescriptor)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retry, err := store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(6*time.Minute))
|
||||
if err != nil || retry.Active != 2 || retry.Previous != 2 || retry.IndexedRows != 0 {
|
||||
t.Fatalf("retry=%+v err=%v", retry, err)
|
||||
}
|
||||
mode := schema.Descriptor{
|
||||
Version: schema.DescriptorVersion, Signal: model.SignalMetrics, Field: "workshop.mode",
|
||||
Type: schema.TypeString, Meaning: "Reviewed operating mode label for the workshop queue.",
|
||||
Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow,
|
||||
Index: schema.IndexExact, Retention: schema.RetentionRaw, ProjectionVersion: 1,
|
||||
}
|
||||
second, err := store.ActivateDescriptor(ctx, "organization-a", mode, now.Add(7*time.Minute))
|
||||
if err != nil || second.Previous != 2 || second.Active != 3 || second.IndexedRows != 5 {
|
||||
t.Fatalf("second=%+v err=%v", second, err)
|
||||
}
|
||||
var retained, current int
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM indexed_fields_v000002`).Scan(&retained); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM indexed_fields_v000003`).Scan(¤t); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retained != 2 || current != 5 {
|
||||
t.Fatalf("retained=%d current=%d", retained, current)
|
||||
}
|
||||
result, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now.Add(8*time.Minute))
|
||||
if err != nil || len(result.Rows) != 2 {
|
||||
t.Fatalf("version-three rows=%d err=%v", len(result.Rows), err)
|
||||
}
|
||||
proposals, err := store.DescriptorProposals(ctx, "organization-a")
|
||||
if err != nil || len(proposals) != 2 || proposals[0].Status != "activated" || proposals[1].Status != "activated" {
|
||||
t.Fatalf("proposals=%+v err=%v", proposals, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptorActivationFailureKeepsPriorVersion(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 9, 0, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request", Attributes: map[string]string{"workshop.label": "ready"}}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
path := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite")
|
||||
db := openTestProjection(t, path)
|
||||
if _, err = db.Exec(`DROP TABLE observations`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Close()
|
||||
reviewed := schema.Descriptor{Version: schema.DescriptorVersion, Signal: model.SignalLogs, Field: "workshop.label", Type: schema.TypeString, Meaning: "Reviewed workshop state label.", Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, Index: schema.IndexExact, Retention: schema.RetentionRaw, ProjectionVersion: 1}
|
||||
if _, err = store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(time.Minute)); err == nil {
|
||||
t.Fatal("activation unexpectedly succeeded without the source projection")
|
||||
}
|
||||
db = openTestProjection(t, path)
|
||||
defer db.Close()
|
||||
var active, versions int
|
||||
if err = db.QueryRow(`SELECT active_version FROM projection_state WHERE id=1`).Scan(&active); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM projection_versions`).Scan(&versions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if active != 1 || versions != 1 {
|
||||
t.Fatalf("active=%d versions=%d", active, versions)
|
||||
}
|
||||
proposal, err := store.descriptorProposal(ctx, "organization-a", model.SignalLogs, "workshop.label")
|
||||
if err != nil || proposal.Status != "pending" {
|
||||
t.Fatalf("proposal=%+v err=%v", proposal, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptorProposalRejectionIsIdempotentAndBlocksActivation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 9, 0, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request", Attributes: map[string]string{"workshop.label": "ready"}}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
if err = store.RejectDescriptorProposal(ctx, "organization-a", model.SignalLogs, "workshop.label"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.RejectDescriptorProposal(ctx, "organization-a", model.SignalLogs, "workshop.label"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reviewed := schema.Descriptor{Version: schema.DescriptorVersion, Signal: model.SignalLogs, Field: "workshop.label", Type: schema.TypeString, Meaning: "Reviewed workshop state label.", Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, Index: schema.IndexExact, Retention: schema.RetentionRaw, ProjectionVersion: 1}
|
||||
if _, err = store.ActivateDescriptor(ctx, "organization-a", reviewed, now.Add(time.Minute)); err == nil {
|
||||
t.Fatal("rejected proposal was activated")
|
||||
}
|
||||
}
|
||||
|
||||
func openTestProjection(t *testing.T, path string) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const baseIndexVersion = 1
|
||||
|
||||
var presenceIndexStatements = []string{
|
||||
`CREATE INDEX IF NOT EXISTS observations_value ON observations(signal,value,timestamp) WHERE value IS NOT NULL`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_http_route ON observations(signal,json_extract(attributes_json,'$."http.route"'),timestamp) WHERE json_extract(attributes_json,'$."http.route"') IS NOT NULL`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_http_status ON observations(signal,CAST(json_extract(attributes_json,'$."http.status_code"') AS INTEGER),timestamp) WHERE CAST(json_extract(attributes_json,'$."http.status_code"') AS INTEGER) IS NOT NULL`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_duration ON observations(signal,CAST(json_extract(attributes_json,'$."duration_ns"') AS REAL),timestamp) WHERE CAST(json_extract(attributes_json,'$."duration_ns"') AS REAL) IS NOT NULL`,
|
||||
}
|
||||
|
||||
// ensureBaseIndexes migrates fields that are absent from most signal types to
|
||||
// presence-only indexes. The DDL and migration marker share one SQLite
|
||||
// transaction: an interrupted migration retains the complete previous index
|
||||
// set and retries before the projection is served.
|
||||
func ensureBaseIndexes(ctx context.Context, db *sql.DB) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin base index migration: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
columns, err := sqliteColumns(tx, "storage_projection_state")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !columns["base_index_version"] {
|
||||
if _, err = tx.ExecContext(ctx, `ALTER TABLE storage_projection_state ADD COLUMN base_index_version INTEGER NOT NULL DEFAULT 0 CHECK(base_index_version BETWEEN 0 AND 1)`); err != nil {
|
||||
return errors.New("add base index migration state")
|
||||
}
|
||||
}
|
||||
var version int
|
||||
if err = tx.QueryRowContext(ctx, `SELECT base_index_version FROM storage_projection_state WHERE id=1`).Scan(&version); err != nil {
|
||||
return errors.New("read base index migration state")
|
||||
}
|
||||
if version < 0 || version > baseIndexVersion {
|
||||
return errors.New("unsupported base index version")
|
||||
}
|
||||
if version == 0 {
|
||||
for _, name := range []string{"observations_value", "observations_http_route", "observations_http_status", "observations_duration"} {
|
||||
if _, err = tx.ExecContext(ctx, `DROP INDEX IF EXISTS `+name); err != nil {
|
||||
return errors.New("remove superseded base index")
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, statement := range presenceIndexStatements {
|
||||
if _, err = tx.ExecContext(ctx, statement); err != nil {
|
||||
return errors.New("create presence-only base index")
|
||||
}
|
||||
}
|
||||
if version == 0 {
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE storage_projection_state SET base_index_version=? WHERE id=1 AND base_index_version=0`, baseIndexVersion); err != nil {
|
||||
return errors.New("activate base index migration")
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return errors.New("commit base index migration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
func TestBaseIndexMigrationReplacesLegacyFullIndexes(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "data")
|
||||
dir := filepath.Join(root, "organizations", "legacy")
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(dir, "projection.sqlite")
|
||||
legacy, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
statements := []string{
|
||||
`CREATE TABLE observations (organization_id TEXT NOT NULL,project_id TEXT NOT NULL,environment_id TEXT NOT NULL,service_id TEXT NOT NULL,source_id TEXT NOT NULL,stream_id TEXT NOT NULL,sequence INTEGER NOT NULL,record_index INTEGER NOT NULL,signal TEXT NOT NULL,timestamp TEXT NOT NULL,name TEXT NOT NULL,severity TEXT,body TEXT,value REAL,trace_id TEXT,span_id TEXT,correlation_id TEXT,attributes_json TEXT NOT NULL,segment_digest TEXT NOT NULL,PRIMARY KEY(source_id,stream_id,sequence,record_index))`,
|
||||
`CREATE TABLE storage_projection_state (id INTEGER PRIMARY KEY CHECK(id=1),metric_rollup_version INTEGER NOT NULL CHECK(metric_rollup_version BETWEEN 0 AND 1))`,
|
||||
`INSERT INTO storage_projection_state(id,metric_rollup_version) VALUES(1,1)`,
|
||||
`CREATE INDEX observations_value ON observations(signal,value,timestamp)`,
|
||||
`CREATE INDEX observations_http_route ON observations(signal,json_extract(attributes_json,'$."http.route"'),timestamp)`,
|
||||
`CREATE INDEX observations_http_status ON observations(signal,CAST(json_extract(attributes_json,'$."http.status_code"') AS INTEGER),timestamp)`,
|
||||
`CREATE INDEX observations_duration ON observations(signal,CAST(json_extract(attributes_json,'$."duration_ns"') AS REAL),timestamp)`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err = legacy.Exec(statement); err != nil {
|
||||
legacy.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err = legacy.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db, err := openProjection(t.Context(), path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
var version int
|
||||
if err = db.QueryRow(`SELECT base_index_version FROM storage_projection_state WHERE id=1`).Scan(&version); err != nil || version != baseIndexVersion {
|
||||
t.Fatalf("base index version=%d err=%v", version, err)
|
||||
}
|
||||
for _, name := range []string{"observations_value", "observations_http_route", "observations_http_status", "observations_duration"} {
|
||||
var definition string
|
||||
if err = db.QueryRow(`SELECT sql FROM sqlite_schema WHERE type='index' AND name=?`, name).Scan(&definition); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(strings.ToUpper(definition), " WHERE ") || !strings.Contains(strings.ToUpper(definition), " IS NOT NULL") {
|
||||
t.Fatalf("index %s was not migrated: %s", name, definition)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPresenceIndexesSupportSelectiveQueries(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Now().UTC()
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
logToken, err := store.CreateSource(ctx, "source-logs", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logs := model.Batch{Version: model.BatchVersion, SourceID: "source-logs", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{
|
||||
{Timestamp: now, Name: "http.server.request", Severity: "error", Attributes: map[string]string{"http.route": "/items", "http.status_code": "503", "duration_ns": "500"}},
|
||||
{Timestamp: now.Add(-time.Second), Name: "application.event", Severity: "information", Attributes: map[string]string{}},
|
||||
}}
|
||||
if _, err = store.Ingest(ctx, logToken, logs, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metricToken, err := store.CreateSource(ctx, "source-metrics", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value := 42.0
|
||||
metrics := model.Batch{Version: model.BatchVersion, SourceID: "source-metrics", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: now, Name: "system.cpu.utilization", Value: &value}}}
|
||||
if _, err = store.Ingest(ctx, metricToken, metrics, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
queries := []string{
|
||||
`logs | where route == "/items" | summarize count() by status | limit 10`,
|
||||
`logs | where status >= 500 | summarize p95(duration) by route | limit 10`,
|
||||
`logs | where duration >= 100 | summarize count() by route | limit 10`,
|
||||
`metrics | where value >= 1 | summarize count() by name | limit 10`,
|
||||
}
|
||||
for _, text := range queries {
|
||||
ast, parseErr := query.Parse(text, 10)
|
||||
if parseErr != nil {
|
||||
t.Fatalf("parse %q: %v", text, parseErr)
|
||||
}
|
||||
result, queryErr := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now)
|
||||
if queryErr != nil || len(result.Rows) == 0 {
|
||||
t.Fatalf("query %q rows=%d err=%v", text, len(result.Rows), queryErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
func migrateControlBatchEnvelopes(db *sql.DB) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return errors.New("begin batch envelope migration")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, statement := range []string{
|
||||
`ALTER TABLE streams ADD COLUMN last_batch_digest TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE streams ADD COLUMN last_wire_digest TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE streams ADD COLUMN last_signal TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE streams ADD COLUMN last_record_count INTEGER NOT NULL DEFAULT 0 CHECK(last_record_count BETWEEN 0 AND 5000)`,
|
||||
`ALTER TABLE streams ADD COLUMN last_encoded_bytes INTEGER NOT NULL DEFAULT 0 CHECK(last_encoded_bytes >= 0)`,
|
||||
`ALTER TABLE streams ADD COLUMN last_first_observed_at TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE streams ADD COLUMN last_last_observed_at TEXT NOT NULL DEFAULT ''`,
|
||||
`UPDATE schema_version SET version=11 WHERE version=10`,
|
||||
} {
|
||||
if _, err = tx.Exec(statement); err != nil {
|
||||
return fmt.Errorf("migrate batch envelope metadata: %w", err)
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return errors.New("commit batch envelope migration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type streamWatermark struct {
|
||||
sequence uint64
|
||||
segmentDigest string
|
||||
envelope model.BatchEnvelope
|
||||
found bool
|
||||
framed bool
|
||||
}
|
||||
|
||||
func (s *Store) streamWatermark(ctx context.Context, sourceID, streamID string) (streamWatermark, error) {
|
||||
var watermark streamWatermark
|
||||
var signal, first, last string
|
||||
err := s.control.QueryRowContext(ctx, `SELECT last_sequence,last_digest,last_batch_digest,last_wire_digest,last_signal,last_record_count,last_encoded_bytes,last_first_observed_at,last_last_observed_at FROM streams WHERE source_id=? AND stream_id=?`, sourceID, streamID).Scan(
|
||||
&watermark.sequence, &watermark.segmentDigest, &watermark.envelope.BatchDigest,
|
||||
&watermark.envelope.WireDigest, &signal, &watermark.envelope.RecordCount,
|
||||
&watermark.envelope.EncodedBytes, &first, &last,
|
||||
)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return streamWatermark{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return streamWatermark{}, fmt.Errorf("read framed stream watermark: %w", err)
|
||||
}
|
||||
watermark.found = true
|
||||
if watermark.envelope.BatchDigest == "" && watermark.envelope.WireDigest == "" {
|
||||
return watermark, nil
|
||||
}
|
||||
firstObserved, firstErr := time.Parse(time.RFC3339Nano, first)
|
||||
lastObserved, lastErr := time.Parse(time.RFC3339Nano, last)
|
||||
if firstErr != nil || lastErr != nil {
|
||||
return streamWatermark{}, errors.New("stored batch envelope time range is invalid")
|
||||
}
|
||||
watermark.envelope.Version = model.BatchEnvelopeVersion
|
||||
watermark.envelope.StreamID = streamID
|
||||
watermark.envelope.Sequence = watermark.sequence
|
||||
watermark.envelope.Signal = model.Signal(signal)
|
||||
watermark.envelope.FirstObservedAt = firstObserved.UTC()
|
||||
watermark.envelope.LastObservedAt = lastObserved.UTC()
|
||||
if err = watermark.envelope.Validate(math.MaxInt64); err != nil {
|
||||
return streamWatermark{}, errors.New("stored batch envelope is invalid")
|
||||
}
|
||||
watermark.framed = true
|
||||
return watermark, nil
|
||||
}
|
||||
|
||||
func (s *Store) checkEnvelope(ctx context.Context, source Source, envelope model.BatchEnvelope) (Ack, bool, error) {
|
||||
watermark, err := s.streamWatermark(ctx, source.ID, envelope.StreamID)
|
||||
if err != nil {
|
||||
return Ack{}, false, err
|
||||
}
|
||||
if !watermark.found {
|
||||
if envelope.Sequence != 1 {
|
||||
return Ack{}, false, errors.New("sequence gap")
|
||||
}
|
||||
return Ack{}, false, nil
|
||||
}
|
||||
if envelope.Sequence < watermark.sequence {
|
||||
return Ack{}, false, errors.New("sequence replay is older than acknowledged watermark")
|
||||
}
|
||||
if watermark.sequence != ^uint64(0) && envelope.Sequence > watermark.sequence+1 {
|
||||
return Ack{}, false, errors.New("sequence gap")
|
||||
}
|
||||
if envelope.Sequence == watermark.sequence {
|
||||
if !watermark.framed {
|
||||
return Ack{}, false, nil
|
||||
}
|
||||
if envelope != watermark.envelope {
|
||||
return Ack{}, false, errors.New("acknowledged sequence reused with different envelope")
|
||||
}
|
||||
return Ack{SourceID: source.ID, StreamID: envelope.StreamID, Sequence: envelope.Sequence, Digest: watermark.segmentDigest, BatchDigest: envelope.BatchDigest, Duplicate: true}, true, nil
|
||||
}
|
||||
return Ack{}, false, nil
|
||||
}
|
||||
|
||||
// CheckNativeReplay performs a cheap, read-only envelope lookup before the
|
||||
// request body is decoded. Callers must still hash the complete bounded body
|
||||
// and ConfirmNativeReplay before acknowledging it.
|
||||
func (s *Store) CheckNativeReplay(ctx context.Context, token string, envelope model.BatchEnvelope) (Ack, bool, error) {
|
||||
if err := envelope.Validate(math.MaxInt64); err != nil {
|
||||
return Ack{}, false, err
|
||||
}
|
||||
source, err := s.Authenticate(ctx, token)
|
||||
if err != nil {
|
||||
return Ack{}, false, err
|
||||
}
|
||||
return s.checkEnvelope(ctx, source, envelope)
|
||||
}
|
||||
|
||||
func (s *Store) ConfirmNativeReplay(ctx context.Context, token string, envelope model.BatchEnvelope) (Ack, error) {
|
||||
if err := envelope.Validate(math.MaxInt64); err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
source, err := s.Authenticate(ctx, token)
|
||||
if err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
lock := s.sourceLock(source.ID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
ack, exact, err := s.checkEnvelope(ctx, source, envelope)
|
||||
if err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
if !exact {
|
||||
return Ack{}, errors.New("batch is not an acknowledged exact replay")
|
||||
}
|
||||
return ack, nil
|
||||
}
|
||||
|
||||
func envelopeSQL(envelope *model.BatchEnvelope) (batchDigest, wireDigest, signal string, recordCount int, encodedBytes int64, first, last string) {
|
||||
if envelope == nil {
|
||||
return "", "", "", 0, 0, "", ""
|
||||
}
|
||||
return envelope.BatchDigest, envelope.WireDigest, string(envelope.Signal), envelope.RecordCount, envelope.EncodedBytes, envelope.FirstObservedAt.UTC().Format(time.RFC3339Nano), envelope.LastObservedAt.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
func (s *Store) backfillAcknowledgedEnvelope(ctx context.Context, batch model.Batch, segmentDigest string, envelope model.BatchEnvelope) error {
|
||||
batchDigest, wireDigest, signal, recordCount, encodedBytes, first, last := envelopeSQL(&envelope)
|
||||
result, err := s.control.ExecContext(ctx, `UPDATE streams SET last_batch_digest=?,last_wire_digest=?,last_signal=?,last_record_count=?,last_encoded_bytes=?,last_first_observed_at=?,last_last_observed_at=? WHERE source_id=? AND stream_id=? AND last_sequence=? AND last_digest=? AND last_batch_digest='' AND last_wire_digest=''`, batchDigest, wireDigest, signal, recordCount, encodedBytes, first, last, batch.SourceID, batch.StreamID, batch.Sequence, segmentDigest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backfill acknowledged batch envelope: %w", err)
|
||||
}
|
||||
n, _ := result.RowsAffected()
|
||||
if n != 1 {
|
||||
return errors.New("acknowledged batch envelope state changed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func migrateControlBatchMetadata(db *sql.DB) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return errors.New("begin batch metadata migration")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, statement := range []string{
|
||||
`ALTER TABLE segments ADD COLUMN record_count INTEGER NOT NULL DEFAULT 0 CHECK(record_count BETWEEN 0 AND 5000)`,
|
||||
`UPDATE schema_version SET version=10 WHERE version=9`,
|
||||
} {
|
||||
if _, err = tx.Exec(statement); err != nil {
|
||||
return fmt.Errorf("migrate batch metadata: %w", err)
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return errors.New("commit batch metadata migration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
func TestCommittedSegmentRecordsBoundedBatchMetadata(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 19, 0, 10, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now.Add(-time.Second), Name: "first"}, {Timestamp: now, Name: "second"}}}
|
||||
ack, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count int
|
||||
var first, last string
|
||||
if err = store.control.QueryRowContext(ctx, `SELECT record_count,first_observed_at,last_observed_at FROM segments WHERE digest=?`, ack.Digest).Scan(&count, &first, &last); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 2 || first != now.Add(-time.Second).Format(time.RFC3339Nano) || last != now.Format(time.RFC3339Nano) {
|
||||
t.Fatalf("count=%d first=%q last=%q", count, first, last)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
const maxRawQuerySegments = 1_000_000
|
||||
|
||||
type rawQuerySegment struct {
|
||||
digest, path, sourceID, streamID string
|
||||
projectID, environmentID, serviceID string
|
||||
tier string
|
||||
sequence uint64
|
||||
uncompressedBytes int64
|
||||
firstObservedAt, lastObservedAt time.Time
|
||||
}
|
||||
|
||||
func (s *Store) coldSegmentsForQuery(ctx context.Context, ast query.AST, scope query.Scope, now time.Time) ([]rawQuerySegment, int64, error) {
|
||||
return s.rawSegmentsForQuery(ctx, ast, scope, now, false)
|
||||
}
|
||||
|
||||
func (s *Store) allRawSegmentsForQuery(ctx context.Context, ast query.AST, scope query.Scope, now time.Time) ([]rawQuerySegment, int64, error) {
|
||||
return s.rawSegmentsForQuery(ctx, ast, scope, now, true)
|
||||
}
|
||||
|
||||
func (s *Store) rawSegmentsForQuery(ctx context.Context, ast query.AST, scope query.Scope, now time.Time, includeHot bool) ([]rawQuerySegment, int64, error) {
|
||||
statement := `SELECT segment.digest,segment.path,segment.source_id,segment.stream_id,segment.sequence,segment.uncompressed_bytes,segment.first_observed_at,segment.last_observed_at,segment.tier,segment.archiving_at,segment.retiring_at,source.project_id,source.environment_id,source.service_id FROM segments segment JOIN sources source ON source.id=segment.source_id WHERE segment.organization_id=? AND source.organization_id=segment.organization_id AND segment.signal=?`
|
||||
if includeHot {
|
||||
statement += ` AND segment.tier IN ('hot','cold')`
|
||||
} else {
|
||||
statement += ` AND segment.tier='cold' AND segment.retiring_at IS NULL`
|
||||
}
|
||||
arguments := []any{scope.OrganizationID, ast.Signal}
|
||||
for _, selected := range []struct{ column, value string }{{"source.project_id", scope.ProjectID}, {"source.environment_id", scope.EnvironmentID}, {"source.service_id", scope.ServiceID}} {
|
||||
if selected.value != "" {
|
||||
statement += " AND " + selected.column + "=?"
|
||||
arguments = append(arguments, selected.value)
|
||||
}
|
||||
}
|
||||
if ast.Window > 0 {
|
||||
statement += " AND segment.last_observed_at>=?"
|
||||
arguments = append(arguments, now.UTC().Add(-ast.Window).Format(time.RFC3339Nano))
|
||||
}
|
||||
statement += ` ORDER BY segment.last_observed_at DESC,segment.source_id,segment.stream_id,segment.sequence DESC`
|
||||
rows, err := s.control.QueryContext(ctx, statement, arguments...)
|
||||
if err != nil {
|
||||
return nil, 0, errors.New("list raw query segments")
|
||||
}
|
||||
defer rows.Close()
|
||||
segments := make([]rawQuerySegment, 0)
|
||||
var estimated int64
|
||||
for rows.Next() {
|
||||
if len(segments) >= maxRawQuerySegments {
|
||||
return nil, 0, errors.New("raw query segment limit exceeded")
|
||||
}
|
||||
var segment rawQuerySegment
|
||||
var firstText, lastText string
|
||||
var archivingAt, retiringAt sql.NullString
|
||||
if err = rows.Scan(&segment.digest, &segment.path, &segment.sourceID, &segment.streamID, &segment.sequence, &segment.uncompressedBytes, &firstText, &lastText, &segment.tier, &archivingAt, &retiringAt, &segment.projectID, &segment.environmentID, &segment.serviceID); err != nil {
|
||||
return nil, 0, errors.New("read raw query segment")
|
||||
}
|
||||
if includeHot && (archivingAt.Valid || retiringAt.Valid) {
|
||||
return nil, 0, errors.New("raw query segment transition is incomplete")
|
||||
}
|
||||
segment.firstObservedAt, err = time.Parse(time.RFC3339Nano, firstText)
|
||||
if err != nil {
|
||||
return nil, 0, errors.New("raw query segment range is invalid")
|
||||
}
|
||||
segment.lastObservedAt, err = time.Parse(time.RFC3339Nano, lastText)
|
||||
if err != nil || segment.lastObservedAt.Before(segment.firstObservedAt) || segment.uncompressedBytes < 1 {
|
||||
return nil, 0, errors.New("raw query segment range is invalid")
|
||||
}
|
||||
var expected string
|
||||
var pathErr error
|
||||
if segment.tier == "cold" {
|
||||
expected, pathErr = s.coldArchivePath(scope.OrganizationID, archivingSegment{digest: segment.digest, path: segment.path, sourceID: segment.sourceID, streamID: segment.streamID, signal: ast.Signal})
|
||||
} else if segment.tier == "hot" && model.ValidateSourceID(scope.OrganizationID) == nil && model.ValidateSourceID(segment.sourceID) == nil && model.ValidateStreamID(segment.streamID) == nil {
|
||||
expected = filepath.Join(s.root, "raw", scope.OrganizationID, segment.sourceID, segment.streamID, fmt.Sprintf("%020d-%s.zst", segment.sequence, segment.digest))
|
||||
} else {
|
||||
pathErr = errors.New("unsupported raw segment tier")
|
||||
}
|
||||
if pathErr != nil || expected != segment.path {
|
||||
return nil, 0, errors.New("raw query segment path is invalid")
|
||||
}
|
||||
if segment.uncompressedBytes > math.MaxInt64-estimated {
|
||||
return nil, 0, errors.New("raw query estimate overflow")
|
||||
}
|
||||
estimated += segment.uncompressedBytes
|
||||
segments = append(segments, segment)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, 0, errors.New("list raw query segments")
|
||||
}
|
||||
return segments, estimated, nil
|
||||
}
|
||||
|
||||
func rawRecord(segment rawQuerySegment, batch model.Batch, index int) projectedRecord {
|
||||
observation := batch.Records[index]
|
||||
return projectedRecord{
|
||||
projectID: segment.projectID, environmentID: segment.environmentID, serviceID: segment.serviceID,
|
||||
sourceID: segment.sourceID, streamID: segment.streamID, sequence: batch.Sequence, recordIndex: index,
|
||||
signal: batch.Signal, timestamp: observation.Timestamp.UTC(), name: observation.Name, severity: observation.Severity,
|
||||
body: observation.Body, value: observation.Value, traceID: observation.TraceID, spanID: observation.SpanID,
|
||||
correlationID: observation.CorrelationID, attributes: observation.Attributes,
|
||||
}
|
||||
}
|
||||
|
||||
func rawRecordMemory(record projectedRecord) int64 {
|
||||
total := int64(256 + len(record.projectID) + len(record.environmentID) + len(record.serviceID) + len(record.sourceID) + len(record.streamID) + len(record.name) + len(record.severity) + len(record.body) + len(record.traceID) + len(record.spanID) + len(record.correlationID))
|
||||
for key, value := range record.attributes {
|
||||
addition := int64(len(key) + len(value) + 32)
|
||||
if addition > math.MaxInt64-total {
|
||||
return math.MaxInt64
|
||||
}
|
||||
total += addition
|
||||
}
|
||||
return total
|
||||
}
|
||||
@@ -0,0 +1,598 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
const (
|
||||
SavedQueryVersion = 1
|
||||
DashboardVersion = 1
|
||||
MaxDashboardPanels = 16
|
||||
)
|
||||
|
||||
var ErrDashboardRevisionConflict = errors.New("dashboard revision conflict")
|
||||
|
||||
type ResourceScope struct {
|
||||
ProjectID string `json:"project_id,omitempty"`
|
||||
EnvironmentID string `json:"environment_id,omitempty"`
|
||||
ServiceID string `json:"service_id,omitempty"`
|
||||
}
|
||||
|
||||
type SavedQuery struct {
|
||||
Version int `json:"version"`
|
||||
Revision int `json:"revision"`
|
||||
OrganizationID string `json:"organization_id"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Query string `json:"query"`
|
||||
AST query.AST `json:"ast"`
|
||||
Scope ResourceScope `json:"scope"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type SavedQueryInput struct {
|
||||
ID string
|
||||
ExpectedRevision int
|
||||
OrganizationID string
|
||||
Name string
|
||||
Description string
|
||||
Query string
|
||||
Scope ResourceScope
|
||||
ActorUserID string
|
||||
MaxRows int
|
||||
}
|
||||
|
||||
type DashboardPanel struct {
|
||||
ID string `json:"id"`
|
||||
Position int `json:"position"`
|
||||
Title string `json:"title"`
|
||||
Visualization string `json:"visualization"`
|
||||
SavedQueryID string `json:"saved_query_id"`
|
||||
}
|
||||
|
||||
type Dashboard struct {
|
||||
Version int `json:"version"`
|
||||
Revision int `json:"revision"`
|
||||
OrganizationID string `json:"organization_id"`
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Panels []DashboardPanel `json:"panels"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type DashboardInput struct {
|
||||
ID string
|
||||
ExpectedRevision int
|
||||
OrganizationID string
|
||||
Slug string
|
||||
Name string
|
||||
Description string
|
||||
Panels []DashboardPanel
|
||||
ActorUserID string
|
||||
}
|
||||
|
||||
type DashboardExport struct {
|
||||
Version int `json:"version"`
|
||||
Dashboard DashboardDefinition `json:"dashboard"`
|
||||
SavedQueries []SavedQueryDefinition `json:"saved_queries"`
|
||||
}
|
||||
|
||||
type DashboardDefinition struct {
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Panels []DashboardPanel `json:"panels"`
|
||||
}
|
||||
|
||||
type SavedQueryDefinition struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Query string `json:"query"`
|
||||
Scope ResourceScope `json:"scope"`
|
||||
}
|
||||
|
||||
type DashboardImportInput struct {
|
||||
OrganizationID string
|
||||
ActorUserID string
|
||||
MaxRows int
|
||||
Bundle DashboardExport
|
||||
}
|
||||
|
||||
func (s *Store) SaveQuery(ctx context.Context, input SavedQueryInput, now time.Time) (SavedQuery, error) {
|
||||
ast, err := validateSavedQueryInput(input, now)
|
||||
if err != nil {
|
||||
return SavedQuery{}, err
|
||||
}
|
||||
astJSON, err := json.Marshal(ast)
|
||||
if err != nil {
|
||||
return SavedQuery{}, errors.New("encode saved query AST")
|
||||
}
|
||||
timestamp := now.UTC().Format(time.RFC3339Nano)
|
||||
if input.ID == "" {
|
||||
input.ID, err = storageID("query")
|
||||
if err != nil {
|
||||
return SavedQuery{}, err
|
||||
}
|
||||
_, err = s.control.ExecContext(ctx, `INSERT INTO saved_queries(organization_id,id,version,revision,name,description,query_text,ast_json,project_id,environment_id,service_id,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?,?,?,?,?)`, input.OrganizationID, input.ID, input.Name, input.Description, input.Query, string(astJSON), input.Scope.ProjectID, input.Scope.EnvironmentID, input.Scope.ServiceID, input.ActorUserID, input.ActorUserID, timestamp, timestamp)
|
||||
if err != nil {
|
||||
return SavedQuery{}, errors.New("create saved query")
|
||||
}
|
||||
} else {
|
||||
if input.ExpectedRevision < 1 || model.ValidateSourceID(input.ID) != nil {
|
||||
return SavedQuery{}, errors.New("saved query revision input is invalid")
|
||||
}
|
||||
result, updateErr := s.control.ExecContext(ctx, `UPDATE saved_queries SET revision=revision+1,name=?,description=?,query_text=?,ast_json=?,project_id=?,environment_id=?,service_id=?,updated_by=?,updated_at=? WHERE organization_id=? AND id=? AND revision=?`, input.Name, input.Description, input.Query, string(astJSON), input.Scope.ProjectID, input.Scope.EnvironmentID, input.Scope.ServiceID, input.ActorUserID, timestamp, input.OrganizationID, input.ID, input.ExpectedRevision)
|
||||
if updateErr != nil {
|
||||
return SavedQuery{}, errors.New("update saved query")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return SavedQuery{}, errors.New("saved query revision conflict")
|
||||
}
|
||||
}
|
||||
return s.SavedQuery(ctx, input.OrganizationID, input.ID)
|
||||
}
|
||||
|
||||
func (s *Store) SavedQuery(ctx context.Context, organizationID, id string) (SavedQuery, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(id) != nil {
|
||||
return SavedQuery{}, errors.New("saved query identity is invalid")
|
||||
}
|
||||
row := s.control.QueryRowContext(ctx, `SELECT version,revision,name,description,query_text,ast_json,project_id,environment_id,service_id,created_by,updated_by,created_at,updated_at FROM saved_queries WHERE organization_id=? AND id=?`, organizationID, id)
|
||||
return scanSavedQuery(row, organizationID, id)
|
||||
}
|
||||
|
||||
func (s *Store) SavedQueries(ctx context.Context, organizationID string) ([]SavedQuery, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil {
|
||||
return nil, errors.New("invalid organization identifier")
|
||||
}
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT id,version,revision,name,description,query_text,ast_json,project_id,environment_id,service_id,created_by,updated_by,created_at,updated_at FROM saved_queries WHERE organization_id=? ORDER BY name,id`, organizationID)
|
||||
if err != nil {
|
||||
return nil, errors.New("list saved queries")
|
||||
}
|
||||
defer rows.Close()
|
||||
var result []SavedQuery
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var value SavedQuery
|
||||
var astJSON, createdAt, updatedAt string
|
||||
value.OrganizationID = organizationID
|
||||
if err = rows.Scan(&id, &value.Version, &value.Revision, &value.Name, &value.Description, &value.Query, &astJSON, &value.Scope.ProjectID, &value.Scope.EnvironmentID, &value.Scope.ServiceID, &value.CreatedBy, &value.UpdatedBy, &createdAt, &updatedAt); err != nil {
|
||||
return nil, errors.New("read saved query")
|
||||
}
|
||||
value.ID = id
|
||||
if err = decodeSavedQuery(&value, astJSON, createdAt, updatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, value)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, errors.New("list saved queries")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type rowScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanSavedQuery(row rowScanner, organizationID, id string) (SavedQuery, error) {
|
||||
value := SavedQuery{OrganizationID: organizationID, ID: id}
|
||||
var astJSON, createdAt, updatedAt string
|
||||
if err := row.Scan(&value.Version, &value.Revision, &value.Name, &value.Description, &value.Query, &astJSON, &value.Scope.ProjectID, &value.Scope.EnvironmentID, &value.Scope.ServiceID, &value.CreatedBy, &value.UpdatedBy, &createdAt, &updatedAt); errors.Is(err, sql.ErrNoRows) {
|
||||
return SavedQuery{}, errors.New("saved query not found")
|
||||
} else if err != nil {
|
||||
return SavedQuery{}, errors.New("read saved query")
|
||||
}
|
||||
if err := decodeSavedQuery(&value, astJSON, createdAt, updatedAt); err != nil {
|
||||
return SavedQuery{}, err
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func decodeSavedQuery(value *SavedQuery, astJSON, createdAt, updatedAt string) error {
|
||||
if value.Version != SavedQueryVersion || value.Revision < 1 || model.ValidateSourceID(value.OrganizationID) != nil || model.ValidateSourceID(value.ID) != nil || model.ValidateSourceID(value.CreatedBy) != nil || model.ValidateSourceID(value.UpdatedBy) != nil || !validResourceScope(value.Scope) || !boundedText(value.Name, 128, false) || !boundedText(value.Description, 1024, true) {
|
||||
return errors.New("stored saved query is invalid")
|
||||
}
|
||||
parsed, err := query.Parse(value.Query, 100_000)
|
||||
if err != nil || json.Unmarshal([]byte(astJSON), &value.AST) != nil || hydrateSavedAST(&value.AST) != nil || query.Validate(value.AST, 100_000) != nil {
|
||||
return errors.New("stored saved query AST is invalid")
|
||||
}
|
||||
parsedJSON, _ := json.Marshal(parsed)
|
||||
storedJSON, _ := json.Marshal(value.AST)
|
||||
if string(parsedJSON) != string(storedJSON) {
|
||||
return errors.New("stored saved query AST does not match query text")
|
||||
}
|
||||
value.CreatedAt, err = time.Parse(time.RFC3339Nano, createdAt)
|
||||
if err != nil {
|
||||
return errors.New("stored saved query created time is invalid")
|
||||
}
|
||||
value.UpdatedAt, err = time.Parse(time.RFC3339Nano, updatedAt)
|
||||
if err != nil || value.UpdatedAt.Before(value.CreatedAt) {
|
||||
return errors.New("stored saved query updated time is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func hydrateSavedAST(ast *query.AST) error {
|
||||
var err error
|
||||
if ast.WindowText != "" {
|
||||
ast.Window, err = time.ParseDuration(ast.WindowText)
|
||||
}
|
||||
if err == nil && ast.BucketText != "" {
|
||||
ast.Bucket, err = time.ParseDuration(ast.BucketText)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.New("stored saved query duration is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) SaveDashboard(ctx context.Context, input DashboardInput, now time.Time) (Dashboard, error) {
|
||||
if err := validateDashboardInput(input, now); err != nil {
|
||||
return Dashboard{}, err
|
||||
}
|
||||
if input.ExpectedRevision == 0 && input.ID != "" {
|
||||
return Dashboard{}, errors.New("new dashboard identity is server-generated")
|
||||
}
|
||||
if input.ExpectedRevision > 0 && model.ValidateSourceID(input.ID) != nil {
|
||||
return Dashboard{}, errors.New("dashboard revision input is invalid")
|
||||
}
|
||||
var err error
|
||||
if input.ID == "" {
|
||||
input.ID, err = storageID("dashboard")
|
||||
if err != nil {
|
||||
return Dashboard{}, err
|
||||
}
|
||||
}
|
||||
panels := append([]DashboardPanel(nil), input.Panels...)
|
||||
for index := range panels {
|
||||
if panels[index].ID == "" {
|
||||
panels[index].ID, err = storageID("panel")
|
||||
if err != nil {
|
||||
return Dashboard{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return Dashboard{}, errors.New("begin dashboard update")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
timestamp := now.UTC().Format(time.RFC3339Nano)
|
||||
if input.ExpectedRevision == 0 {
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO dashboards(organization_id,id,version,revision,slug,name,description,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?)`, input.OrganizationID, input.ID, input.Slug, input.Name, input.Description, input.ActorUserID, input.ActorUserID, timestamp, timestamp)
|
||||
if err != nil {
|
||||
return Dashboard{}, errors.New("create dashboard")
|
||||
}
|
||||
} else {
|
||||
if model.ValidateSourceID(input.ID) != nil || input.ExpectedRevision < 1 {
|
||||
return Dashboard{}, errors.New("dashboard revision input is invalid")
|
||||
}
|
||||
result, updateErr := tx.ExecContext(ctx, `UPDATE dashboards SET revision=revision+1,slug=?,name=?,description=?,updated_by=?,updated_at=? WHERE organization_id=? AND id=? AND revision=?`, input.Slug, input.Name, input.Description, input.ActorUserID, timestamp, input.OrganizationID, input.ID, input.ExpectedRevision)
|
||||
if updateErr != nil {
|
||||
return Dashboard{}, errors.New("update dashboard")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return Dashboard{}, ErrDashboardRevisionConflict
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM dashboard_panels WHERE organization_id=? AND dashboard_id=?`, input.OrganizationID, input.ID); err != nil {
|
||||
return Dashboard{}, errors.New("replace dashboard panels")
|
||||
}
|
||||
}
|
||||
for _, panel := range panels {
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO dashboard_panels(organization_id,dashboard_id,id,position,title,visualization,saved_query_id) VALUES(?,?,?,?,?,?,?)`, input.OrganizationID, input.ID, panel.ID, panel.Position, panel.Title, panel.Visualization, panel.SavedQueryID); err != nil {
|
||||
return Dashboard{}, errors.New("store dashboard panel")
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return Dashboard{}, errors.New("commit dashboard update")
|
||||
}
|
||||
return s.Dashboard(ctx, input.OrganizationID, input.Slug)
|
||||
}
|
||||
|
||||
func (s *Store) Dashboard(ctx context.Context, organizationID, slug string) (Dashboard, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || !validSlug(slug) {
|
||||
return Dashboard{}, errors.New("dashboard identity is invalid")
|
||||
}
|
||||
value := Dashboard{OrganizationID: organizationID}
|
||||
var createdAt, updatedAt string
|
||||
err := s.control.QueryRowContext(ctx, `SELECT id,version,revision,name,description,created_by,updated_by,created_at,updated_at FROM dashboards WHERE organization_id=? AND slug=?`, organizationID, slug).Scan(&value.ID, &value.Version, &value.Revision, &value.Name, &value.Description, &value.CreatedBy, &value.UpdatedBy, &createdAt, &updatedAt)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Dashboard{}, errors.New("dashboard not found")
|
||||
}
|
||||
if err != nil {
|
||||
return Dashboard{}, errors.New("read dashboard")
|
||||
}
|
||||
value.Slug = slug
|
||||
value.CreatedAt, err = time.Parse(time.RFC3339Nano, createdAt)
|
||||
if err != nil {
|
||||
return Dashboard{}, errors.New("stored dashboard created time is invalid")
|
||||
}
|
||||
value.UpdatedAt, err = time.Parse(time.RFC3339Nano, updatedAt)
|
||||
if err != nil || value.UpdatedAt.Before(value.CreatedAt) {
|
||||
return Dashboard{}, errors.New("stored dashboard updated time is invalid")
|
||||
}
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT id,position,title,visualization,saved_query_id FROM dashboard_panels WHERE organization_id=? AND dashboard_id=? ORDER BY position,id`, organizationID, value.ID)
|
||||
if err != nil {
|
||||
return Dashboard{}, errors.New("read dashboard panels")
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var panel DashboardPanel
|
||||
if err = rows.Scan(&panel.ID, &panel.Position, &panel.Title, &panel.Visualization, &panel.SavedQueryID); err != nil || validatePanel(panel) != nil {
|
||||
return Dashboard{}, errors.New("stored dashboard panel is invalid")
|
||||
}
|
||||
value.Panels = append(value.Panels, panel)
|
||||
}
|
||||
if err = rows.Err(); err != nil || validateDashboard(value) != nil {
|
||||
return Dashboard{}, errors.New("stored dashboard is invalid")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (s *Store) Dashboards(ctx context.Context, organizationID string) ([]Dashboard, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil {
|
||||
return nil, errors.New("invalid organization identifier")
|
||||
}
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT slug FROM dashboards WHERE organization_id=? ORDER BY name,slug`, organizationID)
|
||||
if err != nil {
|
||||
return nil, errors.New("list dashboards")
|
||||
}
|
||||
var slugs []string
|
||||
for rows.Next() {
|
||||
var slug string
|
||||
if err = rows.Scan(&slug); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, errors.New("list dashboards")
|
||||
}
|
||||
slugs = append(slugs, slug)
|
||||
}
|
||||
if err = rows.Close(); err != nil || rows.Err() != nil {
|
||||
return nil, errors.New("list dashboards")
|
||||
}
|
||||
result := make([]Dashboard, 0, len(slugs))
|
||||
for _, slug := range slugs {
|
||||
value, loadErr := s.Dashboard(ctx, organizationID, slug)
|
||||
if loadErr != nil {
|
||||
return nil, loadErr
|
||||
}
|
||||
result = append(result, value)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) ExportDashboard(ctx context.Context, organizationID, slug string) (DashboardExport, error) {
|
||||
dashboard, err := s.Dashboard(ctx, organizationID, slug)
|
||||
if err != nil {
|
||||
return DashboardExport{}, err
|
||||
}
|
||||
queries := make([]SavedQueryDefinition, 0, len(dashboard.Panels))
|
||||
seen := map[string]bool{}
|
||||
for _, panel := range dashboard.Panels {
|
||||
if seen[panel.SavedQueryID] {
|
||||
continue
|
||||
}
|
||||
value, loadErr := s.SavedQuery(ctx, organizationID, panel.SavedQueryID)
|
||||
if loadErr != nil {
|
||||
return DashboardExport{}, loadErr
|
||||
}
|
||||
seen[value.ID] = true
|
||||
queries = append(queries, SavedQueryDefinition{ID: value.ID, Name: value.Name, Description: value.Description, Query: value.Query, Scope: value.Scope})
|
||||
}
|
||||
sort.Slice(queries, func(i, j int) bool { return queries[i].ID < queries[j].ID })
|
||||
definition := DashboardDefinition{Slug: dashboard.Slug, Name: dashboard.Name, Description: dashboard.Description, Panels: dashboard.Panels}
|
||||
return DashboardExport{Version: DashboardVersion, Dashboard: definition, SavedQueries: queries}, nil
|
||||
}
|
||||
|
||||
// ImportDashboard validates one source-control-safe export and creates its
|
||||
// queries, dashboard, and panels atomically with new server-owned identities.
|
||||
// Tenant and actor metadata are supplied independently of the bundle.
|
||||
func (s *Store) ImportDashboard(ctx context.Context, input DashboardImportInput, now time.Time) (Dashboard, error) {
|
||||
if input.Bundle.Version != DashboardVersion || model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.ActorUserID) != nil || input.MaxRows < 1 || input.MaxRows > 100_000 || now.IsZero() || len(input.Bundle.SavedQueries) > MaxDashboardPanels {
|
||||
return Dashboard{}, errors.New("dashboard import is invalid")
|
||||
}
|
||||
queryIDs := make(map[string]string, len(input.Bundle.SavedQueries))
|
||||
type importedQuery struct {
|
||||
definition SavedQueryDefinition
|
||||
id string
|
||||
astJSON string
|
||||
ast query.AST
|
||||
}
|
||||
queries := make([]importedQuery, 0, len(input.Bundle.SavedQueries))
|
||||
for _, definition := range input.Bundle.SavedQueries {
|
||||
if model.ValidateSourceID(definition.ID) != nil || queryIDs[definition.ID] != "" {
|
||||
return Dashboard{}, errors.New("dashboard import query identity is invalid or duplicated")
|
||||
}
|
||||
ast, err := validateSavedQueryInput(SavedQueryInput{OrganizationID: input.OrganizationID, ActorUserID: input.ActorUserID, MaxRows: input.MaxRows, Name: definition.Name, Description: definition.Description, Query: definition.Query, Scope: definition.Scope}, now)
|
||||
if err != nil {
|
||||
return Dashboard{}, err
|
||||
}
|
||||
encoded, err := json.Marshal(ast)
|
||||
if err != nil {
|
||||
return Dashboard{}, errors.New("encode imported saved query AST")
|
||||
}
|
||||
id, err := storageID("query")
|
||||
if err != nil {
|
||||
return Dashboard{}, err
|
||||
}
|
||||
queryIDs[definition.ID] = id
|
||||
queries = append(queries, importedQuery{definition: definition, id: id, astJSON: string(encoded), ast: ast})
|
||||
}
|
||||
queryASTs := make(map[string]query.AST, len(queries))
|
||||
for _, imported := range queries {
|
||||
queryASTs[imported.definition.ID] = imported.ast
|
||||
}
|
||||
panels := make([]DashboardPanel, len(input.Bundle.Dashboard.Panels))
|
||||
referenced := make(map[string]bool, len(queries))
|
||||
for index, panel := range input.Bundle.Dashboard.Panels {
|
||||
mapped := queryIDs[panel.SavedQueryID]
|
||||
if mapped == "" {
|
||||
return Dashboard{}, errors.New("dashboard import panel references an unknown saved query")
|
||||
}
|
||||
ast := queryASTs[panel.SavedQueryID]
|
||||
if panel.Visualization == "stat" && ast.Summary == nil || panel.Visualization == "timeseries" && (ast.Summary == nil || ast.Bucket <= 0) {
|
||||
return Dashboard{}, errors.New("dashboard import presentation does not match its saved query")
|
||||
}
|
||||
panelID, err := storageID("panel")
|
||||
if err != nil {
|
||||
return Dashboard{}, err
|
||||
}
|
||||
panel.ID, panel.SavedQueryID = panelID, mapped
|
||||
panels[index] = panel
|
||||
referenced[panel.SavedQueryID] = true
|
||||
}
|
||||
if len(referenced) != len(queries) {
|
||||
return Dashboard{}, errors.New("dashboard import contains an unreferenced saved query")
|
||||
}
|
||||
dashboardID, err := storageID("dashboard")
|
||||
if err != nil {
|
||||
return Dashboard{}, err
|
||||
}
|
||||
dashboardInput := DashboardInput{ID: dashboardID, ExpectedRevision: 1, OrganizationID: input.OrganizationID, ActorUserID: input.ActorUserID, Slug: input.Bundle.Dashboard.Slug, Name: input.Bundle.Dashboard.Name, Description: input.Bundle.Dashboard.Description, Panels: panels}
|
||||
if err = validateDashboardInput(dashboardInput, now); err != nil {
|
||||
return Dashboard{}, err
|
||||
}
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return Dashboard{}, errors.New("begin dashboard import")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
timestamp := now.UTC().Format(time.RFC3339Nano)
|
||||
for _, imported := range queries {
|
||||
definition := imported.definition
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO saved_queries(organization_id,id,version,revision,name,description,query_text,ast_json,project_id,environment_id,service_id,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?,?,?,?,?)`, input.OrganizationID, imported.id, definition.Name, definition.Description, strings.TrimSpace(definition.Query), imported.astJSON, definition.Scope.ProjectID, definition.Scope.EnvironmentID, definition.Scope.ServiceID, input.ActorUserID, input.ActorUserID, timestamp, timestamp)
|
||||
if err != nil {
|
||||
return Dashboard{}, errors.New("import saved query")
|
||||
}
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO dashboards(organization_id,id,version,revision,slug,name,description,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?)`, input.OrganizationID, dashboardID, input.Bundle.Dashboard.Slug, input.Bundle.Dashboard.Name, input.Bundle.Dashboard.Description, input.ActorUserID, input.ActorUserID, timestamp, timestamp)
|
||||
if err != nil {
|
||||
return Dashboard{}, errors.New("import dashboard")
|
||||
}
|
||||
for _, panel := range panels {
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO dashboard_panels(organization_id,dashboard_id,id,position,title,visualization,saved_query_id) VALUES(?,?,?,?,?,?,?)`, input.OrganizationID, dashboardID, panel.ID, panel.Position, panel.Title, panel.Visualization, panel.SavedQueryID); err != nil {
|
||||
return Dashboard{}, errors.New("import dashboard panel")
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return Dashboard{}, errors.New("commit dashboard import")
|
||||
}
|
||||
return s.Dashboard(ctx, input.OrganizationID, input.Bundle.Dashboard.Slug)
|
||||
}
|
||||
|
||||
func validateSavedQueryInput(input SavedQueryInput, now time.Time) (query.AST, error) {
|
||||
if model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.ActorUserID) != nil || !boundedText(input.Name, 128, false) || !boundedText(input.Description, 1024, true) || !validResourceScope(input.Scope) || input.MaxRows < 1 || input.MaxRows > 100_000 || now.IsZero() {
|
||||
return query.AST{}, errors.New("saved query input is invalid")
|
||||
}
|
||||
ast, err := query.Parse(strings.TrimSpace(input.Query), input.MaxRows)
|
||||
if err != nil {
|
||||
return query.AST{}, fmt.Errorf("saved query: %w", err)
|
||||
}
|
||||
return ast, nil
|
||||
}
|
||||
|
||||
func validateDashboardInput(input DashboardInput, now time.Time) error {
|
||||
if model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.ActorUserID) != nil || !validSlug(input.Slug) || !boundedText(input.Name, 128, false) || !boundedText(input.Description, 1024, true) || len(input.Panels) > MaxDashboardPanels || now.IsZero() {
|
||||
return errors.New("dashboard input is invalid")
|
||||
}
|
||||
positions := map[int]bool{}
|
||||
ids := map[string]bool{}
|
||||
for _, panel := range input.Panels {
|
||||
if err := validatePanelInput(panel); err != nil || positions[panel.Position] || panel.ID != "" && ids[panel.ID] {
|
||||
return errors.New("dashboard panel input is invalid")
|
||||
}
|
||||
positions[panel.Position] = true
|
||||
if panel.ID != "" {
|
||||
ids[panel.ID] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDashboard(value Dashboard) error {
|
||||
if value.Version != DashboardVersion || value.Revision < 1 || model.ValidateSourceID(value.OrganizationID) != nil || model.ValidateSourceID(value.ID) != nil || model.ValidateSourceID(value.CreatedBy) != nil || model.ValidateSourceID(value.UpdatedBy) != nil || !validSlug(value.Slug) || !boundedText(value.Name, 128, false) || !boundedText(value.Description, 1024, true) || len(value.Panels) > MaxDashboardPanels {
|
||||
return errors.New("dashboard is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePanel(panel DashboardPanel) error {
|
||||
if model.ValidateSourceID(panel.ID) != nil {
|
||||
return errors.New("dashboard panel is invalid")
|
||||
}
|
||||
return validatePanelInput(panel)
|
||||
}
|
||||
|
||||
func validatePanelInput(panel DashboardPanel) error {
|
||||
if panel.ID != "" && model.ValidateSourceID(panel.ID) != nil || panel.Position < 0 || panel.Position >= 64 || !boundedText(panel.Title, 128, false) || model.ValidateSourceID(panel.SavedQueryID) != nil {
|
||||
return errors.New("dashboard panel is invalid")
|
||||
}
|
||||
switch panel.Visualization {
|
||||
case "table", "stat", "timeseries":
|
||||
return nil
|
||||
default:
|
||||
return errors.New("dashboard panel visualization is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func validResourceScope(scope ResourceScope) bool {
|
||||
values := []string{scope.ProjectID, scope.EnvironmentID, scope.ServiceID}
|
||||
for _, value := range values {
|
||||
if value != "" && model.ValidateSourceID(value) != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if scope.EnvironmentID != "" && scope.ProjectID == "" {
|
||||
return false
|
||||
}
|
||||
if scope.ServiceID != "" && (scope.ProjectID == "" || scope.EnvironmentID == "") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func boundedText(value string, maximum int, empty bool) bool {
|
||||
return utf8.ValidString(value) && !strings.ContainsAny(value, "\x00\r\n") && len(value) <= maximum && (empty || value != "")
|
||||
}
|
||||
|
||||
func validSlug(value string) bool {
|
||||
if len(value) < 2 || len(value) > 63 || value[0] < 'a' || value[0] > 'z' {
|
||||
return false
|
||||
}
|
||||
for _, character := range value {
|
||||
if character != '-' && (character < 'a' || character > 'z') && (character < '0' || character > '9') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func storageID(prefix string) (string, error) {
|
||||
random := make([]byte, 18)
|
||||
if _, err := rand.Read(random); err != nil {
|
||||
return "", errors.New("cryptographic randomness unavailable")
|
||||
}
|
||||
return prefix + "_" + base64.RawURLEncoding.EncodeToString(random), nil
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSavedQueriesAreTypedVersionedAndOptimisticallyUpdated(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 10, 0, 0, 0, time.UTC)
|
||||
input := SavedQueryInput{
|
||||
OrganizationID: "organization-a", Name: "Recent failures",
|
||||
Description: "Recent failed application requests grouped by route.",
|
||||
Query: `logs | where status >= 500 | window 1h | summarize count() by route | sort count desc | limit 50`,
|
||||
Scope: ResourceScope{ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"},
|
||||
ActorUserID: "operator-a", MaxRows: 1000,
|
||||
}
|
||||
created, err := store.SaveQuery(ctx, input, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if created.Version != SavedQueryVersion || created.Revision != 1 || created.ID == "" || created.AST.Signal != "logs" || created.AST.Window != time.Hour || created.CreatedAt != now || created.UpdatedAt != now {
|
||||
t.Fatalf("created=%+v", created)
|
||||
}
|
||||
input.ID, input.ExpectedRevision = created.ID, created.Revision
|
||||
input.Description = "Reviewed application failures grouped by normalized route."
|
||||
updated, err := store.SaveQuery(ctx, input, now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if updated.Revision != 2 || updated.Description != input.Description || updated.UpdatedBy != "operator-a" || !updated.UpdatedAt.Equal(now.Add(time.Minute)) {
|
||||
t.Fatalf("updated=%+v", updated)
|
||||
}
|
||||
if _, err = store.SaveQuery(ctx, input, now.Add(2*time.Minute)); err == nil {
|
||||
t.Fatal("stale saved-query revision was accepted")
|
||||
}
|
||||
queries, err := store.SavedQueries(ctx, "organization-a")
|
||||
if err != nil || len(queries) != 1 || queries[0].Revision != 2 {
|
||||
t.Fatalf("queries=%+v err=%v", queries, err)
|
||||
}
|
||||
if _, err = store.control.Exec(`UPDATE saved_queries SET ast_json='{}' WHERE organization_id='organization-a' AND id=?`, created.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.SavedQuery(ctx, "organization-a", created.ID); err == nil {
|
||||
t.Fatal("query text and stored AST disagreement was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardPanelsRemainOrganizationScopedAndExportSafeDefinitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 10, 0, 0, 0, time.UTC)
|
||||
queryA, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", Name: "Request rate", Description: "Five-minute request counts.", Query: `logs | window 1h | summarize count() by window(5m) | limit 50`, ActorUserID: "operator-a", MaxRows: 1000}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queryB, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-b", Name: "Other organization", Description: "Must not cross the tenant boundary.", Query: `logs | limit 10`, ActorUserID: "operator-b", MaxRows: 1000}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := DashboardInput{
|
||||
OrganizationID: "organization-a", Slug: "operations", Name: "Operations",
|
||||
Description: "Recent service activity and evidence.", ActorUserID: "operator-a",
|
||||
Panels: []DashboardPanel{{Position: 0, Title: "Request rate", Visualization: "timeseries", SavedQueryID: queryA.ID}},
|
||||
}
|
||||
created, err := store.SaveDashboard(ctx, input, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if created.Version != DashboardVersion || created.Revision != 1 || len(created.Panels) != 1 || created.Panels[0].ID == "" {
|
||||
t.Fatalf("created=%+v", created)
|
||||
}
|
||||
unsafe := input
|
||||
unsafe.Slug = "cross-tenant"
|
||||
unsafe.Panels = []DashboardPanel{{Position: 0, Title: "Other", Visualization: "table", SavedQueryID: queryB.ID}}
|
||||
if _, err = store.SaveDashboard(ctx, unsafe, now); err == nil {
|
||||
t.Fatal("cross-organization saved query entered dashboard")
|
||||
}
|
||||
input.ID, input.ExpectedRevision = created.ID, created.Revision
|
||||
input.Name = "Service operations"
|
||||
input.Panels[0].ID = created.Panels[0].ID
|
||||
updated, err := store.SaveDashboard(ctx, input, now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if updated.Revision != 2 || updated.Name != input.Name {
|
||||
t.Fatalf("updated=%+v", updated)
|
||||
}
|
||||
if _, err = store.SaveDashboard(ctx, input, now.Add(2*time.Minute)); !errors.Is(err, ErrDashboardRevisionConflict) {
|
||||
t.Fatalf("stale dashboard revision err=%v", err)
|
||||
}
|
||||
exported, err := store.ExportDashboard(ctx, "organization-a", "operations")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := json.Marshal(exported)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"organization-a", "operator-a", "created_at", "updated_at", "ast"} {
|
||||
if strings.Contains(string(body), forbidden) {
|
||||
t.Fatalf("private runtime metadata %q entered export: %s", forbidden, body)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(body), `"version":1`) || !strings.Contains(string(body), `"query":"logs | window 1h`) {
|
||||
t.Fatalf("export=%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardValidationBoundsScopePanelsAndServerGeneratedIdentity(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 10, 0, 0, 0, time.UTC)
|
||||
if _, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", Name: "Invalid scope", Query: `logs | limit 10`, Scope: ResourceScope{ServiceID: "service-a"}, ActorUserID: "operator-a", MaxRows: 1000}, now); err == nil {
|
||||
t.Fatal("service-only query scope was accepted")
|
||||
}
|
||||
queryValue, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", Name: "Valid", Query: `logs | limit 10`, ActorUserID: "operator-a", MaxRows: 1000}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.SaveDashboard(ctx, DashboardInput{ID: "caller-selected", OrganizationID: "organization-a", Slug: "invalid-id", Name: "Invalid", ActorUserID: "operator-a"}, now); err == nil {
|
||||
t.Fatal("caller-selected new dashboard identity was accepted")
|
||||
}
|
||||
duplicate := DashboardInput{OrganizationID: "organization-a", Slug: "duplicate-panels", Name: "Duplicate panels", ActorUserID: "operator-a", Panels: []DashboardPanel{
|
||||
{Position: 0, Title: "First", Visualization: "table", SavedQueryID: queryValue.ID},
|
||||
{Position: 0, Title: "Second", Visualization: "stat", SavedQueryID: queryValue.ID},
|
||||
}}
|
||||
if _, err = store.SaveDashboard(ctx, duplicate, now); err == nil {
|
||||
t.Fatal("duplicate dashboard panel positions were accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardImportIsAtomicTenantIndependentAndRevalidated(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 11, 0, 0, 0, time.UTC)
|
||||
bundle := DashboardExport{
|
||||
Version: DashboardVersion,
|
||||
Dashboard: DashboardDefinition{Slug: "operations", Name: "Operations", Description: "A portable service view.", Panels: []DashboardPanel{
|
||||
{ID: "old-panel-1", Position: 0, Title: "Recent failures", Visualization: "table", SavedQueryID: "portable-query-1"},
|
||||
}},
|
||||
SavedQueries: []SavedQueryDefinition{{ID: "portable-query-1", Name: "Recent failures", Description: "Bounded failures.", Query: `logs | where status >= 500 | window 1h | limit 50`}},
|
||||
}
|
||||
imported, err := store.ImportDashboard(ctx, DashboardImportInput{OrganizationID: "organization-b", ActorUserID: "operator-b", MaxRows: 1_000, Bundle: bundle}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if imported.OrganizationID != "organization-b" || imported.CreatedBy != "operator-b" || imported.ID == "" || len(imported.Panels) != 1 || imported.Panels[0].ID == "old-panel-1" || imported.Panels[0].SavedQueryID == "portable-query-1" {
|
||||
t.Fatalf("imported=%+v", imported)
|
||||
}
|
||||
queries, err := store.SavedQueries(ctx, "organization-b")
|
||||
if err != nil || len(queries) != 1 || queries[0].OrganizationID != "organization-b" || queries[0].CreatedBy != "operator-b" || queries[0].ID != imported.Panels[0].SavedQueryID {
|
||||
t.Fatalf("queries=%+v err=%v", queries, err)
|
||||
}
|
||||
invalid := bundle
|
||||
invalid.Dashboard.Slug = "partial-import"
|
||||
invalid.SavedQueries = append(invalid.SavedQueries, SavedQueryDefinition{ID: "unused-query-2", Name: "Unused", Query: `logs | limit 10`})
|
||||
if _, err = store.ImportDashboard(ctx, DashboardImportInput{OrganizationID: "organization-b", ActorUserID: "operator-b", MaxRows: 1_000, Bundle: invalid}, now); err == nil {
|
||||
t.Fatal("unreferenced imported query was accepted")
|
||||
}
|
||||
if dashboards, listErr := store.Dashboards(ctx, "organization-b"); listErr != nil || len(dashboards) != 1 {
|
||||
t.Fatalf("failed import left partial dashboard: dashboards=%+v err=%v", dashboards, listErr)
|
||||
}
|
||||
if queries, listErr := store.SavedQueries(ctx, "organization-b"); listErr != nil || len(queries) != 1 {
|
||||
t.Fatalf("failed import left partial queries: queries=%+v err=%v", queries, listErr)
|
||||
}
|
||||
incompatible := bundle
|
||||
incompatible.Dashboard.Slug = "invalid-presentation"
|
||||
incompatible.Dashboard.Panels[0].Visualization = "timeseries"
|
||||
if _, err = store.ImportDashboard(ctx, DashboardImportInput{OrganizationID: "organization-b", ActorUserID: "operator-b", MaxRows: 1_000, Bundle: incompatible}, now); err == nil {
|
||||
t.Fatal("timeseries dashboard without a bucketed summary was accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
type StoredProposal struct {
|
||||
Proposal schema.Proposal `json:"proposal"`
|
||||
Status string `json:"status"`
|
||||
FirstSeenAt time.Time `json:"first_seen_at"`
|
||||
LastSeenAt time.Time `json:"last_seen_at"`
|
||||
}
|
||||
|
||||
type fieldEvidence struct {
|
||||
descriptor schema.Descriptor
|
||||
count int64
|
||||
bytes int64
|
||||
}
|
||||
|
||||
func (s *Store) descriptorProposal(ctx context.Context, organizationID string, signal model.Signal, field string) (StoredProposal, error) {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil {
|
||||
return StoredProposal{}, errors.New("invalid organization identifier")
|
||||
}
|
||||
var descriptorJSON, examplesJSON, firstSeen, lastSeen string
|
||||
var proposal StoredProposal
|
||||
err := s.control.QueryRowContext(ctx, `SELECT descriptor_json,observed_values,estimated_bytes,example_queries_json,status,first_seen_at,last_seen_at FROM descriptor_proposals WHERE organization_id=? AND signal=? AND field=?`, organizationID, signal, query.CanonicalField(field)).Scan(&descriptorJSON, &proposal.Proposal.ObservedValues, &proposal.Proposal.EstimatedBytes, &examplesJSON, &proposal.Status, &firstSeen, &lastSeen)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return StoredProposal{}, errors.New("descriptor proposal not found")
|
||||
}
|
||||
if err != nil {
|
||||
return StoredProposal{}, errors.New("load descriptor proposal")
|
||||
}
|
||||
if err = json.Unmarshal([]byte(descriptorJSON), &proposal.Proposal.Descriptor); err != nil {
|
||||
return StoredProposal{}, errors.New("decode descriptor proposal")
|
||||
}
|
||||
if err = json.Unmarshal([]byte(examplesJSON), &proposal.Proposal.ExampleQueries); err != nil {
|
||||
return StoredProposal{}, errors.New("decode descriptor examples")
|
||||
}
|
||||
proposal.FirstSeenAt, err = time.Parse(time.RFC3339Nano, firstSeen)
|
||||
if err != nil {
|
||||
return StoredProposal{}, errors.New("descriptor proposal first-seen time is invalid")
|
||||
}
|
||||
proposal.LastSeenAt, err = time.Parse(time.RFC3339Nano, lastSeen)
|
||||
if err != nil || proposal.Proposal.Validate() != nil || proposal.Status != "pending" && proposal.Status != "activated" && proposal.Status != "rejected" {
|
||||
return StoredProposal{}, errors.New("stored descriptor proposal is invalid")
|
||||
}
|
||||
return proposal, nil
|
||||
}
|
||||
|
||||
func (s *Store) markProposalActivated(ctx context.Context, organizationID string, descriptor schema.Descriptor) error {
|
||||
encoded, err := json.Marshal(descriptor)
|
||||
if err != nil {
|
||||
return errors.New("encode activated descriptor")
|
||||
}
|
||||
result, err := s.control.ExecContext(ctx, `UPDATE descriptor_proposals SET descriptor_json=?,status='activated' WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, string(encoded), organizationID, descriptor.Signal, descriptor.Field)
|
||||
if err != nil {
|
||||
return errors.New("acknowledge activated descriptor")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed == 1 {
|
||||
return nil
|
||||
}
|
||||
proposal, err := s.descriptorProposal(ctx, organizationID, descriptor.Signal, descriptor.Field)
|
||||
if err != nil || proposal.Status != "activated" || !sameDescriptorIgnoringProjection(proposal.Proposal.Descriptor, descriptor) || proposal.Proposal.Descriptor.ProjectionVersion != descriptor.ProjectionVersion {
|
||||
return errors.New("activated descriptor acknowledgement is inconsistent")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) RejectDescriptorProposal(ctx context.Context, organizationID string, signal model.Signal, field string) error {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil {
|
||||
return errors.New("invalid organization identifier")
|
||||
}
|
||||
lock := s.namedLock("organization:" + organizationID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
registry, _, err := s.ActiveDescriptors(ctx, organizationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, active := registry.Lookup(signal, field); active {
|
||||
return errors.New("active descriptor proposal cannot be rejected")
|
||||
}
|
||||
result, err := s.control.ExecContext(ctx, `UPDATE descriptor_proposals SET status='rejected' WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, organizationID, signal, query.CanonicalField(field))
|
||||
if err != nil {
|
||||
return errors.New("reject descriptor proposal")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed == 1 {
|
||||
return nil
|
||||
}
|
||||
proposal, err := s.descriptorProposal(ctx, organizationID, signal, field)
|
||||
if err == nil && proposal.Status == "rejected" {
|
||||
return nil
|
||||
}
|
||||
return errors.New("pending descriptor proposal not found")
|
||||
}
|
||||
|
||||
func (s *Store) recordDescriptorProposals(ctx context.Context, organizationID string, batch model.Batch, segmentDigest string, now time.Time) error {
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err = recordDescriptorProposalsTx(ctx, tx, organizationID, batch, segmentDigest, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return errors.New("commit descriptor proposals")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func recordDescriptorProposalsTx(ctx context.Context, tx *sql.Tx, organizationID string, batch model.Batch, segmentDigest string, now time.Time) error {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil || !validDigest(segmentDigest) || now.IsZero() {
|
||||
return errors.New("descriptor proposal identity is invalid")
|
||||
}
|
||||
evidence := map[string]fieldEvidence{}
|
||||
for _, observation := range batch.Records {
|
||||
for field, value := range observation.Attributes {
|
||||
canonical := query.CanonicalField(field)
|
||||
if _, known := query.BuiltinDescriptor(batch.Signal, canonical); known {
|
||||
continue
|
||||
}
|
||||
current, exists := evidence[canonical]
|
||||
if !exists && len(evidence) >= model.MaxDistinctFields {
|
||||
return errors.New("descriptor proposal field limit exceeded")
|
||||
}
|
||||
descriptor := proposedDescriptor(batch.Signal, canonical, inferType(value))
|
||||
if err := descriptor.Validate(); err != nil {
|
||||
continue
|
||||
}
|
||||
if current.count > 0 {
|
||||
descriptor.Type = mergeType(current.descriptor.Type, descriptor.Type)
|
||||
}
|
||||
current.descriptor = descriptor
|
||||
current.count++
|
||||
current.bytes += int64(len(canonical) + len(value))
|
||||
evidence[canonical] = current
|
||||
}
|
||||
}
|
||||
if len(evidence) == 0 {
|
||||
return nil
|
||||
}
|
||||
fields := make([]string, 0, len(evidence))
|
||||
for field := range evidence {
|
||||
fields = append(fields, field)
|
||||
}
|
||||
sort.Strings(fields)
|
||||
for _, field := range fields {
|
||||
observed := evidence[field]
|
||||
result, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO descriptor_proposal_segments(segment_digest,organization_id,signal,field,observed_values,estimated_bytes) VALUES(?,?,?,?,?,?)`, segmentDigest, organizationID, batch.Signal, field, observed.count, observed.bytes)
|
||||
if err != nil {
|
||||
return errors.New("record descriptor proposal segment")
|
||||
}
|
||||
inserted, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return errors.New("inspect descriptor proposal segment")
|
||||
}
|
||||
if inserted == 0 {
|
||||
continue
|
||||
}
|
||||
if err = upsertProposal(ctx, tx, organizationID, batch.Signal, observed, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func upsertProposal(ctx context.Context, tx *sql.Tx, organizationID string, signal model.Signal, observed fieldEvidence, now time.Time) error {
|
||||
field := observed.descriptor.Field
|
||||
var descriptorJSON, examplesJSON, status string
|
||||
var count, estimated int64
|
||||
err := tx.QueryRowContext(ctx, `SELECT descriptor_json,observed_values,estimated_bytes,example_queries_json,status FROM descriptor_proposals WHERE organization_id=? AND signal=? AND field=?`, organizationID, signal, field).Scan(&descriptorJSON, &count, &estimated, &examplesJSON, &status)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
descriptorBody, marshalErr := json.Marshal(observed.descriptor)
|
||||
if marshalErr != nil {
|
||||
return errors.New("encode descriptor proposal")
|
||||
}
|
||||
examples := []string{fmt.Sprintf(`%s | where %s == "value" | limit 50`, signal, field)}
|
||||
exampleBody, marshalErr := json.Marshal(examples)
|
||||
if marshalErr != nil {
|
||||
return errors.New("encode descriptor examples")
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO descriptor_proposals(organization_id,signal,field,descriptor_json,observed_values,estimated_bytes,example_queries_json,status,first_seen_at,last_seen_at) VALUES(?,?,?,?,?,?,?,'pending',?,?)`, organizationID, signal, field, string(descriptorBody), observed.count, observed.bytes, string(exampleBody), now.UTC().Format(time.RFC3339Nano), now.UTC().Format(time.RFC3339Nano))
|
||||
if err != nil {
|
||||
return errors.New("create descriptor proposal")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.New("load descriptor proposal")
|
||||
}
|
||||
if status != "pending" {
|
||||
return nil
|
||||
}
|
||||
var descriptor schema.Descriptor
|
||||
if err = json.Unmarshal([]byte(descriptorJSON), &descriptor); err != nil || descriptor.Validate() != nil {
|
||||
return errors.New("stored descriptor proposal is invalid")
|
||||
}
|
||||
descriptor.Type = mergeType(descriptor.Type, observed.descriptor.Type)
|
||||
if count > math.MaxInt64-observed.count || estimated > math.MaxInt64-observed.bytes {
|
||||
return errors.New("descriptor proposal evidence overflow")
|
||||
}
|
||||
descriptorBody, err := json.Marshal(descriptor)
|
||||
if err != nil {
|
||||
return errors.New("encode descriptor proposal")
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE descriptor_proposals SET descriptor_json=?,observed_values=?,estimated_bytes=?,last_seen_at=? WHERE organization_id=? AND signal=? AND field=? AND status='pending'`, string(descriptorBody), count+observed.count, estimated+observed.bytes, now.UTC().Format(time.RFC3339Nano), organizationID, signal, field)
|
||||
if err != nil {
|
||||
return errors.New("update descriptor proposal")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) DescriptorProposals(ctx context.Context, organizationID string) ([]StoredProposal, error) {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil {
|
||||
return nil, errors.New("invalid organization identifier")
|
||||
}
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT descriptor_json,observed_values,estimated_bytes,example_queries_json,status,first_seen_at,last_seen_at FROM descriptor_proposals WHERE organization_id=? ORDER BY signal,field`, organizationID)
|
||||
if err != nil {
|
||||
return nil, errors.New("list descriptor proposals")
|
||||
}
|
||||
defer rows.Close()
|
||||
var proposals []StoredProposal
|
||||
for rows.Next() {
|
||||
var descriptorJSON, examplesJSON, firstSeen, lastSeen string
|
||||
var proposal StoredProposal
|
||||
if err = rows.Scan(&descriptorJSON, &proposal.Proposal.ObservedValues, &proposal.Proposal.EstimatedBytes, &examplesJSON, &proposal.Status, &firstSeen, &lastSeen); err != nil {
|
||||
return nil, errors.New("read descriptor proposal")
|
||||
}
|
||||
if err = json.Unmarshal([]byte(descriptorJSON), &proposal.Proposal.Descriptor); err != nil {
|
||||
return nil, errors.New("decode descriptor proposal")
|
||||
}
|
||||
if err = json.Unmarshal([]byte(examplesJSON), &proposal.Proposal.ExampleQueries); err != nil {
|
||||
return nil, errors.New("decode descriptor examples")
|
||||
}
|
||||
proposal.FirstSeenAt, err = time.Parse(time.RFC3339Nano, firstSeen)
|
||||
if err != nil {
|
||||
return nil, errors.New("descriptor proposal first-seen time is invalid")
|
||||
}
|
||||
proposal.LastSeenAt, err = time.Parse(time.RFC3339Nano, lastSeen)
|
||||
if err != nil || proposal.Proposal.Validate() != nil || proposal.Status != "pending" && proposal.Status != "activated" && proposal.Status != "rejected" {
|
||||
return nil, errors.New("stored descriptor proposal is invalid")
|
||||
}
|
||||
proposals = append(proposals, proposal)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, errors.New("list descriptor proposals")
|
||||
}
|
||||
return proposals, nil
|
||||
}
|
||||
|
||||
func proposedDescriptor(signal model.Signal, field string, valueType schema.Type) schema.Descriptor {
|
||||
return schema.Descriptor{
|
||||
Version: schema.DescriptorVersion, Signal: signal, Field: field,
|
||||
Type: valueType, Meaning: "Observed unreviewed field awaiting administrator classification.",
|
||||
Sensitivity: schema.SensitivitySensitive, Cardinality: schema.CardinalityHigh,
|
||||
Index: schema.IndexNone, Retention: schema.RetentionRaw, ProjectionVersion: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func inferType(value string) schema.Type {
|
||||
if _, err := strconv.ParseInt(value, 10, 64); err == nil {
|
||||
return schema.TypeInteger
|
||||
}
|
||||
if number, err := strconv.ParseFloat(value, 64); err == nil && !math.IsNaN(number) && !math.IsInf(number, 0) {
|
||||
return schema.TypeFloat
|
||||
}
|
||||
if value == "true" || value == "false" {
|
||||
return schema.TypeBoolean
|
||||
}
|
||||
if _, err := time.Parse(time.RFC3339Nano, value); err == nil {
|
||||
return schema.TypeTime
|
||||
}
|
||||
return schema.TypeString
|
||||
}
|
||||
|
||||
func mergeType(left, right schema.Type) schema.Type {
|
||||
if left == right {
|
||||
return left
|
||||
}
|
||||
if left == schema.TypeInteger && right == schema.TypeFloat || left == schema.TypeFloat && right == schema.TypeInteger {
|
||||
return schema.TypeFloat
|
||||
}
|
||||
return schema.TypeString
|
||||
}
|
||||
|
||||
func validDigest(value string) bool {
|
||||
if len(value) != 64 {
|
||||
return false
|
||||
}
|
||||
for _, character := range value {
|
||||
if !strings.ContainsRune("0123456789abcdef", character) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
func TestDescriptorProposalsAreSegmentIdempotentAndDefaultSensitive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 7, 30, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{
|
||||
{Timestamp: now, Name: "custom.metric", Value: floatPointer(1), Attributes: map[string]string{"workshop.queue_depth": "9238471928374", "http.route": "/known", "invalid-field": "ignored"}},
|
||||
{Timestamp: now, Name: "custom.metric", Value: floatPointer(2), Attributes: map[string]string{"workshop.queue_depth": "9238471928375"}},
|
||||
}}
|
||||
ack, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
proposals, err := store.DescriptorProposals(ctx, "organization-a")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(proposals) != 1 {
|
||||
t.Fatalf("proposals=%+v", proposals)
|
||||
}
|
||||
proposal := proposals[0]
|
||||
if proposal.Proposal.Descriptor.Field != "workshop.queue_depth" || proposal.Proposal.Descriptor.Type != schema.TypeInteger || proposal.Proposal.Descriptor.Sensitivity != schema.SensitivitySensitive || proposal.Proposal.Descriptor.Index != schema.IndexNone || proposal.Proposal.ObservedValues != 2 || proposal.Status != "pending" {
|
||||
t.Fatalf("proposal=%+v", proposal)
|
||||
}
|
||||
if len(proposal.Proposal.ExampleQueries) != 1 || proposal.Proposal.ExampleQueries[0] != `metrics | where workshop.queue_depth == "value" | limit 50` {
|
||||
t.Fatalf("examples=%v", proposal.Proposal.ExampleQueries)
|
||||
}
|
||||
encoded, err := json.Marshal(proposals)
|
||||
if err != nil || strings.Contains(string(encoded), "9238471928374") || strings.Contains(string(encoded), "9238471928375") {
|
||||
t.Fatalf("observed values entered proposal metadata: %s err=%v", encoded, err)
|
||||
}
|
||||
if err = store.recordDescriptorProposals(ctx, "organization-a", batch, ack.Digest, now.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proposals, err = store.DescriptorProposals(ctx, "organization-a")
|
||||
if err != nil || proposals[0].Proposal.ObservedValues != 2 || !proposals[0].LastSeenAt.Equal(now) {
|
||||
t.Fatalf("idempotent proposals=%+v err=%v", proposals, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptorProposalTypesWidenWithoutMixingOrganizations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 7, 30, 0, 0, time.UTC)
|
||||
for index, organization := range []string{"organization-a", "organization-b"} {
|
||||
source := "source-" + string(rune('a'+index))
|
||||
token, err := store.CreateSource(ctx, source, model.Scope{OrganizationID: organization, ProjectID: "project", EnvironmentID: "production", ServiceID: "service"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value := "10"
|
||||
if organization == "organization-b" {
|
||||
value = "label"
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: source, StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: now, Name: "custom.metric", Value: floatPointer(1), Attributes: map[string]string{"workshop.value": value}}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if organization == "organization-a" {
|
||||
batch.Sequence = 2
|
||||
batch.Records[0].Attributes["workshop.value"] = "10.5"
|
||||
if _, err = store.Ingest(ctx, token, batch, now.Add(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
projectAll(t, store)
|
||||
proposalA, err := store.DescriptorProposals(ctx, "organization-a")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proposalB, err := store.DescriptorProposals(ctx, "organization-b")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if proposalA[0].Proposal.Descriptor.Type != schema.TypeFloat || proposalB[0].Proposal.Descriptor.Type != schema.TypeString {
|
||||
t.Fatalf("organization-a=%+v organization-b=%+v", proposalA, proposalB)
|
||||
}
|
||||
}
|
||||
|
||||
func floatPointer(value float64) *float64 { return &value }
|
||||
@@ -0,0 +1,600 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
const (
|
||||
AlertRuleVersion = 1
|
||||
IncidentVersion = 1
|
||||
MaxDueRules = 64
|
||||
)
|
||||
|
||||
type AlertRule struct {
|
||||
Version int `json:"version"`
|
||||
Revision int `json:"revision"`
|
||||
OrganizationID string `json:"organization_id"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
SavedQueryID string `json:"saved_query_id"`
|
||||
Severity string `json:"severity"`
|
||||
MinimumMatches int `json:"minimum_matches"`
|
||||
RequiredConsecutive int `json:"required_consecutive"`
|
||||
EvaluationInterval time.Duration `json:"evaluation_interval"`
|
||||
Enabled bool `json:"enabled"`
|
||||
LastEvaluatedAt *time.Time `json:"last_evaluated_at,omitempty"`
|
||||
NextEvaluationAt time.Time `json:"next_evaluation_at"`
|
||||
LastResult *int `json:"last_result,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type AlertRuleInput struct {
|
||||
ID string
|
||||
ExpectedRevision int
|
||||
OrganizationID string
|
||||
Name string
|
||||
Description string
|
||||
SavedQueryID string
|
||||
Severity string
|
||||
MinimumMatches int
|
||||
RequiredConsecutive int
|
||||
EvaluationInterval time.Duration
|
||||
Enabled bool
|
||||
ActorUserID string
|
||||
}
|
||||
|
||||
type Incident struct {
|
||||
Version int `json:"version"`
|
||||
OrganizationID string `json:"organization_id"`
|
||||
ID string `json:"id"`
|
||||
RuleID string `json:"rule_id"`
|
||||
State string `json:"state"`
|
||||
Severity string `json:"severity"`
|
||||
Title string `json:"title"`
|
||||
ConsecutiveMatches int `json:"consecutive_matches"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
LastObservedAt time.Time `json:"last_observed_at"`
|
||||
AcknowledgedBy string `json:"acknowledged_by,omitempty"`
|
||||
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
|
||||
SilencedBy string `json:"silenced_by,omitempty"`
|
||||
SilencedUntil *time.Time `json:"silenced_until,omitempty"`
|
||||
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type IncidentEvent struct {
|
||||
Sequence int `json:"sequence"`
|
||||
Event string `json:"event"`
|
||||
Actor string `json:"actor"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type AlertEvaluation struct {
|
||||
RuleID string `json:"rule_id"`
|
||||
OrganizationID string `json:"organization_id"`
|
||||
Matched bool `json:"matched"`
|
||||
Rows int `json:"rows"`
|
||||
IncidentID string `json:"incident_id,omitempty"`
|
||||
IncidentState string `json:"incident_state,omitempty"`
|
||||
IncidentChanged bool `json:"incident_changed"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Store) SaveAlertRule(ctx context.Context, input AlertRuleInput, now time.Time) (AlertRule, error) {
|
||||
if err := validateAlertRuleInput(input, now); err != nil {
|
||||
return AlertRule{}, err
|
||||
}
|
||||
if _, err := s.SavedQuery(ctx, input.OrganizationID, input.SavedQueryID); err != nil {
|
||||
return AlertRule{}, errors.New("alert rule saved query is unavailable")
|
||||
}
|
||||
var err error
|
||||
if input.ID == "" {
|
||||
input.ID, err = storageID("rule")
|
||||
if err != nil {
|
||||
return AlertRule{}, err
|
||||
}
|
||||
}
|
||||
timestamp := now.UTC().Format(time.RFC3339Nano)
|
||||
if input.ExpectedRevision == 0 {
|
||||
_, err = s.control.ExecContext(ctx, `INSERT INTO alert_rules(organization_id,id,version,revision,name,description,saved_query_id,severity,minimum_matches,required_consecutive,evaluation_interval_seconds,enabled,next_evaluation_at,created_by,updated_by,created_at,updated_at) VALUES(?,?,1,1,?,?,?,?,?,?,?,?,?,?,?,?,?)`, input.OrganizationID, input.ID, input.Name, input.Description, input.SavedQueryID, input.Severity, input.MinimumMatches, input.RequiredConsecutive, int(input.EvaluationInterval/time.Second), boolInt(input.Enabled), timestamp, input.ActorUserID, input.ActorUserID, timestamp, timestamp)
|
||||
} else {
|
||||
if model.ValidateSourceID(input.ID) != nil || input.ExpectedRevision < 1 {
|
||||
return AlertRule{}, errors.New("alert rule revision input is invalid")
|
||||
}
|
||||
var result sql.Result
|
||||
result, err = s.control.ExecContext(ctx, `UPDATE alert_rules SET revision=revision+1,name=?,description=?,saved_query_id=?,severity=?,minimum_matches=?,required_consecutive=?,evaluation_interval_seconds=?,enabled=?,next_evaluation_at=?,updated_by=?,updated_at=? WHERE organization_id=? AND id=? AND revision=?`, input.Name, input.Description, input.SavedQueryID, input.Severity, input.MinimumMatches, input.RequiredConsecutive, int(input.EvaluationInterval/time.Second), boolInt(input.Enabled), timestamp, input.ActorUserID, timestamp, input.OrganizationID, input.ID, input.ExpectedRevision)
|
||||
if err == nil {
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return AlertRule{}, errors.New("alert rule revision conflict")
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return AlertRule{}, errors.New("save alert rule")
|
||||
}
|
||||
return s.AlertRule(ctx, input.OrganizationID, input.ID)
|
||||
}
|
||||
|
||||
func (s *Store) AlertRule(ctx context.Context, organizationID, id string) (AlertRule, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(id) != nil {
|
||||
return AlertRule{}, errors.New("alert rule identity is invalid")
|
||||
}
|
||||
row := s.control.QueryRowContext(ctx, `SELECT version,revision,name,description,saved_query_id,severity,minimum_matches,required_consecutive,evaluation_interval_seconds,enabled,last_evaluated_at,next_evaluation_at,last_result,last_error,created_by,updated_by,created_at,updated_at FROM alert_rules WHERE organization_id=? AND id=?`, organizationID, id)
|
||||
return scanAlertRule(row, organizationID, id)
|
||||
}
|
||||
|
||||
func (s *Store) AlertRules(ctx context.Context, organizationID string) ([]AlertRule, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil {
|
||||
return nil, errors.New("invalid organization identifier")
|
||||
}
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT id FROM alert_rules WHERE organization_id=? ORDER BY name,id`, organizationID)
|
||||
if err != nil {
|
||||
return nil, errors.New("list alert rules")
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err = rows.Scan(&id); err != nil {
|
||||
return nil, errors.New("list alert rules")
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, errors.New("list alert rules")
|
||||
}
|
||||
result := make([]AlertRule, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
value, loadErr := s.AlertRule(ctx, organizationID, id)
|
||||
if loadErr != nil {
|
||||
return nil, loadErr
|
||||
}
|
||||
result = append(result, value)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) EvaluateDueAlertRules(ctx context.Context, budget query.Budget, now time.Time) ([]AlertEvaluation, error) {
|
||||
if now.IsZero() {
|
||||
return nil, errors.New("alert evaluation time is required")
|
||||
}
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT organization_id,id FROM alert_rules WHERE enabled=1 AND next_evaluation_at<=? ORDER BY next_evaluation_at,organization_id,id LIMIT ?`, now.UTC().Format(time.RFC3339Nano), MaxDueRules)
|
||||
if err != nil {
|
||||
return nil, errors.New("list due alert rules")
|
||||
}
|
||||
type identity struct{ organizationID, ruleID string }
|
||||
var identities []identity
|
||||
for rows.Next() {
|
||||
var item identity
|
||||
if err = rows.Scan(&item.organizationID, &item.ruleID); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, errors.New("list due alert rules")
|
||||
}
|
||||
identities = append(identities, item)
|
||||
}
|
||||
if err = rows.Close(); err != nil || rows.Err() != nil {
|
||||
return nil, errors.New("list due alert rules")
|
||||
}
|
||||
result := make([]AlertEvaluation, 0, len(identities))
|
||||
for _, item := range identities {
|
||||
evaluation, evaluated, evaluationErr := s.evaluateAlertRule(ctx, item.organizationID, item.ruleID, budget, now.UTC())
|
||||
if evaluationErr != nil {
|
||||
return result, evaluationErr
|
||||
}
|
||||
if evaluated {
|
||||
result = append(result, evaluation)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) evaluateAlertRule(ctx context.Context, organizationID, ruleID string, budget query.Budget, now time.Time) (AlertEvaluation, bool, error) {
|
||||
rule, err := s.AlertRule(ctx, organizationID, ruleID)
|
||||
if err != nil {
|
||||
return AlertEvaluation{}, false, err
|
||||
}
|
||||
claim, err := s.control.ExecContext(ctx, `UPDATE alert_rules SET next_evaluation_at=? WHERE organization_id=? AND id=? AND enabled=1 AND next_evaluation_at<=?`, now.Add(rule.EvaluationInterval).Format(time.RFC3339Nano), organizationID, ruleID, now.Format(time.RFC3339Nano))
|
||||
if err != nil {
|
||||
return AlertEvaluation{}, false, errors.New("claim alert rule evaluation")
|
||||
}
|
||||
if changed, _ := claim.RowsAffected(); changed != 1 {
|
||||
return AlertEvaluation{}, false, nil
|
||||
}
|
||||
saved, err := s.SavedQuery(ctx, organizationID, rule.SavedQueryID)
|
||||
if err != nil {
|
||||
return AlertEvaluation{RuleID: rule.ID, OrganizationID: organizationID, Error: "query_unavailable"}, true, s.recordRuleError(ctx, rule, now)
|
||||
}
|
||||
result, err := s.Query(ctx, saved.AST, query.Scope{OrganizationID: organizationID, ProjectID: saved.Scope.ProjectID, EnvironmentID: saved.Scope.EnvironmentID, ServiceID: saved.Scope.ServiceID}, budget, now)
|
||||
if err != nil {
|
||||
return AlertEvaluation{RuleID: rule.ID, OrganizationID: organizationID, Error: "query_unavailable"}, true, s.recordRuleError(ctx, rule, now)
|
||||
}
|
||||
matched := len(result.Rows) >= rule.MinimumMatches
|
||||
incident, changed, err := s.recordAlertResult(ctx, rule, len(result.Rows), matched, now)
|
||||
if err != nil {
|
||||
return AlertEvaluation{}, true, err
|
||||
}
|
||||
evaluation := AlertEvaluation{RuleID: rule.ID, OrganizationID: organizationID, Matched: matched, Rows: len(result.Rows)}
|
||||
if incident.ID != "" {
|
||||
evaluation.IncidentID, evaluation.IncidentState = incident.ID, incident.State
|
||||
}
|
||||
evaluation.IncidentChanged = changed
|
||||
return evaluation, true, nil
|
||||
}
|
||||
|
||||
func (s *Store) recordRuleError(ctx context.Context, rule AlertRule, now time.Time) error {
|
||||
_, err := s.control.ExecContext(ctx, `UPDATE alert_rules SET last_evaluated_at=?,last_result=NULL,last_error='query_unavailable' WHERE organization_id=? AND id=?`, now.Format(time.RFC3339Nano), rule.OrganizationID, rule.ID)
|
||||
if err != nil {
|
||||
return errors.New("record alert rule failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) recordAlertResult(ctx context.Context, rule AlertRule, rows int, matched bool, now time.Time) (Incident, bool, error) {
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return Incident{}, false, errors.New("begin alert evaluation")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE alert_rules SET last_evaluated_at=?,last_result=?,last_error='' WHERE organization_id=? AND id=?`, now.Format(time.RFC3339Nano), rows, rule.OrganizationID, rule.ID); err != nil {
|
||||
return Incident{}, false, errors.New("record alert evaluation")
|
||||
}
|
||||
incident, found, err := openIncidentTx(ctx, tx, rule.OrganizationID, rule.ID)
|
||||
if err != nil {
|
||||
return Incident{}, false, err
|
||||
}
|
||||
changed := false
|
||||
if !matched {
|
||||
if !found {
|
||||
if err = tx.Commit(); err != nil {
|
||||
return Incident{}, false, errors.New("commit alert evaluation")
|
||||
}
|
||||
return Incident{}, false, nil
|
||||
}
|
||||
if err = resolveIncidentTx(ctx, tx, &incident, "system", now); err != nil {
|
||||
return Incident{}, false, err
|
||||
}
|
||||
changed = true
|
||||
} else if !found {
|
||||
incidentID, idErr := storageID("incident")
|
||||
if idErr != nil {
|
||||
return Incident{}, false, idErr
|
||||
}
|
||||
state := "pending"
|
||||
if rule.RequiredConsecutive == 1 {
|
||||
state = "firing"
|
||||
}
|
||||
stamp := now.Format(time.RFC3339Nano)
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO incidents(organization_id,id,version,rule_id,state,severity,title,consecutive_matches,started_at,last_observed_at,updated_at) VALUES(?,?,1,?,?,?,?,1,?,?,?)`, rule.OrganizationID, incidentID, rule.ID, state, rule.Severity, rule.Name, stamp, stamp, stamp)
|
||||
if err != nil {
|
||||
return Incident{}, false, errors.New("open incident")
|
||||
}
|
||||
incident = Incident{Version: IncidentVersion, OrganizationID: rule.OrganizationID, ID: incidentID, RuleID: rule.ID, State: state, Severity: rule.Severity, Title: rule.Name, ConsecutiveMatches: 1, StartedAt: now, LastObservedAt: now, UpdatedAt: now}
|
||||
if err = appendIncidentEventTx(ctx, tx, incident, "opened", "system", now); err != nil {
|
||||
return Incident{}, false, err
|
||||
}
|
||||
changed = true
|
||||
} else {
|
||||
incident.ConsecutiveMatches++
|
||||
incident.LastObservedAt, incident.UpdatedAt = now, now
|
||||
event := ""
|
||||
if incident.State == "pending" && incident.ConsecutiveMatches >= rule.RequiredConsecutive {
|
||||
incident.State, event = "firing", "promoted"
|
||||
} else if incident.State == "silenced" && incident.SilencedUntil != nil && !now.Before(*incident.SilencedUntil) {
|
||||
incident.State, incident.SilencedBy, incident.SilencedUntil, event = "firing", "", nil, "unsilenced"
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE incidents SET state=?,consecutive_matches=?,last_observed_at=?,silenced_by=?,silenced_until=?,updated_at=? WHERE organization_id=? AND id=?`, incident.State, incident.ConsecutiveMatches, now.Format(time.RFC3339Nano), nullableText(incident.SilencedBy), nullableTime(incident.SilencedUntil), now.Format(time.RFC3339Nano), incident.OrganizationID, incident.ID)
|
||||
if err != nil {
|
||||
return Incident{}, false, errors.New("update incident observation")
|
||||
}
|
||||
if event != "" {
|
||||
if err = appendIncidentEventTx(ctx, tx, incident, event, "system", now); err != nil {
|
||||
return Incident{}, false, err
|
||||
}
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return Incident{}, false, errors.New("commit alert evaluation")
|
||||
}
|
||||
return incident, changed, nil
|
||||
}
|
||||
|
||||
func (s *Store) Incidents(ctx context.Context, organizationID string, includeResolved bool, limit int) ([]Incident, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || limit < 1 || limit > 1000 {
|
||||
return nil, errors.New("incident list input is invalid")
|
||||
}
|
||||
statement := `SELECT id,version,rule_id,state,severity,title,consecutive_matches,started_at,last_observed_at,acknowledged_by,acknowledged_at,silenced_by,silenced_until,resolved_at,updated_at FROM incidents WHERE organization_id=?`
|
||||
if !includeResolved {
|
||||
statement += ` AND state!='resolved'`
|
||||
}
|
||||
statement += ` ORDER BY CASE state WHEN 'firing' THEN 0 WHEN 'pending' THEN 1 WHEN 'acknowledged' THEN 2 WHEN 'silenced' THEN 3 ELSE 4 END,updated_at DESC,id LIMIT ?`
|
||||
rows, err := s.control.QueryContext(ctx, statement, organizationID, limit)
|
||||
if err != nil {
|
||||
return nil, errors.New("list incidents")
|
||||
}
|
||||
defer rows.Close()
|
||||
var result []Incident
|
||||
for rows.Next() {
|
||||
value, scanErr := scanIncident(rows, organizationID)
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
result = append(result, value)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, errors.New("list incidents")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) TransitionIncident(ctx context.Context, organizationID, incidentID, action, actor string, silenceUntil *time.Time, now time.Time) (Incident, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(incidentID) != nil || model.ValidateSourceID(actor) != nil || now.IsZero() {
|
||||
return Incident{}, errors.New("incident transition input is invalid")
|
||||
}
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return Incident{}, errors.New("begin incident transition")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
incident, err := incidentByIDTx(ctx, tx, organizationID, incidentID)
|
||||
if err != nil {
|
||||
return Incident{}, err
|
||||
}
|
||||
if incident.State == "resolved" {
|
||||
return Incident{}, errors.New("resolved incident cannot transition")
|
||||
}
|
||||
switch action {
|
||||
case "acknowledge":
|
||||
incident.State, incident.AcknowledgedBy = "acknowledged", actor
|
||||
incident.SilencedBy, incident.SilencedUntil = "", nil
|
||||
stamp := now.UTC()
|
||||
incident.AcknowledgedAt = &stamp
|
||||
if err = appendIncidentEventTx(ctx, tx, incident, "acknowledged", actor, stamp); err != nil {
|
||||
return Incident{}, err
|
||||
}
|
||||
case "silence":
|
||||
if silenceUntil == nil || !silenceUntil.After(now) || silenceUntil.After(now.Add(30*24*time.Hour)) {
|
||||
return Incident{}, errors.New("incident silence expiry is invalid")
|
||||
}
|
||||
stamp := silenceUntil.UTC()
|
||||
incident.State, incident.SilencedBy, incident.SilencedUntil = "silenced", actor, &stamp
|
||||
if err = appendIncidentEventTx(ctx, tx, incident, "silenced", actor, now.UTC()); err != nil {
|
||||
return Incident{}, err
|
||||
}
|
||||
case "resolve":
|
||||
if err = resolveIncidentTx(ctx, tx, &incident, actor, now.UTC()); err != nil {
|
||||
return Incident{}, err
|
||||
}
|
||||
default:
|
||||
return Incident{}, errors.New("incident action is invalid")
|
||||
}
|
||||
incident.UpdatedAt = now.UTC()
|
||||
if action != "resolve" {
|
||||
_, err = tx.ExecContext(ctx, `UPDATE incidents SET state=?,acknowledged_by=?,acknowledged_at=?,silenced_by=?,silenced_until=?,updated_at=? WHERE organization_id=? AND id=?`, incident.State, nullableText(incident.AcknowledgedBy), nullableTime(incident.AcknowledgedAt), nullableText(incident.SilencedBy), nullableTime(incident.SilencedUntil), incident.UpdatedAt.Format(time.RFC3339Nano), organizationID, incidentID)
|
||||
if err != nil {
|
||||
return Incident{}, errors.New("update incident")
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return Incident{}, errors.New("commit incident transition")
|
||||
}
|
||||
return incident, nil
|
||||
}
|
||||
|
||||
func (s *Store) IncidentEvents(ctx context.Context, organizationID, incidentID string) ([]IncidentEvent, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(incidentID) != nil {
|
||||
return nil, errors.New("incident event identity is invalid")
|
||||
}
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT sequence,event,actor,created_at FROM incident_events WHERE organization_id=? AND incident_id=? ORDER BY sequence`, organizationID, incidentID)
|
||||
if err != nil {
|
||||
return nil, errors.New("list incident events")
|
||||
}
|
||||
defer rows.Close()
|
||||
var result []IncidentEvent
|
||||
for rows.Next() {
|
||||
var value IncidentEvent
|
||||
var created string
|
||||
if err = rows.Scan(&value.Sequence, &value.Event, &value.Actor, &created); err != nil {
|
||||
return nil, errors.New("read incident event")
|
||||
}
|
||||
value.CreatedAt, err = time.Parse(time.RFC3339Nano, created)
|
||||
if err != nil {
|
||||
return nil, errors.New("stored incident event is invalid")
|
||||
}
|
||||
result = append(result, value)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
type alertRuleScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanAlertRule(row alertRuleScanner, organizationID, id string) (AlertRule, error) {
|
||||
value := AlertRule{OrganizationID: organizationID, ID: id}
|
||||
var enabled int
|
||||
var interval int64
|
||||
var lastEvaluated, nextEvaluation, created, updated sql.NullString
|
||||
var lastResult sql.NullInt64
|
||||
if err := row.Scan(&value.Version, &value.Revision, &value.Name, &value.Description, &value.SavedQueryID, &value.Severity, &value.MinimumMatches, &value.RequiredConsecutive, &interval, &enabled, &lastEvaluated, &nextEvaluation, &lastResult, &value.LastError, &value.CreatedBy, &value.UpdatedBy, &created, &updated); errors.Is(err, sql.ErrNoRows) {
|
||||
return AlertRule{}, errors.New("alert rule not found")
|
||||
} else if err != nil {
|
||||
return AlertRule{}, errors.New("read alert rule")
|
||||
}
|
||||
value.Enabled = enabled == 1
|
||||
value.EvaluationInterval = time.Duration(interval) * time.Second
|
||||
if lastResult.Valid {
|
||||
result := int(lastResult.Int64)
|
||||
value.LastResult = &result
|
||||
}
|
||||
var err error
|
||||
if lastEvaluated.Valid {
|
||||
parsed, parseErr := time.Parse(time.RFC3339Nano, lastEvaluated.String)
|
||||
if parseErr != nil {
|
||||
return AlertRule{}, errors.New("stored alert rule evaluation time is invalid")
|
||||
}
|
||||
value.LastEvaluatedAt = &parsed
|
||||
}
|
||||
value.NextEvaluationAt, err = time.Parse(time.RFC3339Nano, nextEvaluation.String)
|
||||
if err == nil {
|
||||
value.CreatedAt, err = time.Parse(time.RFC3339Nano, created.String)
|
||||
}
|
||||
if err == nil {
|
||||
value.UpdatedAt, err = time.Parse(time.RFC3339Nano, updated.String)
|
||||
}
|
||||
if err != nil || validateAlertRule(value) != nil {
|
||||
return AlertRule{}, errors.New("stored alert rule is invalid")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
type incidentScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanIncident(row incidentScanner, organizationID string) (Incident, error) {
|
||||
value := Incident{OrganizationID: organizationID}
|
||||
var started, observed, updated string
|
||||
var acknowledgedBy, acknowledgedAt, silencedBy, silencedUntil, resolvedAt sql.NullString
|
||||
if err := row.Scan(&value.ID, &value.Version, &value.RuleID, &value.State, &value.Severity, &value.Title, &value.ConsecutiveMatches, &started, &observed, &acknowledgedBy, &acknowledgedAt, &silencedBy, &silencedUntil, &resolvedAt, &updated); errors.Is(err, sql.ErrNoRows) {
|
||||
return Incident{}, sql.ErrNoRows
|
||||
} else if err != nil {
|
||||
return Incident{}, errors.New("read incident")
|
||||
}
|
||||
value.AcknowledgedBy, value.SilencedBy = acknowledgedBy.String, silencedBy.String
|
||||
var err error
|
||||
value.StartedAt, err = time.Parse(time.RFC3339Nano, started)
|
||||
if err == nil {
|
||||
value.LastObservedAt, err = time.Parse(time.RFC3339Nano, observed)
|
||||
}
|
||||
if err == nil {
|
||||
value.UpdatedAt, err = time.Parse(time.RFC3339Nano, updated)
|
||||
}
|
||||
if err == nil {
|
||||
value.AcknowledgedAt, err = parseOptionalTime(acknowledgedAt)
|
||||
}
|
||||
if err == nil {
|
||||
value.SilencedUntil, err = parseOptionalTime(silencedUntil)
|
||||
}
|
||||
if err == nil {
|
||||
value.ResolvedAt, err = parseOptionalTime(resolvedAt)
|
||||
}
|
||||
if err != nil || validateIncident(value) != nil {
|
||||
return Incident{}, errors.New("stored incident is invalid")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func openIncidentTx(ctx context.Context, tx *sql.Tx, organizationID, ruleID string) (Incident, bool, error) {
|
||||
row := tx.QueryRowContext(ctx, `SELECT id,version,rule_id,state,severity,title,consecutive_matches,started_at,last_observed_at,acknowledged_by,acknowledged_at,silenced_by,silenced_until,resolved_at,updated_at FROM incidents WHERE organization_id=? AND rule_id=? AND state!='resolved'`, organizationID, ruleID)
|
||||
value, err := scanIncident(row, organizationID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Incident{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Incident{}, false, err
|
||||
}
|
||||
return value, true, nil
|
||||
}
|
||||
|
||||
func incidentByIDTx(ctx context.Context, tx *sql.Tx, organizationID, incidentID string) (Incident, error) {
|
||||
row := tx.QueryRowContext(ctx, `SELECT id,version,rule_id,state,severity,title,consecutive_matches,started_at,last_observed_at,acknowledged_by,acknowledged_at,silenced_by,silenced_until,resolved_at,updated_at FROM incidents WHERE organization_id=? AND id=?`, organizationID, incidentID)
|
||||
value, err := scanIncident(row, organizationID)
|
||||
if err != nil {
|
||||
return Incident{}, errors.New("incident not found")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func resolveIncidentTx(ctx context.Context, tx *sql.Tx, incident *Incident, actor string, now time.Time) error {
|
||||
incident.State, incident.ResolvedAt, incident.UpdatedAt = "resolved", &now, now
|
||||
_, err := tx.ExecContext(ctx, `UPDATE incidents SET state='resolved',resolved_at=?,updated_at=? WHERE organization_id=? AND id=?`, now.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano), incident.OrganizationID, incident.ID)
|
||||
if err != nil {
|
||||
return errors.New("resolve incident")
|
||||
}
|
||||
return appendIncidentEventTx(ctx, tx, *incident, "resolved", actor, now)
|
||||
}
|
||||
|
||||
func appendIncidentEventTx(ctx context.Context, tx *sql.Tx, incident Incident, event, actor string, now time.Time) error {
|
||||
var sequence int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(sequence),0)+1 FROM incident_events WHERE organization_id=? AND incident_id=?`, incident.OrganizationID, incident.ID).Scan(&sequence); err != nil {
|
||||
return errors.New("sequence incident event")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO incident_events(organization_id,incident_id,sequence,event,actor,created_at) VALUES(?,?,?,?,?,?)`, incident.OrganizationID, incident.ID, sequence, event, actor, now.UTC().Format(time.RFC3339Nano)); err != nil {
|
||||
return errors.New("append incident event")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAlertRuleInput(input AlertRuleInput, now time.Time) error {
|
||||
if model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.ActorUserID) != nil || model.ValidateSourceID(input.SavedQueryID) != nil || !boundedText(input.Name, 128, false) || !boundedText(input.Description, 1024, true) || input.MinimumMatches < 1 || input.MinimumMatches > 100_000 || input.RequiredConsecutive < 1 || input.RequiredConsecutive > 10 || input.EvaluationInterval < 15*time.Second || input.EvaluationInterval > 24*time.Hour || input.EvaluationInterval%time.Second != 0 || now.IsZero() {
|
||||
return errors.New("alert rule input is invalid")
|
||||
}
|
||||
if !validSeverity(input.Severity) || input.ExpectedRevision == 0 && input.ID != "" {
|
||||
return errors.New("alert rule input is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAlertRule(value AlertRule) error {
|
||||
if value.Version != AlertRuleVersion || value.Revision < 1 || model.ValidateSourceID(value.OrganizationID) != nil || model.ValidateSourceID(value.ID) != nil || model.ValidateSourceID(value.SavedQueryID) != nil || model.ValidateSourceID(value.CreatedBy) != nil || model.ValidateSourceID(value.UpdatedBy) != nil || !boundedText(value.Name, 128, false) || !boundedText(value.Description, 1024, true) || !validSeverity(value.Severity) || value.MinimumMatches < 1 || value.MinimumMatches > 100_000 || value.RequiredConsecutive < 1 || value.RequiredConsecutive > 10 || value.EvaluationInterval < 15*time.Second || value.EvaluationInterval > 24*time.Hour || value.NextEvaluationAt.IsZero() || value.CreatedAt.IsZero() || value.UpdatedAt.Before(value.CreatedAt) || value.LastError != "" && value.LastError != "query_unavailable" {
|
||||
return errors.New("alert rule is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateIncident(value Incident) error {
|
||||
states := map[string]bool{"pending": true, "firing": true, "acknowledged": true, "silenced": true, "resolved": true}
|
||||
if value.Version != IncidentVersion || model.ValidateSourceID(value.OrganizationID) != nil || model.ValidateSourceID(value.ID) != nil || model.ValidateSourceID(value.RuleID) != nil || !states[value.State] || !validSeverity(value.Severity) || !boundedText(value.Title, 128, false) || value.ConsecutiveMatches < 0 || value.StartedAt.IsZero() || value.LastObservedAt.Before(value.StartedAt) || value.UpdatedAt.Before(value.StartedAt) {
|
||||
return errors.New("incident is invalid")
|
||||
}
|
||||
if value.State == "acknowledged" && (value.AcknowledgedBy == "" || value.AcknowledgedAt == nil) || value.State == "silenced" && (value.SilencedBy == "" || value.SilencedUntil == nil) || value.State == "resolved" && value.ResolvedAt == nil {
|
||||
return errors.New("incident state metadata is invalid")
|
||||
}
|
||||
if value.AcknowledgedBy != "" && (model.ValidateSourceID(value.AcknowledgedBy) != nil || value.AcknowledgedAt == nil) || value.AcknowledgedAt != nil && (value.AcknowledgedBy == "" || value.AcknowledgedAt.Before(value.StartedAt)) || value.SilencedBy != "" && (model.ValidateSourceID(value.SilencedBy) != nil || value.SilencedUntil == nil) || value.SilencedUntil != nil && (value.SilencedBy == "" || value.SilencedUntil.Before(value.StartedAt)) || value.ResolvedAt != nil && value.ResolvedAt.Before(value.StartedAt) {
|
||||
return errors.New("incident actor metadata is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseOptionalTime(value sql.NullString) (*time.Time, error) {
|
||||
if !value.Valid {
|
||||
return nil, nil
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339Nano, value.String)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &parsed, nil
|
||||
}
|
||||
|
||||
func validSeverity(value string) bool {
|
||||
return value == "information" || value == "warning" || value == "critical"
|
||||
}
|
||||
func boolInt(value bool) int {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
func nullableText(value string) any {
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
func nullableTime(value *time.Time) any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
return value.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
func TestAlertRuleEvaluationAndIncidentLifecycle(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
organizationID := "organization-a"
|
||||
actor := "operator-a"
|
||||
now := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC)
|
||||
saved, err := store.SaveQuery(ctx, SavedQueryInput{
|
||||
OrganizationID: organizationID, ActorUserID: actor, MaxRows: 100,
|
||||
Name: "Recent failures", Description: "Recent HTTP failures for an alert.",
|
||||
Query: "logs | where status >= 500 | window 1h | limit 50",
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rule, err := store.SaveAlertRule(ctx, AlertRuleInput{
|
||||
OrganizationID: organizationID, ActorUserID: actor, SavedQueryID: saved.ID,
|
||||
Name: "HTTP failures", Description: "Open after two consecutive matching evaluations.",
|
||||
Severity: "critical", MinimumMatches: 1, RequiredConsecutive: 2,
|
||||
EvaluationInterval: 15 * time.Second, Enabled: true,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
budget := query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 16 << 20, MaxMemoryBytes: 8 << 20}
|
||||
|
||||
empty, err := store.EvaluateDueAlertRules(ctx, budget, now)
|
||||
if err != nil || len(empty) != 1 || empty[0].Matched || empty[0].Rows != 0 || empty[0].IncidentID != "" {
|
||||
t.Fatalf("empty evaluation=%+v err=%v", empty, err)
|
||||
}
|
||||
if incidents, listErr := store.Incidents(ctx, organizationID, false, 100); listErr != nil || len(incidents) != 0 {
|
||||
t.Fatalf("empty incidents=%+v err=%v", incidents, listErr)
|
||||
}
|
||||
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: organizationID, ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observed := now.Add(10 * time.Second)
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: observed, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: observed, Name: "http.request", Attributes: map[string]string{"http.status_code": "503", "http.route": "/failed"}}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, observed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
|
||||
first, err := store.EvaluateDueAlertRules(ctx, budget, now.Add(15*time.Second))
|
||||
if err != nil || len(first) != 1 || !first[0].Matched || first[0].IncidentState != "pending" {
|
||||
t.Fatalf("first evaluation=%+v err=%v", first, err)
|
||||
}
|
||||
second, err := store.EvaluateDueAlertRules(ctx, budget, now.Add(30*time.Second))
|
||||
if err != nil || len(second) != 1 || second[0].IncidentID != first[0].IncidentID || second[0].IncidentState != "firing" {
|
||||
t.Fatalf("second evaluation=%+v err=%v", second, err)
|
||||
}
|
||||
incidentID := second[0].IncidentID
|
||||
third, err := store.EvaluateDueAlertRules(ctx, budget, now.Add(45*time.Second))
|
||||
if err != nil || len(third) != 1 || third[0].IncidentChanged || third[0].IncidentState != "firing" {
|
||||
t.Fatalf("steady evaluation=%+v err=%v", third, err)
|
||||
}
|
||||
steadyEvents, err := store.IncidentEvents(ctx, organizationID, incidentID)
|
||||
if err != nil || len(steadyEvents) != 2 {
|
||||
t.Fatalf("steady events=%+v err=%v", steadyEvents, err)
|
||||
}
|
||||
|
||||
acknowledged, err := store.TransitionIncident(ctx, organizationID, incidentID, "acknowledge", actor, nil, now.Add(46*time.Second))
|
||||
if err != nil || acknowledged.State != "acknowledged" || acknowledged.AcknowledgedBy != actor {
|
||||
t.Fatalf("acknowledged=%+v err=%v", acknowledged, err)
|
||||
}
|
||||
silenceUntil := now.Add(time.Hour)
|
||||
silenced, err := store.TransitionIncident(ctx, organizationID, incidentID, "silence", actor, &silenceUntil, now.Add(47*time.Second))
|
||||
if err != nil || silenced.State != "silenced" || silenced.SilencedUntil == nil || !silenced.SilencedUntil.Equal(silenceUntil) {
|
||||
t.Fatalf("silenced=%+v err=%v", silenced, err)
|
||||
}
|
||||
resolved, err := store.TransitionIncident(ctx, organizationID, incidentID, "resolve", actor, nil, now.Add(48*time.Second))
|
||||
if err != nil || resolved.State != "resolved" || resolved.ResolvedAt == nil {
|
||||
t.Fatalf("resolved=%+v err=%v", resolved, err)
|
||||
}
|
||||
if _, err = store.TransitionIncident(ctx, organizationID, incidentID, "acknowledge", actor, nil, now.Add(49*time.Second)); err == nil {
|
||||
t.Fatal("resolved incident accepted another transition")
|
||||
}
|
||||
|
||||
events, err := store.IncidentEvents(ctx, organizationID, incidentID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"opened", "promoted", "acknowledged", "silenced", "resolved"}
|
||||
if len(events) != len(want) {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
for index, event := range events {
|
||||
if event.Sequence != index+1 || event.Event != want[index] {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
}
|
||||
all, err := store.Incidents(ctx, organizationID, true, 100)
|
||||
if err != nil || len(all) != 1 || all[0].State != "resolved" {
|
||||
t.Fatalf("all incidents=%+v err=%v", all, err)
|
||||
}
|
||||
loadedRule, err := store.AlertRule(ctx, organizationID, rule.ID)
|
||||
if err != nil || loadedRule.LastEvaluatedAt == nil || loadedRule.LastResult == nil || *loadedRule.LastResult != 1 || loadedRule.LastError != "" {
|
||||
t.Fatalf("rule=%+v err=%v", loadedRule, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertQueryFailureDoesNotOpenOrResolveAnIncident(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 14, 0, 0, 0, time.UTC)
|
||||
saved, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", ActorUserID: "operator-a", MaxRows: 100, Name: "Unknown field", Description: "Requires reviewed sensitive-field access.", Query: "logs | where private.value == secret | limit 10"}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rule, err := store.SaveAlertRule(ctx, AlertRuleInput{OrganizationID: "organization-a", ActorUserID: "operator-a", SavedQueryID: saved.ID, Name: "Fail closed", Description: "A query failure is not a healthy result.", Severity: "critical", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
budget := query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 16 << 20, MaxMemoryBytes: 8 << 20}
|
||||
evaluations, err := store.EvaluateDueAlertRules(ctx, budget, now)
|
||||
if err != nil || len(evaluations) != 1 || evaluations[0].Error != "query_unavailable" || evaluations[0].IncidentID != "" || evaluations[0].IncidentChanged {
|
||||
t.Fatalf("evaluations=%+v err=%v", evaluations, err)
|
||||
}
|
||||
loaded, err := store.AlertRule(ctx, "organization-a", rule.ID)
|
||||
if err != nil || loaded.LastError != "query_unavailable" || loaded.LastResult != nil {
|
||||
t.Fatalf("rule=%+v err=%v", loaded, err)
|
||||
}
|
||||
incidents, err := store.Incidents(ctx, "organization-a", true, 10)
|
||||
if err != nil || len(incidents) != 0 {
|
||||
t.Fatalf("incidents=%+v err=%v", incidents, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDueAlertRuleClaimPreventsConcurrentDuplicateEvaluation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 13, 0, 0, 0, time.UTC)
|
||||
saved, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", ActorUserID: "operator-a", MaxRows: 100, Name: "Any logs", Description: "Any recent log.", Query: "logs | window 1h | limit 10"}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.SaveAlertRule(ctx, AlertRuleInput{OrganizationID: "organization-a", ActorUserID: "operator-a", SavedQueryID: saved.ID, Name: "Any log", Description: "Concurrency claim test.", Severity: "warning", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
budget := query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 16 << 20, MaxMemoryBytes: 8 << 20}
|
||||
var group sync.WaitGroup
|
||||
results := make(chan []AlertEvaluation, 2)
|
||||
errors := make(chan error, 2)
|
||||
for range 2 {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
value, evaluationErr := store.EvaluateDueAlertRules(ctx, budget, now)
|
||||
results <- value
|
||||
errors <- evaluationErr
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
close(results)
|
||||
close(errors)
|
||||
total := 0
|
||||
for err = range errors {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for result := range results {
|
||||
total += len(result)
|
||||
}
|
||||
if total != 1 {
|
||||
t.Fatalf("evaluations=%d, want 1", total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertRulesRejectCrossOrganizationAndUnboundedInputs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Now().UTC()
|
||||
saved, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: "organization-a", ActorUserID: "operator-a", MaxRows: 100, Name: "Query", Description: "Scoped query.", Query: "logs | limit 10"}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := AlertRuleInput{OrganizationID: "organization-b", ActorUserID: "operator-b", SavedQueryID: saved.ID, Name: "Cross scope", Description: "Must fail.", Severity: "warning", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true}
|
||||
if _, err = store.SaveAlertRule(ctx, base, now); err == nil {
|
||||
t.Fatal("cross-organization saved query was accepted")
|
||||
}
|
||||
base.OrganizationID = "organization-a"
|
||||
base.EvaluationInterval = 14 * time.Second
|
||||
if _, err = store.SaveAlertRule(ctx, base, now); err == nil {
|
||||
t.Fatal("too-frequent evaluation was accepted")
|
||||
}
|
||||
base.EvaluationInterval = 15 * time.Second
|
||||
base.RequiredConsecutive = 11
|
||||
if _, err = store.SaveAlertRule(ctx, base, now); err == nil {
|
||||
t.Fatal("unbounded confirmation count was accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
logRollupVersion = 1
|
||||
logRollupWindow = 5 * time.Minute
|
||||
maxLogRollupGroupsBatch = model.MaxRecords
|
||||
)
|
||||
|
||||
const logObservationBytesSQL = `64+LENGTH(project_id)+LENGTH(environment_id)+LENGTH(service_id)+LENGTH(source_id)+LENGTH(stream_id)+LENGTH(signal)+LENGTH(timestamp)+LENGTH(name)+COALESCE(LENGTH(severity),0)+COALESCE(LENGTH(body),0)+COALESCE(LENGTH(trace_id),0)+COALESCE(LENGTH(span_id),0)+COALESCE(LENGTH(correlation_id),0)+LENGTH(attributes_json)`
|
||||
|
||||
type logRollup struct {
|
||||
projectID, environmentID, serviceID, route string
|
||||
bucket int64
|
||||
status, routePresent int
|
||||
count, scannedBytes int64
|
||||
}
|
||||
|
||||
func ensureLogRollups(ctx context.Context, db *sql.DB) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin log rollup migration: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, statement := range []string{
|
||||
`CREATE TABLE IF NOT EXISTS log_rollup_state (id INTEGER PRIMARY KEY CHECK(id=1),version INTEGER NOT NULL CHECK(version BETWEEN 0 AND 1))`,
|
||||
`INSERT OR IGNORE INTO log_rollup_state(id,version) VALUES(1,0)`,
|
||||
`CREATE TABLE IF NOT EXISTS log_status_route_rollups_5m (
|
||||
organization_id TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
environment_id TEXT NOT NULL,
|
||||
service_id TEXT NOT NULL,
|
||||
bucket_start INTEGER NOT NULL,
|
||||
status INTEGER NOT NULL CHECK(status BETWEEN 100 AND 999),
|
||||
route TEXT NOT NULL,
|
||||
route_present INTEGER NOT NULL CHECK(route_present IN (0,1)),
|
||||
observation_count INTEGER NOT NULL CHECK(typeof(observation_count)='integer' AND observation_count > 0),
|
||||
scanned_bytes INTEGER NOT NULL CHECK(typeof(scanned_bytes)='integer' AND scanned_bytes >= 0),
|
||||
PRIMARY KEY(organization_id,project_id,environment_id,service_id,bucket_start,status,route,route_present)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS log_rollup_segments (segment_digest TEXT PRIMARY KEY)`,
|
||||
`CREATE INDEX IF NOT EXISTS log_rollups_status_time ON log_status_route_rollups_5m(organization_id,status,bucket_start)`,
|
||||
`CREATE INDEX IF NOT EXISTS log_rollups_scope_time ON log_status_route_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start,status)`,
|
||||
} {
|
||||
if _, err = tx.ExecContext(ctx, statement); err != nil {
|
||||
return fmt.Errorf("migrate log rollups: %w", err)
|
||||
}
|
||||
}
|
||||
var version int
|
||||
if err = tx.QueryRowContext(ctx, `SELECT version FROM log_rollup_state WHERE id=1`).Scan(&version); err != nil {
|
||||
return errors.New("read log rollup migration state")
|
||||
}
|
||||
if version == 0 {
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM log_status_route_rollups_5m`); err != nil {
|
||||
return errors.New("clear incomplete log rollup migration")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM log_rollup_segments`); err != nil {
|
||||
return errors.New("clear incomplete log rollup ledger")
|
||||
}
|
||||
status := `json_extract(attributes_json,'$."http.status_code"')`
|
||||
route := `json_extract(attributes_json,'$."http.route"')`
|
||||
statement := `INSERT INTO log_status_route_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start,status,route,route_present,observation_count,scanned_bytes)
|
||||
SELECT organization_id,project_id,environment_id,service_id,CAST(unixepoch(timestamp)/300 AS INTEGER)*300,CAST(` + status + ` AS INTEGER),COALESCE(CAST(` + route + ` AS TEXT),''),CASE WHEN json_type(attributes_json,'$."http.route"') IS NULL THEN 0 ELSE 1 END,COUNT(*),SUM(` + logObservationBytesSQL + `)
|
||||
FROM observations WHERE signal=? AND printf('%d',CAST(` + status + ` AS INTEGER))=` + status + ` AND CAST(` + status + ` AS INTEGER) BETWEEN 100 AND 999
|
||||
GROUP BY organization_id,project_id,environment_id,service_id,CAST(unixepoch(timestamp)/300 AS INTEGER)*300,CAST(` + status + ` AS INTEGER),COALESCE(CAST(` + route + ` AS TEXT),''),CASE WHEN json_type(attributes_json,'$."http.route"') IS NULL THEN 0 ELSE 1 END`
|
||||
if _, err = tx.ExecContext(ctx, statement, model.SignalLogs); err != nil {
|
||||
return errors.New("backfill log rollups")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO log_rollup_segments(segment_digest) SELECT DISTINCT segment_digest FROM observations WHERE signal=?`, model.SignalLogs); err != nil {
|
||||
return errors.New("record backfilled log rollup segments")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE log_rollup_state SET version=? WHERE id=1 AND version=0`, logRollupVersion); err != nil {
|
||||
return errors.New("activate log rollup migration")
|
||||
}
|
||||
}
|
||||
if version != 0 && version != logRollupVersion {
|
||||
return errors.New("unsupported log rollup version")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return errors.New("commit log rollup migration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func projectLogRollups(ctx context.Context, tx *sql.Tx, scope model.Scope, batch model.Batch, segmentDigest string, observationBytes []int64) error {
|
||||
if batch.Signal != model.SignalLogs {
|
||||
return nil
|
||||
}
|
||||
if len(observationBytes) != len(batch.Records) {
|
||||
return errors.New("log projection byte evidence is incomplete")
|
||||
}
|
||||
ledger, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO log_rollup_segments(segment_digest) VALUES(?)`, segmentDigest)
|
||||
if err != nil {
|
||||
return errors.New("record log rollup segment")
|
||||
}
|
||||
inserted, err := ledger.RowsAffected()
|
||||
if err != nil {
|
||||
return errors.New("inspect log rollup segment")
|
||||
}
|
||||
if inserted == 0 {
|
||||
return nil
|
||||
}
|
||||
groups := map[string]*logRollup{}
|
||||
for index, observation := range batch.Records {
|
||||
statusText, ok := observation.Attributes["http.status_code"]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
status, statusErr := strconv.Atoi(statusText)
|
||||
if statusErr != nil || strconv.Itoa(status) != statusText || status < 100 || status > 999 {
|
||||
continue
|
||||
}
|
||||
route, routeOK := observation.Attributes["http.route"]
|
||||
routePresent := 0
|
||||
if routeOK {
|
||||
routePresent = 1
|
||||
}
|
||||
bucket := observation.Timestamp.UTC().Truncate(logRollupWindow).Unix()
|
||||
key := scope.ProjectID + "\x00" + scope.EnvironmentID + "\x00" + scope.ServiceID + "\x00" + strconv.FormatInt(bucket, 10) + "\x00" + statusText + "\x00" + strconv.Itoa(routePresent) + "\x00" + route
|
||||
group := groups[key]
|
||||
if group == nil {
|
||||
group = &logRollup{projectID: scope.ProjectID, environmentID: scope.EnvironmentID, serviceID: scope.ServiceID, bucket: bucket, status: status, route: route, routePresent: routePresent}
|
||||
groups[key] = group
|
||||
if len(groups) > maxLogRollupGroupsBatch {
|
||||
return fmt.Errorf("log batch exceeds %d rollup groups", maxLogRollupGroupsBatch)
|
||||
}
|
||||
}
|
||||
bytes := observationBytes[index]
|
||||
if bytes < 0 {
|
||||
return errors.New("log projection byte evidence is invalid")
|
||||
}
|
||||
if group.count == math.MaxInt64 || bytes > math.MaxInt64-group.scannedBytes {
|
||||
return errors.New("log rollup count exceeds integer range")
|
||||
}
|
||||
group.count++
|
||||
group.scannedBytes += bytes
|
||||
}
|
||||
for _, group := range groups {
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO log_status_route_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start,status,route,route_present,observation_count,scanned_bytes) VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(organization_id,project_id,environment_id,service_id,bucket_start,status,route,route_present) DO UPDATE SET observation_count=log_status_route_rollups_5m.observation_count+excluded.observation_count,scanned_bytes=log_status_route_rollups_5m.scanned_bytes+excluded.scanned_bytes`, scope.OrganizationID, group.projectID, group.environmentID, group.serviceID, group.bucket, group.status, group.route, group.routePresent, group.count, group.scannedBytes)
|
||||
if err != nil {
|
||||
return errors.New("merge log rollup")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func projectedObservationBytes(scope model.Scope, batch model.Batch, observation model.Observation, attributesBytes int) int64 {
|
||||
return int64(64 + len(scope.ProjectID) + len(scope.EnvironmentID) + len(scope.ServiceID) + len(batch.SourceID) + len(batch.StreamID) + len(batch.Signal) + len(observation.Timestamp.UTC().Format(time.RFC3339Nano)) + len(observation.Name) + len(observation.Severity) + len(observation.Body) + len(observation.TraceID) + len(observation.SpanID) + len(observation.CorrelationID) + attributesBytes)
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"math"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
// indexedLogCountSummaryEligible identifies the typed status/route count
|
||||
// summary maintained as an exact five-minute projection. Larger bucket sizes
|
||||
// can combine those rows; a partial lower window boundary scans at most one
|
||||
// five-minute fragment from the primary projection.
|
||||
func indexedLogCountSummaryEligible(ast query.AST) bool {
|
||||
if ast.Signal != model.SignalLogs || ast.Summary == nil || len(ast.Summary.Aggregates) != 1 || len(ast.Summary.GroupBy) != 1 {
|
||||
return false
|
||||
}
|
||||
aggregate := ast.Summary.Aggregates[0]
|
||||
if aggregate.Function != "count" || aggregate.Field != "" || query.CanonicalField(ast.Summary.GroupBy[0]) != "http.route" {
|
||||
return false
|
||||
}
|
||||
if ast.Bucket != 0 && (ast.Bucket < logRollupWindow || ast.Bucket%logRollupWindow != 0) {
|
||||
return false
|
||||
}
|
||||
if len(ast.Filters) != 1 || query.CanonicalField(ast.Filters[0].Field) != "http.status_code" {
|
||||
return false
|
||||
}
|
||||
threshold, err := strconv.Atoi(ast.Filters[0].Value)
|
||||
return err == nil && threshold >= 100 && threshold <= 999 && ast.Filters[0].Op == ">="
|
||||
}
|
||||
|
||||
// estimateLogRollupBytes accounts for the compact rows the optimized query
|
||||
// reads, plus at most one raw five-minute fragment when the requested lower
|
||||
// boundary does not align with a rollup bucket. It deliberately does not use
|
||||
// a client-supplied estimate or treat logical observation bytes as physical
|
||||
// scan cost.
|
||||
func (s *Store) estimateLogRollupBytes(ctx context.Context, scope query.Scope, ast query.AST, now time.Time) (int64, error) {
|
||||
path := s.organizationProjectionPath(scope.OrganizationID)
|
||||
info, err := os.Lstat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return 0, errors.New("organization projection is unavailable")
|
||||
}
|
||||
dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String()
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return 0, errors.New("open log rollup estimate")
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
status, err := typedFilterValue(ast.Filters[0].Value, schema.TypeInteger)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
cutoff, rollupStart := logRollupWindowStart(ast, now)
|
||||
statement := `SELECT COALESCE(SUM(96+LENGTH(project_id)+LENGTH(environment_id)+LENGTH(service_id)+LENGTH(route)),0) FROM log_status_route_rollups_5m WHERE organization_id=? AND status>=?`
|
||||
arguments := []any{scope.OrganizationID, status}
|
||||
for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} {
|
||||
if selected.value != "" {
|
||||
statement += " AND " + selected.column + "=?"
|
||||
arguments = append(arguments, selected.value)
|
||||
}
|
||||
}
|
||||
if ast.Window > 0 {
|
||||
statement += ` AND bucket_start>=?`
|
||||
arguments = append(arguments, rollupStart)
|
||||
}
|
||||
var estimated int64
|
||||
if err = db.QueryRowContext(ctx, statement, arguments...).Scan(&estimated); err != nil || estimated < 0 {
|
||||
return 0, errors.New("estimate log rollup scan")
|
||||
}
|
||||
if ast.Window == 0 || !cutoff.Before(time.Unix(rollupStart, 0).UTC()) {
|
||||
return estimated, nil
|
||||
}
|
||||
raw := `SELECT COALESCE(SUM(` + logObservationBytesSQL + `),0) FROM observations o INDEXED BY observations_http_status WHERE o.organization_id=? AND o.signal=?`
|
||||
rawArguments := []any{scope.OrganizationID, string(ast.Signal)}
|
||||
for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} {
|
||||
if selected.value != "" {
|
||||
raw += " AND o." + selected.column + "=?"
|
||||
rawArguments = append(rawArguments, selected.value)
|
||||
}
|
||||
}
|
||||
raw += ` AND o.timestamp>=? AND o.timestamp<?`
|
||||
rawArguments = append(rawArguments, cutoff.Format(time.RFC3339Nano), time.Unix(rollupStart, 0).UTC().Format(time.RFC3339Nano))
|
||||
statusExpression := `json_extract(o.attributes_json,'$."http.status_code"')`
|
||||
raw += ` AND printf('%d',CAST(` + statusExpression + ` AS INTEGER))=` + statusExpression + ` AND CAST(` + statusExpression + ` AS INTEGER)>=?`
|
||||
rawArguments = append(rawArguments, status)
|
||||
var partial int64
|
||||
if err = db.QueryRowContext(ctx, raw, rawArguments...).Scan(&partial); err != nil || partial < 0 || partial > math.MaxInt64-estimated {
|
||||
return 0, errors.New("estimate partial log rollup scan")
|
||||
}
|
||||
return estimated + partial, nil
|
||||
}
|
||||
|
||||
func logRollupWindowStart(ast query.AST, now time.Time) (time.Time, int64) {
|
||||
if ast.Window == 0 {
|
||||
return time.Time{}, math.MinInt64
|
||||
}
|
||||
cutoff := now.UTC().Add(-ast.Window)
|
||||
start := cutoff.Truncate(logRollupWindow)
|
||||
if !start.Equal(cutoff) {
|
||||
start = start.Add(logRollupWindow)
|
||||
}
|
||||
return cutoff, start.Unix()
|
||||
}
|
||||
|
||||
type logSummaryKey struct {
|
||||
bucket int64
|
||||
route string
|
||||
routePresent bool
|
||||
}
|
||||
|
||||
type logSummaryValue struct {
|
||||
count, scannedBytes int64
|
||||
}
|
||||
|
||||
func (s *Store) queryIndexedLogCountSummary(ctx context.Context, path string, ast query.AST, scope query.Scope, budget query.Budget, now time.Time, result query.Result) (query.Result, error) {
|
||||
runContext, cancel := context.WithTimeout(ctx, budget.MaxDuration)
|
||||
defer cancel()
|
||||
started := time.Now()
|
||||
dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String()
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return query.Result{}, errors.New("open indexed log summary")
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
status, statusErr := typedFilterValue(ast.Filters[0].Value, schema.TypeInteger)
|
||||
if statusErr != nil {
|
||||
return query.Result{}, statusErr
|
||||
}
|
||||
groups := map[logSummaryKey]logSummaryValue{}
|
||||
cutoff, rollupStart := logRollupWindowStart(ast, now)
|
||||
|
||||
selects := []string{}
|
||||
arguments := []any{}
|
||||
groupColumns := "route,route_present"
|
||||
if ast.Bucket > 0 {
|
||||
seconds := int64(ast.Bucket / time.Second)
|
||||
selects = append(selects, `CAST(bucket_start/? AS INTEGER)*?`)
|
||||
arguments = append(arguments, seconds, seconds)
|
||||
groupColumns = "1,route,route_present"
|
||||
}
|
||||
selects = append(selects, `route`, `route_present`, `SUM(observation_count)`, `SUM(scanned_bytes)`)
|
||||
statement := `SELECT ` + joinSQL(selects) + ` FROM log_status_route_rollups_5m WHERE organization_id=?`
|
||||
arguments = append(arguments, scope.OrganizationID)
|
||||
for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} {
|
||||
if selected.value != "" {
|
||||
statement += " AND " + selected.column + "=?"
|
||||
arguments = append(arguments, selected.value)
|
||||
}
|
||||
}
|
||||
statement += ` AND status>=?`
|
||||
arguments = append(arguments, status)
|
||||
if ast.Window > 0 {
|
||||
statement += ` AND bucket_start>=?`
|
||||
arguments = append(arguments, rollupStart)
|
||||
}
|
||||
statement += ` GROUP BY ` + groupColumns
|
||||
rows, err := db.QueryContext(runContext, statement, arguments...)
|
||||
if err != nil {
|
||||
return query.Result{}, queryExecutionError(runContext, err)
|
||||
}
|
||||
if err = readLogSummaryRows(rows, ast.Bucket > 0, groups); err != nil {
|
||||
return query.Result{}, queryExecutionError(runContext, err)
|
||||
}
|
||||
|
||||
if ast.Window > 0 && cutoff.Unix() < rollupStart {
|
||||
rawSelects := []string{}
|
||||
rawArguments := []any{}
|
||||
rawGroupColumns := `json_extract(o.attributes_json,'$."http.route"'),CASE WHEN json_type(o.attributes_json,'$."http.route"') IS NULL THEN 0 ELSE 1 END`
|
||||
if ast.Bucket > 0 {
|
||||
seconds := int64(ast.Bucket / time.Second)
|
||||
rawSelects = append(rawSelects, `CAST(unixepoch(o.timestamp)/? AS INTEGER)*?`)
|
||||
rawArguments = append(rawArguments, seconds, seconds)
|
||||
rawGroupColumns = `1,` + rawGroupColumns
|
||||
}
|
||||
rawSelects = append(rawSelects, `json_extract(o.attributes_json,'$."http.route"')`, `CASE WHEN json_type(o.attributes_json,'$."http.route"') IS NULL THEN 0 ELSE 1 END`, `COUNT(*)`, `COALESCE(SUM(`+logObservationBytesSQL+`),0)`)
|
||||
raw := `SELECT ` + joinSQL(rawSelects) + ` FROM observations o INDEXED BY observations_http_status WHERE o.organization_id=? AND o.signal=?`
|
||||
rawArguments = append(rawArguments, scope.OrganizationID, string(ast.Signal))
|
||||
for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} {
|
||||
if selected.value != "" {
|
||||
raw += " AND o." + selected.column + "=?"
|
||||
rawArguments = append(rawArguments, selected.value)
|
||||
}
|
||||
}
|
||||
raw += ` AND o.timestamp>=? AND o.timestamp<?`
|
||||
rawArguments = append(rawArguments, cutoff.Format(time.RFC3339Nano), time.Unix(rollupStart, 0).UTC().Format(time.RFC3339Nano))
|
||||
statusExpression := `json_extract(o.attributes_json,'$."http.status_code"')`
|
||||
raw += ` AND printf('%d',CAST(` + statusExpression + ` AS INTEGER))=` + statusExpression + ` AND CAST(` + statusExpression + ` AS INTEGER)>=? GROUP BY ` + rawGroupColumns
|
||||
rawArguments = append(rawArguments, status)
|
||||
partial, queryErr := db.QueryContext(runContext, raw, rawArguments...)
|
||||
if queryErr != nil {
|
||||
return query.Result{}, queryExecutionError(runContext, queryErr)
|
||||
}
|
||||
if err = readLogSummaryRows(partial, ast.Bucket > 0, groups); err != nil {
|
||||
return query.Result{}, queryExecutionError(runContext, err)
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]logSummaryKey, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Slice(keys, func(left, right int) bool {
|
||||
if keys[left].bucket != keys[right].bucket {
|
||||
return keys[left].bucket < keys[right].bucket
|
||||
}
|
||||
if keys[left].routePresent != keys[right].routePresent {
|
||||
return !keys[left].routePresent
|
||||
}
|
||||
return keys[left].route < keys[right].route
|
||||
})
|
||||
var memoryBytes int64
|
||||
maximumInt := int64(^uint(0) >> 1)
|
||||
for _, key := range keys {
|
||||
value := groups[key]
|
||||
if value.count < 1 || value.scannedBytes < 0 || value.count > maximumInt-int64(result.Stats.ScannedRows) || value.scannedBytes > budget.MaxScannedBytes-result.Stats.ScannedBytes {
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
addition := int64(256 + len(key.route))
|
||||
if addition > budget.MaxMemoryBytes-memoryBytes {
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
memoryBytes += addition
|
||||
result.Stats.ScannedRows += int(value.count)
|
||||
result.Stats.MatchedRows += int(value.count)
|
||||
result.Stats.ScannedBytes += value.scannedBytes
|
||||
values := make([]*string, 0, len(result.Columns))
|
||||
if ast.Bucket > 0 {
|
||||
bucket := time.Unix(key.bucket, 0).UTC().Format(time.RFC3339Nano)
|
||||
values = append(values, stringPointer(bucket))
|
||||
}
|
||||
if key.routePresent {
|
||||
values = append(values, stringPointer(key.route))
|
||||
} else {
|
||||
values = append(values, nil)
|
||||
}
|
||||
values = append(values, stringPointer(strconv.FormatInt(value.count, 10)))
|
||||
result.Rows = append(result.Rows, query.Row{Values: values})
|
||||
}
|
||||
if ast.Sort != nil {
|
||||
if err = sortRows(result.Rows, result.Columns, ast.Sort.Field, ast.Sort.Descending); err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
}
|
||||
if len(result.Rows) > ast.Limit {
|
||||
result.Rows = result.Rows[:ast.Limit]
|
||||
result.Stats.Truncated = true
|
||||
}
|
||||
result.Stats.DurationNS = time.Since(started).Nanoseconds()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func readLogSummaryRows(rows *sql.Rows, bucketed bool, groups map[logSummaryKey]logSummaryValue) error {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var bucket sql.NullInt64
|
||||
var route sql.NullString
|
||||
var routePresent int
|
||||
var count, scannedBytes int64
|
||||
var err error
|
||||
if bucketed {
|
||||
err = rows.Scan(&bucket, &route, &routePresent, &count, &scannedBytes)
|
||||
} else {
|
||||
err = rows.Scan(&route, &routePresent, &count, &scannedBytes)
|
||||
}
|
||||
if err != nil || bucketed && !bucket.Valid || routePresent < 0 || routePresent > 1 || routePresent == 1 && !route.Valid || count < 1 || scannedBytes < 0 {
|
||||
return errors.New("read indexed log summary")
|
||||
}
|
||||
key := logSummaryKey{routePresent: routePresent == 1}
|
||||
if bucketed {
|
||||
key.bucket = bucket.Int64
|
||||
}
|
||||
if route.Valid {
|
||||
key.route = route.String
|
||||
}
|
||||
current := groups[key]
|
||||
if count > math.MaxInt64-current.count || scannedBytes > math.MaxInt64-current.scannedBytes {
|
||||
return query.ErrBudgetExceeded
|
||||
}
|
||||
current.count += count
|
||||
current.scannedBytes += scannedBytes
|
||||
groups[key] = current
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func joinSQL(parts []string) string {
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
joined := parts[0]
|
||||
for _, part := range parts[1:] {
|
||||
joined += "," + part
|
||||
}
|
||||
return joined
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
func TestIndexedLogCountSummaryPreservesScopeBucketsAndStatistics(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 18, 12, 4, 0, 0, time.UTC)
|
||||
primaryScope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "web"}
|
||||
primaryToken, err := store.CreateSource(ctx, "source-a", primaryScope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
primary := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{
|
||||
requestObservation(now.Add(-time.Minute), "/items", 503, 300),
|
||||
requestObservation(now.Add(-2*time.Minute), "/items", 500, 200),
|
||||
requestObservation(now.Add(-3*time.Minute), "/ignored", 200, 100),
|
||||
requestObservation(now.Add(-10*time.Minute), "/about", 503, 400),
|
||||
{Timestamp: now.Add(-time.Minute), Name: "application.http.request", Attributes: map[string]string{"http.route": "/invalid", "http.status_code": "500suffix", "duration_ns": "100"}},
|
||||
}}
|
||||
if _, err = store.Ingest(ctx, primaryToken, primary, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondaryScope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-b", EnvironmentID: "production", ServiceID: "worker"}
|
||||
secondaryToken, err := store.CreateSource(ctx, "source-b", secondaryScope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondary := model.Batch{Version: model.BatchVersion, SourceID: "source-b", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{
|
||||
requestObservation(now.Add(-time.Minute), "/other", 500, 500),
|
||||
}}
|
||||
if _, err = store.Ingest(ctx, secondaryToken, secondary, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
|
||||
ast, err := query.Parse(`logs | where status >= 500 | window 1h | summarize count() by route, window(5m) | sort count desc | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Explain.ProjectedSources) != 1 || !strings.HasSuffix(result.Explain.ProjectedSources[0], "/rollup:http-status-route:5m") {
|
||||
t.Fatalf("sources=%v", result.Explain.ProjectedSources)
|
||||
}
|
||||
if len(result.Rows) != 3 || columnValue(t, result, 0, "http.route") != "/items" || columnValue(t, result, 0, "count") != "2" {
|
||||
t.Fatalf("result=%+v", result)
|
||||
}
|
||||
if result.Stats.ScannedRows != 4 || result.Stats.MatchedRows != 4 || result.Stats.ScannedBytes < 1 || result.Stats.Truncated {
|
||||
t.Fatalf("statistics=%+v", result.Stats)
|
||||
}
|
||||
if result.Explain.EstimatedScanBytes < 1 || result.Explain.EstimatedScanBytes >= result.Stats.ScannedBytes {
|
||||
t.Fatalf("estimate=%d logical_scan=%d", result.Explain.EstimatedScanBytes, result.Stats.ScannedBytes)
|
||||
}
|
||||
if bucket := columnValue(t, result, 0, "window_start"); bucket != now.Add(-time.Minute).Truncate(5*time.Minute).Format(time.RFC3339Nano) {
|
||||
t.Fatalf("bucket=%q", bucket)
|
||||
}
|
||||
|
||||
scoped, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a", ProjectID: "project-a"}, testQueryBudget(), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(scoped.Rows) != 2 || scoped.Stats.ScannedRows != 3 || scoped.Stats.MatchedRows != 3 {
|
||||
t.Fatalf("scoped=%+v", scoped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexedLogCountSummaryUsesRawOnlyForPartialLowerBucket(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 18, 12, 7, 30, 0, time.UTC)
|
||||
scope := model.Scope{OrganizationID: "organization-partial", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-partial", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-partial", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{
|
||||
requestObservation(time.Date(2026, 8, 18, 12, 1, 0, 0, time.UTC), "/before", 500, 1),
|
||||
requestObservation(time.Date(2026, 8, 18, 12, 2, 0, 0, time.UTC), "/partial", 500, 2),
|
||||
requestObservation(time.Date(2026, 8, 18, 12, 4, 59, 0, time.UTC), "/partial", 503, 3),
|
||||
requestObservation(time.Date(2026, 8, 18, 12, 5, 0, 0, time.UTC), "/full", 500, 4),
|
||||
requestObservation(time.Date(2026, 8, 18, 12, 6, 0, 0, time.UTC), "/ignored", 200, 5),
|
||||
{Timestamp: time.Date(2026, 8, 18, 12, 5, 1, 0, time.UTC), Name: "application.http.request", Attributes: map[string]string{"http.status_code": "500"}},
|
||||
{Timestamp: time.Date(2026, 8, 18, 12, 5, 2, 0, time.UTC), Name: "application.http.request", Attributes: map[string]string{"http.route": "", "http.status_code": "500"}},
|
||||
}}
|
||||
ack, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
// Replaying the already projected segment must not increment the additive
|
||||
// rollup. The segment ledger makes projection recovery idempotent.
|
||||
if err = projectAt(ctx, store.organizationProjectionPath(scope.OrganizationID), scope, batch, ack.Digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ast, err := query.Parse(`logs | where status >= 500 | window 6m | summarize count() by route, window(5m) | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Stats.ScannedRows != 5 || result.Stats.MatchedRows != 5 || len(result.Rows) != 4 {
|
||||
t.Fatalf("result=%+v", result)
|
||||
}
|
||||
if bucket := columnValue(t, result, 0, "window_start"); bucket != "2026-08-18T12:00:00Z" {
|
||||
t.Fatalf("partial bucket=%q", bucket)
|
||||
}
|
||||
if route := columnValue(t, result, 0, "http.route"); route != "/partial" || columnValue(t, result, 0, "count") != "2" {
|
||||
t.Fatalf("partial row=%+v", result.Rows[0])
|
||||
}
|
||||
// Missing and explicitly empty routes remain distinct typed values.
|
||||
if result.Rows[1].Values[1] != nil || result.Rows[2].Values[1] == nil || *result.Rows[2].Values[1] != "" {
|
||||
t.Fatalf("route presence was collapsed: %+v", result.Rows)
|
||||
}
|
||||
if columnValue(t, result, 3, "http.route") != "/full" || columnValue(t, result, 3, "count") != "1" {
|
||||
t.Fatalf("full row=%+v", result.Rows[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogRollupMigrationBackfillsExistingProjection(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
root := filepath.Join(t.TempDir(), "data")
|
||||
if err := os.MkdirAll(filepath.Join(root, "organizations", "legacy"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(root, "organizations", "legacy", "projection.sqlite")
|
||||
legacy, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = legacy.Exec(`CREATE TABLE observations (
|
||||
organization_id TEXT NOT NULL, project_id TEXT NOT NULL, environment_id TEXT NOT NULL, service_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL, stream_id TEXT NOT NULL, sequence INTEGER NOT NULL, record_index INTEGER NOT NULL,
|
||||
signal TEXT NOT NULL, timestamp TEXT NOT NULL, name TEXT NOT NULL, severity TEXT, body TEXT, value REAL,
|
||||
trace_id TEXT, span_id TEXT, correlation_id TEXT, attributes_json TEXT NOT NULL, segment_digest TEXT NOT NULL,
|
||||
PRIMARY KEY(source_id,stream_id,sequence,record_index));
|
||||
INSERT INTO observations VALUES
|
||||
('legacy','project','prod','web','source','logs',1,0,'logs','2026-08-18T12:01:00Z','request',NULL,NULL,NULL,NULL,NULL,NULL,'{"http.route":"/one","http.status_code":"503"}','digest'),
|
||||
('legacy','project','prod','web','source','logs',1,1,'logs','2026-08-18T12:02:00Z','request',NULL,NULL,NULL,NULL,NULL,NULL,'{"http.route":"/one","http.status_code":"503"}','digest'),
|
||||
('legacy','project','prod','web','source','logs',1,2,'logs','2026-08-18T12:02:01Z','request',NULL,NULL,NULL,NULL,NULL,NULL,'{"http.route":"/invalid","http.status_code":"503suffix"}','digest')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = legacy.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projection, err := openProjection(ctx, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer projection.Close()
|
||||
var count, segments, version int
|
||||
if err = projection.QueryRow(`SELECT observation_count FROM log_status_route_rollups_5m`).Scan(&count); err != nil || count != 2 {
|
||||
t.Fatalf("count=%d err=%v", count, err)
|
||||
}
|
||||
if err = projection.QueryRow(`SELECT COUNT(*) FROM log_rollup_segments`).Scan(&segments); err != nil || segments != 1 {
|
||||
t.Fatalf("segments=%d err=%v", segments, err)
|
||||
}
|
||||
if err = projection.QueryRow(`SELECT version FROM log_rollup_state WHERE id=1`).Scan(&version); err != nil || version != logRollupVersion {
|
||||
t.Fatalf("version=%d err=%v", version, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexedLogCountSummaryEligibilityIsNarrow(t *testing.T) {
|
||||
tests := []struct {
|
||||
text string
|
||||
want bool
|
||||
}{
|
||||
{`logs | where status >= 500 | summarize count() by route, window(5m) | limit 10`, true},
|
||||
{`logs | where status >= 400 | summarize count() by route | limit 10`, true},
|
||||
{`logs | where status == 500 | summarize count() by route | limit 10`, false},
|
||||
{`logs | where status =~ "5.." | summarize count() by route, window(5m) | limit 10`, false},
|
||||
{`logs | where route == "/items" | summarize count() by route, window(5m) | limit 10`, false},
|
||||
{`logs | where status >= 500 | summarize count() by status, window(5m) | limit 10`, false},
|
||||
{`logs | where status >= 500 | summarize p95(duration) by route, window(5m) | limit 10`, false},
|
||||
{`logs | where status >= 500 | summarize count() by route, window(1500ms) | limit 10`, false},
|
||||
{`traces | where status >= 500 | summarize count() by route, window(5m) | limit 10`, false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
ast, err := query.Parse(test.text, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("parse %q: %v", test.text, err)
|
||||
}
|
||||
if got := indexedLogCountSummaryEligible(ast); got != test.want {
|
||||
t.Fatalf("eligible(%q)=%t want %t", test.text, got, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestControlSchemaFourMigratesThroughRetentionSchema(t *testing.T) {
|
||||
db := openSchemaDatabase(t, 4)
|
||||
defer db.Close()
|
||||
var err error
|
||||
if err = migrateControl(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var version int
|
||||
if err = db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != controlSchema {
|
||||
t.Fatalf("version=%d err=%v", version, err)
|
||||
}
|
||||
for _, table := range []string{"alert_rules", "incidents", "incident_events", "organization_retention_policies", "retention_policy_events", "source_alert_transitions"} {
|
||||
var count int
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatalf("table=%s count=%d err=%v", table, count, err)
|
||||
}
|
||||
}
|
||||
columns, err := sqliteColumns(db, "segments")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, column := range []string{"signal", "first_observed_at", "last_observed_at", "record_count", "tier", "archiving_at", "archive_path", "cold_at", "retiring_at"} {
|
||||
if !columns[column] {
|
||||
t.Fatalf("missing segment column %s", column)
|
||||
}
|
||||
}
|
||||
policyColumns, err := sqliteColumns(db, "organization_retention_policies")
|
||||
if err != nil || !policyColumns["cold_raw_days"] || !policyColumns["delete_cold_raw"] {
|
||||
t.Fatalf("retention policy cold=%t delete=%t err=%v", policyColumns["cold_raw_days"], policyColumns["delete_cold_raw"], err)
|
||||
}
|
||||
streamColumns, err := sqliteColumns(db, "streams")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, column := range []string{"last_batch_digest", "last_wire_digest", "last_signal", "last_record_count", "last_encoded_bytes", "last_first_observed_at", "last_last_observed_at"} {
|
||||
if !streamColumns[column] {
|
||||
t.Fatalf("missing stream envelope column %s", column)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlSchemaSevenMigratesToForensicPreservation(t *testing.T) {
|
||||
db := openSchemaDatabase(t, 6)
|
||||
defer db.Close()
|
||||
if err := migrateControlRetention(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var version int
|
||||
if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != 7 {
|
||||
t.Fatalf("pre-migration version=%d err=%v", version, err)
|
||||
}
|
||||
if err := migrateControl(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(`INSERT INTO organization_retention_policies(organization_id,raw_logs_days,raw_traces_days,raw_metrics_days,cold_raw_days,metric_rollups_days,evidence_days,updated_by,updated_at) VALUES('org',30,30,14,400,400,400,'owner','2026-08-17T00:00:00Z')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var deleteColdRaw bool
|
||||
if err := db.QueryRow(`SELECT delete_cold_raw FROM organization_retention_policies WHERE organization_id='org'`).Scan(&deleteColdRaw); err != nil || deleteColdRaw {
|
||||
t.Fatalf("delete_cold_raw=%t err=%v", deleteColdRaw, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlSchemaTenPreservesExistingStreamWatermark(t *testing.T) {
|
||||
db := openSchemaDatabase(t, 10)
|
||||
defer db.Close()
|
||||
if _, err := db.Exec(`
|
||||
CREATE TABLE sources (
|
||||
id TEXT PRIMARY KEY, organization_id TEXT NOT NULL, project_id TEXT NOT NULL,
|
||||
environment_id TEXT NOT NULL, service_id TEXT NOT NULL,
|
||||
credential_digest BLOB NOT NULL UNIQUE, active INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL, rotated_at TEXT
|
||||
);
|
||||
CREATE TABLE streams (
|
||||
source_id TEXT NOT NULL REFERENCES sources(id), stream_id TEXT NOT NULL,
|
||||
last_sequence INTEGER NOT NULL, last_digest TEXT NOT NULL,
|
||||
PRIMARY KEY(source_id,stream_id)
|
||||
);
|
||||
INSERT INTO sources VALUES('source','org','project','prod','service',X'01',1,'2026-08-18T20:00:00Z',NULL);
|
||||
INSERT INTO streams VALUES('source','logs',42,'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa');
|
||||
`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := migrateControl(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sequence int
|
||||
var segmentDigest, batchDigest, wireDigest string
|
||||
if err := db.QueryRow(`SELECT last_sequence,last_digest,last_batch_digest,last_wire_digest FROM streams WHERE source_id='source' AND stream_id='logs'`).Scan(&sequence, &segmentDigest, &batchDigest, &wireDigest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sequence != 42 || segmentDigest != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || batchDigest != "" || wireDigest != "" {
|
||||
t.Fatalf("sequence=%d segment=%q batch=%q wire=%q", sequence, segmentDigest, batchDigest, wireDigest)
|
||||
}
|
||||
}
|
||||
|
||||
func openSchemaDatabase(t *testing.T, version int) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "control.sqlite"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL); INSERT INTO schema_version(version) VALUES(?)`, version); err != nil {
|
||||
db.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type ProcessLock struct {
|
||||
file *os.File
|
||||
}
|
||||
|
||||
// AcquireProcessLock coordinates the long-running server and offline migration
|
||||
// commands. Servers hold a shared lock; projection rebuilds require exclusive
|
||||
// ownership and therefore cannot overlap a live server process.
|
||||
func AcquireProcessLock(root string, exclusive bool) (*ProcessLock, error) {
|
||||
if !filepath.IsAbs(root) || filepath.Clean(root) != root {
|
||||
return nil, errors.New("process lock root must be absolute and clean")
|
||||
}
|
||||
if err := os.MkdirAll(root, 0o700); err != nil {
|
||||
return nil, errors.New("create process lock root")
|
||||
}
|
||||
rootInfo, err := os.Lstat(root)
|
||||
if err != nil || !rootInfo.IsDir() || rootInfo.Mode()&os.ModeSymlink != 0 || rootInfo.Mode().Perm()&0o077 != 0 {
|
||||
return nil, errors.New("process lock root must be a private non-symlink directory")
|
||||
}
|
||||
path := filepath.Join(root, "process.lock")
|
||||
fd, err := syscall.Open(path, syscall.O_RDWR|syscall.O_CREAT|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0o600)
|
||||
if err != nil {
|
||||
return nil, errors.New("open process lock")
|
||||
}
|
||||
file := os.NewFile(uintptr(fd), path)
|
||||
if file == nil {
|
||||
_ = syscall.Close(fd)
|
||||
return nil, errors.New("open process lock")
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
_ = file.Close()
|
||||
return nil, errors.New("process lock must be a regular non-symlink file")
|
||||
}
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok || stat.Nlink != 1 {
|
||||
_ = file.Close()
|
||||
return nil, errors.New("process lock must not have additional hard links")
|
||||
}
|
||||
operation := syscall.LOCK_SH | syscall.LOCK_NB
|
||||
if exclusive {
|
||||
operation = syscall.LOCK_EX | syscall.LOCK_NB
|
||||
}
|
||||
if err = syscall.Flock(fd, operation); err != nil {
|
||||
_ = file.Close()
|
||||
return nil, errors.New("Observatory data directory is active in another process")
|
||||
}
|
||||
if err = file.Chmod(0o600); err != nil {
|
||||
_ = syscall.Flock(fd, syscall.LOCK_UN)
|
||||
_ = file.Close()
|
||||
return nil, errors.New("secure process lock")
|
||||
}
|
||||
return &ProcessLock{file: file}, nil
|
||||
}
|
||||
|
||||
func (lock *ProcessLock) Close() error {
|
||||
if lock == nil || lock.file == nil {
|
||||
return nil
|
||||
}
|
||||
err := syscall.Flock(int(lock.file.Fd()), syscall.LOCK_UN)
|
||||
closeErr := lock.file.Close()
|
||||
lock.file = nil
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProcessLockSeparatesLiveServerFromOfflineMigration(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "data")
|
||||
first, err := AcquireProcessLock(root, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer first.Close()
|
||||
second, err := AcquireProcessLock(root, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = AcquireProcessLock(root, true); err == nil {
|
||||
t.Fatal("exclusive migration lock overlapped live server locks")
|
||||
}
|
||||
if err = second.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = first.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exclusive, err := AcquireProcessLock(root, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer exclusive.Close()
|
||||
if _, err = AcquireProcessLock(root, false); err == nil {
|
||||
t.Fatal("server lock overlapped exclusive migration lock")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessLockRejectsUnsafeFilesystemObjects(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
private := filepath.Join(base, "private")
|
||||
if err := os.Mkdir(private, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
symlinkRoot := filepath.Join(base, "symlink-root")
|
||||
if err := os.Symlink(private, symlinkRoot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := AcquireProcessLock(symlinkRoot, false); err == nil {
|
||||
t.Fatal("symlink process-lock root was accepted")
|
||||
}
|
||||
public := filepath.Join(base, "public")
|
||||
if err := os.Mkdir(public, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := AcquireProcessLock(public, false); err == nil {
|
||||
t.Fatal("public process-lock root was accepted")
|
||||
}
|
||||
lockPath := filepath.Join(private, "process.lock")
|
||||
target := filepath.Join(private, "target")
|
||||
if err := os.WriteFile(target, []byte("target"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(target, lockPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := AcquireProcessLock(private, false); err == nil {
|
||||
t.Fatal("symlink process-lock file was accepted")
|
||||
}
|
||||
if err := os.Remove(lockPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Link(target, lockPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := AcquireProcessLock(private, false); err == nil {
|
||||
t.Fatal("hard-linked process-lock file was accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
projectionGroupMaxSegments = 16
|
||||
projectionGroupMaxBytes = 32 << 20
|
||||
projectionWorkerLimit = 4
|
||||
defaultProjectorInterval = time.Second
|
||||
)
|
||||
|
||||
// ProjectionReport describes one bounded projector pass. Accepted raw
|
||||
// segments remain durable and replayable independently of this report.
|
||||
type ProjectionReport struct {
|
||||
ProjectedSegments int `json:"projected_segments"`
|
||||
ProjectedRecords int `json:"projected_records"`
|
||||
ProjectedBytes int64 `json:"projected_bytes"`
|
||||
}
|
||||
|
||||
// ProjectionStatus makes asynchronous query visibility explicit. Lag is the
|
||||
// age of the oldest durable segment that has not yet reached the read model.
|
||||
type ProjectionStatus struct {
|
||||
PendingSegments int `json:"pending_segments"`
|
||||
PendingBytes int64 `json:"pending_bytes"`
|
||||
OldestCommitted time.Time `json:"oldest_committed_at,omitempty"`
|
||||
OldestPendingLag time.Duration `json:"oldest_pending_lag"`
|
||||
}
|
||||
|
||||
type pendingProjection struct {
|
||||
digest string
|
||||
path string
|
||||
uncompressedBytes int64
|
||||
sourceID string
|
||||
streamID string
|
||||
sequence uint64
|
||||
committedAt time.Time
|
||||
catalogOrgID string
|
||||
scope model.Scope
|
||||
}
|
||||
|
||||
type pendingProjectionGroup struct {
|
||||
organizationID string
|
||||
segments []pendingProjection
|
||||
bytes int64
|
||||
}
|
||||
|
||||
// ProjectionStatus returns a bounded control-database view and never opens an
|
||||
// organization projection.
|
||||
func (s *Store) ProjectionStatus(ctx context.Context, now time.Time) (ProjectionStatus, error) {
|
||||
return s.projectionStatus(ctx, "", now)
|
||||
}
|
||||
|
||||
// OrganizationProjectionStatus restricts lag evidence to one tenant so the
|
||||
// authenticated UI never discloses another organization's ingestion volume.
|
||||
func (s *Store) OrganizationProjectionStatus(ctx context.Context, organizationID string, now time.Time) (ProjectionStatus, error) {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil {
|
||||
return ProjectionStatus{}, errors.New("invalid organization identifier")
|
||||
}
|
||||
return s.projectionStatus(ctx, organizationID, now)
|
||||
}
|
||||
|
||||
func (s *Store) projectionStatus(ctx context.Context, organizationID string, now time.Time) (ProjectionStatus, error) {
|
||||
if now.IsZero() {
|
||||
return ProjectionStatus{}, errors.New("projection status time is required")
|
||||
}
|
||||
var status ProjectionStatus
|
||||
var oldest sql.NullString
|
||||
var err error
|
||||
if organizationID == "" {
|
||||
err = s.control.QueryRowContext(ctx, `SELECT COUNT(*),COALESCE(SUM(uncompressed_bytes),0),MIN(committed_at) FROM segments WHERE projected_at IS NULL`).Scan(&status.PendingSegments, &status.PendingBytes, &oldest)
|
||||
} else {
|
||||
err = s.control.QueryRowContext(ctx, `SELECT COUNT(*),COALESCE(SUM(uncompressed_bytes),0),MIN(committed_at) FROM segments WHERE projected_at IS NULL AND organization_id=?`, organizationID).Scan(&status.PendingSegments, &status.PendingBytes, &oldest)
|
||||
}
|
||||
if err != nil {
|
||||
return ProjectionStatus{}, fmt.Errorf("read projection status: %w", err)
|
||||
}
|
||||
if oldest.Valid {
|
||||
parsed, parseErr := time.Parse(time.RFC3339Nano, oldest.String)
|
||||
if parseErr != nil {
|
||||
return ProjectionStatus{}, errors.New("projection status contains an invalid timestamp")
|
||||
}
|
||||
status.OldestCommitted = parsed.UTC()
|
||||
if now.After(status.OldestCommitted) {
|
||||
status.OldestPendingLag = now.Sub(status.OldestCommitted)
|
||||
}
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
|
||||
// ProjectPending projects a bounded set of already-durable segments. It
|
||||
// groups work by organization so one SQLite transaction can safely amortize
|
||||
// multiple agent batches without crossing tenant databases.
|
||||
func (s *Store) ProjectPending(ctx context.Context) (ProjectionReport, error) {
|
||||
s.projectorMu.Lock()
|
||||
defer s.projectorMu.Unlock()
|
||||
|
||||
pending, err := s.pendingProjections(ctx)
|
||||
if err != nil || len(pending) == 0 {
|
||||
return ProjectionReport{}, err
|
||||
}
|
||||
groups := boundedProjectionGroups(pending)
|
||||
type projectionResult struct {
|
||||
report ProjectionReport
|
||||
err error
|
||||
}
|
||||
results := make([]projectionResult, len(groups))
|
||||
workers := min(len(groups), projectionWorkerLimit)
|
||||
jobs := make(chan int)
|
||||
var wait sync.WaitGroup
|
||||
wait.Add(workers)
|
||||
for range workers {
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
for index := range jobs {
|
||||
results[index].report, results[index].err = s.projectPendingGroup(ctx, groups[index])
|
||||
}
|
||||
}()
|
||||
}
|
||||
for index := range groups {
|
||||
jobs <- index
|
||||
}
|
||||
close(jobs)
|
||||
wait.Wait()
|
||||
|
||||
var report ProjectionReport
|
||||
var projectionErrors []error
|
||||
for _, result := range results {
|
||||
if result.err != nil {
|
||||
projectionErrors = append(projectionErrors, result.err)
|
||||
continue
|
||||
}
|
||||
report.ProjectedSegments += result.report.ProjectedSegments
|
||||
report.ProjectedRecords += result.report.ProjectedRecords
|
||||
report.ProjectedBytes += result.report.ProjectedBytes
|
||||
}
|
||||
return report, errors.Join(projectionErrors...)
|
||||
}
|
||||
|
||||
func (s *Store) pendingProjections(ctx context.Context) ([]pendingProjection, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `WITH ranked AS (
|
||||
SELECT g.digest,g.path,g.uncompressed_bytes,g.source_id,g.stream_id,g.sequence,g.committed_at,g.organization_id,
|
||||
s.organization_id AS source_organization_id,s.project_id,s.environment_id,s.service_id,
|
||||
ROW_NUMBER() OVER (PARTITION BY g.organization_id ORDER BY g.committed_at,g.digest) AS organization_rank
|
||||
FROM segments AS g JOIN sources AS s ON s.id=g.source_id
|
||||
WHERE g.projected_at IS NULL
|
||||
)
|
||||
SELECT digest,path,uncompressed_bytes,source_id,stream_id,sequence,committed_at,organization_id,source_organization_id,project_id,environment_id,service_id
|
||||
FROM ranked WHERE organization_rank<=? ORDER BY committed_at,digest LIMIT ?`, projectionGroupMaxSegments, recoveryPageSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list unprojected segments: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
pending := make([]pendingProjection, 0, recoveryPageSize)
|
||||
for rows.Next() {
|
||||
var item pendingProjection
|
||||
var committedAt string
|
||||
if err = rows.Scan(&item.digest, &item.path, &item.uncompressedBytes, &item.sourceID, &item.streamID, &item.sequence, &committedAt, &item.catalogOrgID, &item.scope.OrganizationID, &item.scope.ProjectID, &item.scope.EnvironmentID, &item.scope.ServiceID); err != nil {
|
||||
return nil, fmt.Errorf("scan unprojected segment: %w", err)
|
||||
}
|
||||
item.committedAt, err = time.Parse(time.RFC3339Nano, committedAt)
|
||||
if err != nil {
|
||||
return nil, errors.New("unprojected segment has an invalid committed time")
|
||||
}
|
||||
pending = append(pending, item)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate unprojected segments: %w", err)
|
||||
}
|
||||
return pending, nil
|
||||
}
|
||||
|
||||
func boundedProjectionGroups(pending []pendingProjection) []pendingProjectionGroup {
|
||||
groups := make([]pendingProjectionGroup, 0)
|
||||
byOrganization := make(map[string]int)
|
||||
for _, item := range pending {
|
||||
index, exists := byOrganization[item.scope.OrganizationID]
|
||||
if !exists {
|
||||
index = len(groups)
|
||||
byOrganization[item.scope.OrganizationID] = index
|
||||
groups = append(groups, pendingProjectionGroup{organizationID: item.scope.OrganizationID})
|
||||
}
|
||||
group := &groups[index]
|
||||
if len(group.segments) >= projectionGroupMaxSegments {
|
||||
continue
|
||||
}
|
||||
if len(group.segments) > 0 && group.bytes+item.uncompressedBytes > projectionGroupMaxBytes {
|
||||
continue
|
||||
}
|
||||
group.segments = append(group.segments, item)
|
||||
group.bytes += item.uncompressedBytes
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func (s *Store) projectPendingGroup(ctx context.Context, group pendingProjectionGroup) (ProjectionReport, error) {
|
||||
if len(group.segments) == 0 {
|
||||
return ProjectionReport{}, nil
|
||||
}
|
||||
lock := s.namedLock("organization:" + group.organizationID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
items := make([]projectionItem, 0, len(group.segments))
|
||||
report := ProjectionReport{ProjectedSegments: len(group.segments), ProjectedBytes: group.bytes}
|
||||
for _, pending := range group.segments {
|
||||
batch, err := s.segments.Read(pending.path, pending.digest)
|
||||
if err != nil {
|
||||
return ProjectionReport{}, fmt.Errorf("read pending projection segment: %w", err)
|
||||
}
|
||||
if batch.SourceID != pending.sourceID || batch.StreamID != pending.streamID || batch.Sequence != pending.sequence || pending.catalogOrgID != group.organizationID || pending.scope.OrganizationID != group.organizationID {
|
||||
return ProjectionReport{}, errors.New("pending projection identity does not match durable catalog")
|
||||
}
|
||||
if err = batch.Validate(batch.ObservedAt); err != nil {
|
||||
return ProjectionReport{}, fmt.Errorf("validate pending projection batch: %w", err)
|
||||
}
|
||||
if err = validateMetricRollupCardinality(batch); err != nil {
|
||||
return ProjectionReport{}, fmt.Errorf("validate pending projection cardinality: %w", err)
|
||||
}
|
||||
report.ProjectedRecords += len(batch.Records)
|
||||
items = append(items, projectionItem{scope: pending.scope, batch: batch, digest: pending.digest})
|
||||
}
|
||||
db, err := s.projection(ctx, group.organizationID)
|
||||
if err != nil {
|
||||
return ProjectionReport{}, err
|
||||
}
|
||||
if err = projectGroupWithDB(ctx, db, items); err != nil {
|
||||
return ProjectionReport{}, err
|
||||
}
|
||||
projectedAt := time.Now().UTC()
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return ProjectionReport{}, fmt.Errorf("begin projection acknowledgement: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for index, item := range items {
|
||||
if err = recordDescriptorProposalsTx(ctx, tx, group.organizationID, item.batch, item.digest, group.segments[index].committedAt); err != nil {
|
||||
return ProjectionReport{}, err
|
||||
}
|
||||
if err = markProjectedTx(ctx, tx, item.batch, item.digest, projectedAt); err != nil {
|
||||
return ProjectionReport{}, err
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return ProjectionReport{}, fmt.Errorf("commit projection acknowledgement: %w", err)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// RunProjector continuously drains durable work, then sleeps until ingestion
|
||||
// wakes it or the reconciliation interval expires. Errors leave raw segments
|
||||
// pending and are retried without making acknowledgement availability depend
|
||||
// on query projection health.
|
||||
func (s *Store) RunProjector(ctx context.Context, interval time.Duration, onError func(error)) {
|
||||
if interval <= 0 {
|
||||
interval = defaultProjectorInterval
|
||||
}
|
||||
timer := time.NewTimer(0)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
case <-s.projectionWake:
|
||||
}
|
||||
report, err := s.ProjectPending(ctx)
|
||||
if err != nil && !errors.Is(err, context.Canceled) && onError != nil {
|
||||
onError(err)
|
||||
}
|
||||
delay := interval
|
||||
if err == nil && report.ProjectedSegments > 0 {
|
||||
delay = 0
|
||||
}
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
timer.Reset(delay)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
func TestDurableAcknowledgementPrecedesProjection(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Now().UTC()
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
|
||||
ack, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil || ack.Duplicate {
|
||||
t.Fatalf("ack=%+v err=%v", ack, err)
|
||||
}
|
||||
duplicate, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil || !duplicate.Duplicate || duplicate.Digest != ack.Digest {
|
||||
t.Fatalf("duplicate=%+v err=%v", duplicate, err)
|
||||
}
|
||||
status, err := store.ProjectionStatus(ctx, now.Add(time.Second))
|
||||
if err != nil || status.PendingSegments != 1 || status.PendingBytes < 1 || status.OldestPendingLag < time.Second {
|
||||
t.Fatalf("status=%+v err=%v", status, err)
|
||||
}
|
||||
var projected int
|
||||
if err = store.control.QueryRowContext(ctx, `SELECT COUNT(projected_at) FROM segments`).Scan(&projected); err != nil || projected != 0 {
|
||||
t.Fatalf("projected=%d err=%v", projected, err)
|
||||
}
|
||||
report, err := store.ProjectPending(ctx)
|
||||
if err != nil || report.ProjectedSegments != 1 || report.ProjectedRecords != 1 || report.ProjectedBytes < 1 {
|
||||
t.Fatalf("report=%+v err=%v", report, err)
|
||||
}
|
||||
status, err = store.ProjectionStatus(ctx, now.Add(2*time.Second))
|
||||
if err != nil || status.PendingSegments != 0 || status.PendingBytes != 0 || !status.OldestCommitted.IsZero() || status.OldestPendingLag != 0 {
|
||||
t.Fatalf("projected status=%+v err=%v", status, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectorGroupsBoundedSegmentsAndResumesAfterReopen(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
store := testStore(t)
|
||||
root := store.root
|
||||
now := time.Now().UTC()
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const total = projectionGroupMaxSegments + 3
|
||||
for sequence := uint64(1); sequence <= total; sequence++ {
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: sequence, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
first, err := store.ProjectPending(ctx)
|
||||
if err != nil || first.ProjectedSegments != projectionGroupMaxSegments {
|
||||
t.Fatalf("first=%+v err=%v", first, err)
|
||||
}
|
||||
if err = store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err = Open(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if err = store.RecoverRaw(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, err := store.ProjectionStatus(ctx, now.Add(time.Second))
|
||||
if err != nil || status.PendingSegments != total-projectionGroupMaxSegments {
|
||||
t.Fatalf("status=%+v err=%v", status, err)
|
||||
}
|
||||
second, err := store.ProjectPending(ctx)
|
||||
if err != nil || second.ProjectedSegments != total-projectionGroupMaxSegments {
|
||||
t.Fatalf("second=%+v err=%v", second, err)
|
||||
}
|
||||
ast, err := query.Parse(`logs | limit 100`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now)
|
||||
if err != nil || len(result.Rows) != total {
|
||||
t.Fatalf("rows=%d err=%v", len(result.Rows), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectorFailureIsolatedFromAcknowledgementAndOtherOrganization(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Now().UTC()
|
||||
for index, organizationID := range []string{"organization-a", "organization-b"} {
|
||||
sourceID := "source-" + string(rune('a'+index))
|
||||
scope := model.Scope{OrganizationID: organizationID, ProjectID: "project", EnvironmentID: "production", ServiceID: "service"}
|
||||
token, err := store.CreateSource(ctx, sourceID, scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: sourceID, StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var corruptPath string
|
||||
if err := store.control.QueryRowContext(ctx, `SELECT path FROM segments WHERE organization_id='organization-a'`).Scan(&corruptPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
corruptBody, err := os.ReadFile(corruptPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
corruptBody[0] ^= 0xff
|
||||
if err := os.WriteFile(corruptPath, corruptBody, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err := store.ProjectPending(ctx)
|
||||
if err == nil || report.ProjectedSegments != 1 {
|
||||
t.Fatalf("report=%+v err=%v", report, err)
|
||||
}
|
||||
var projectedA, projectedB int
|
||||
if err = store.control.QueryRowContext(ctx, `SELECT COUNT(projected_at) FROM segments WHERE organization_id='organization-a'`).Scan(&projectedA); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.control.QueryRowContext(ctx, `SELECT COUNT(projected_at) FROM segments WHERE organization_id='organization-b'`).Scan(&projectedB); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if projectedA != 0 || projectedB != 1 {
|
||||
t.Fatalf("projected organization-a=%d organization-b=%d", projectedA, projectedB)
|
||||
}
|
||||
if err = store.RecoverRaw(ctx); err != nil {
|
||||
t.Fatalf("raw reconciliation decoded catalogued pending evidence: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProjectorWakesOnAcceptedBatch(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
store.RunProjector(ctx, time.Minute, func(err error) { t.Errorf("projector: %v", err) })
|
||||
}()
|
||||
now := time.Now().UTC()
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
status, statusErr := store.ProjectionStatus(t.Context(), time.Now().UTC())
|
||||
if statusErr != nil {
|
||||
t.Fatal(statusErr)
|
||||
}
|
||||
if status.PendingSegments == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("projection remained pending: %+v", status)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("projector did not stop after cancellation")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxPushSubscriptionsPerUser = 8
|
||||
MaxPushSubscriptionsPerPass = 256
|
||||
)
|
||||
|
||||
type PushSubscription struct {
|
||||
OrganizationID string
|
||||
ID string
|
||||
UserID string
|
||||
Endpoint string
|
||||
P256DH []byte
|
||||
Auth []byte
|
||||
FailureCount int
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
LastSentAt *time.Time
|
||||
}
|
||||
|
||||
type PushSubscriptionInput struct {
|
||||
OrganizationID string
|
||||
UserID string
|
||||
Endpoint string
|
||||
P256DH []byte
|
||||
Auth []byte
|
||||
}
|
||||
|
||||
func (s *Store) SavePushSubscription(ctx context.Context, input PushSubscriptionInput, now time.Time) (PushSubscription, error) {
|
||||
if err := validatePushSubscription(input); err != nil {
|
||||
return PushSubscription{}, err
|
||||
}
|
||||
digest := sha256.Sum256([]byte(input.Endpoint))
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return PushSubscription{}, errors.New("save push subscription")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var endpointID, endpointUser string
|
||||
err = tx.QueryRowContext(ctx, `SELECT id,user_id FROM push_endpoints WHERE endpoint_digest=?`, digest[:]).Scan(&endpointID, &endpointUser)
|
||||
switch {
|
||||
case err == nil:
|
||||
if endpointUser != input.UserID {
|
||||
return PushSubscription{}, errors.New("push subscription is already registered")
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE push_endpoints SET endpoint=?,p256dh=?,auth_secret=?,active=1,failure_count=0,updated_at=? WHERE id=?`, input.Endpoint, input.P256DH, input.Auth, now.UTC().Format(time.RFC3339Nano), endpointID)
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
endpointID, err = storageID("endpoint")
|
||||
if err == nil {
|
||||
stamp := now.UTC().Format(time.RFC3339Nano)
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO push_endpoints(id,user_id,endpoint,endpoint_digest,p256dh,auth_secret,active,failure_count,created_at,updated_at) VALUES(?,?,?,?,?,?,1,0,?,?)`, endpointID, input.UserID, input.Endpoint, digest[:], input.P256DH, input.Auth, stamp, stamp)
|
||||
}
|
||||
default:
|
||||
return PushSubscription{}, errors.New("save push subscription")
|
||||
}
|
||||
if err != nil {
|
||||
return PushSubscription{}, errors.New("save push subscription")
|
||||
}
|
||||
var subscriptionID string
|
||||
err = tx.QueryRowContext(ctx, `SELECT id FROM push_subscriptions WHERE organization_id=? AND user_id=? AND endpoint_id=?`, input.OrganizationID, input.UserID, endpointID).Scan(&subscriptionID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
var count int
|
||||
if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM push_subscriptions WHERE organization_id=? AND user_id=?`, input.OrganizationID, input.UserID).Scan(&count); err != nil {
|
||||
return PushSubscription{}, errors.New("save push subscription")
|
||||
}
|
||||
if count >= MaxPushSubscriptionsPerUser {
|
||||
return PushSubscription{}, errors.New("push subscription limit reached")
|
||||
}
|
||||
subscriptionID, err = storageID("push")
|
||||
if err == nil {
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO push_subscriptions(organization_id,id,user_id,endpoint_id,created_at) VALUES(?,?,?,?,?)`, input.OrganizationID, subscriptionID, input.UserID, endpointID, now.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return PushSubscription{}, errors.New("save push subscription")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return PushSubscription{}, errors.New("save push subscription")
|
||||
}
|
||||
return s.PushSubscription(ctx, input.OrganizationID, subscriptionID)
|
||||
}
|
||||
|
||||
func (s *Store) HasPushSubscription(ctx context.Context, organizationID, userID, endpoint string) (bool, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(userID) != nil || !utf8.ValidString(endpoint) || len(endpoint) < 1 || len(endpoint) > 2048 {
|
||||
return false, errors.New("push subscription lookup is invalid")
|
||||
}
|
||||
digest := sha256.Sum256([]byte(endpoint))
|
||||
var count int
|
||||
err := s.control.QueryRowContext(ctx, `SELECT COUNT(*) FROM push_subscriptions s JOIN push_endpoints e ON e.id=s.endpoint_id WHERE s.organization_id=? AND s.user_id=? AND e.endpoint_digest=? AND e.active=1`, organizationID, userID, digest[:]).Scan(&count)
|
||||
if err != nil {
|
||||
return false, errors.New("lookup push subscription")
|
||||
}
|
||||
return count == 1, nil
|
||||
}
|
||||
|
||||
// DeletePushSubscription removes one organization mapping. The returned
|
||||
// value reports whether the browser endpoint remains mapped elsewhere for
|
||||
// the same user and therefore must remain subscribed in the user agent.
|
||||
func (s *Store) DeletePushSubscription(ctx context.Context, organizationID, userID, endpoint string) (bool, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(userID) != nil || !utf8.ValidString(endpoint) || len(endpoint) < 1 || len(endpoint) > 2048 {
|
||||
return false, errors.New("push subscription deletion is invalid")
|
||||
}
|
||||
digest := sha256.Sum256([]byte(endpoint))
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return false, errors.New("delete push subscription")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var endpointID string
|
||||
err = tx.QueryRowContext(ctx, `SELECT e.id FROM push_subscriptions s JOIN push_endpoints e ON e.id=s.endpoint_id WHERE s.organization_id=? AND s.user_id=? AND e.endpoint_digest=?`, organizationID, userID, digest[:]).Scan(&endpointID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, errors.New("push subscription not found")
|
||||
}
|
||||
if err != nil {
|
||||
return false, errors.New("delete push subscription")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM push_subscriptions WHERE organization_id=? AND user_id=? AND endpoint_id=?`, organizationID, userID, endpointID); err != nil {
|
||||
return false, errors.New("delete push subscription")
|
||||
}
|
||||
var remaining int
|
||||
if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM push_subscriptions WHERE endpoint_id=?`, endpointID).Scan(&remaining); err != nil {
|
||||
return false, errors.New("delete push subscription")
|
||||
}
|
||||
if remaining == 0 {
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM push_endpoints WHERE id=?`, endpointID); err != nil {
|
||||
return false, errors.New("delete push subscription")
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return false, errors.New("delete push subscription")
|
||||
}
|
||||
return remaining > 0, nil
|
||||
}
|
||||
|
||||
func (s *Store) PushSubscription(ctx context.Context, organizationID, id string) (PushSubscription, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(id) != nil {
|
||||
return PushSubscription{}, errors.New("push subscription identity is invalid")
|
||||
}
|
||||
return scanPushSubscription(s.control.QueryRowContext(ctx, `SELECT s.organization_id,s.id,s.user_id,e.endpoint,e.p256dh,e.auth_secret,e.failure_count,s.created_at,e.updated_at,e.last_sent_at FROM push_subscriptions s JOIN push_endpoints e ON e.id=s.endpoint_id WHERE s.organization_id=? AND s.id=? AND e.active=1`, organizationID, id))
|
||||
}
|
||||
|
||||
func (s *Store) PushSubscriptions(ctx context.Context, organizationID string) ([]PushSubscription, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil {
|
||||
return nil, errors.New("organization identity is invalid")
|
||||
}
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT s.organization_id,s.id,s.user_id,e.endpoint,e.p256dh,e.auth_secret,e.failure_count,s.created_at,e.updated_at,e.last_sent_at FROM push_subscriptions s JOIN push_endpoints e ON e.id=s.endpoint_id WHERE s.organization_id=? AND e.active=1 ORDER BY s.id LIMIT ?`, organizationID, MaxPushSubscriptionsPerPass)
|
||||
if err != nil {
|
||||
return nil, errors.New("list push subscriptions")
|
||||
}
|
||||
defer rows.Close()
|
||||
var subscriptions []PushSubscription
|
||||
for rows.Next() {
|
||||
subscription, scanErr := scanPushSubscription(rows)
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
subscriptions = append(subscriptions, subscription)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, errors.New("list push subscriptions")
|
||||
}
|
||||
return subscriptions, nil
|
||||
}
|
||||
|
||||
func (s *Store) RecordPushResult(ctx context.Context, organizationID, id string, outcome string, now time.Time) error {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(id) != nil {
|
||||
return errors.New("push subscription identity is invalid")
|
||||
}
|
||||
var result sql.Result
|
||||
var err error
|
||||
switch outcome {
|
||||
case "sent":
|
||||
result, err = s.control.ExecContext(ctx, `UPDATE push_endpoints SET failure_count=0,last_sent_at=?,updated_at=? WHERE id=(SELECT endpoint_id FROM push_subscriptions WHERE organization_id=? AND id=?) AND active=1`, now.UTC().Format(time.RFC3339Nano), now.UTC().Format(time.RFC3339Nano), organizationID, id)
|
||||
case "gone":
|
||||
result, err = s.control.ExecContext(ctx, `UPDATE push_endpoints SET active=0,updated_at=? WHERE id=(SELECT endpoint_id FROM push_subscriptions WHERE organization_id=? AND id=?) AND active=1`, now.UTC().Format(time.RFC3339Nano), organizationID, id)
|
||||
case "failed":
|
||||
result, err = s.control.ExecContext(ctx, `UPDATE push_endpoints SET failure_count=failure_count+1,active=CASE WHEN failure_count+1>=5 THEN 0 ELSE 1 END,updated_at=? WHERE id=(SELECT endpoint_id FROM push_subscriptions WHERE organization_id=? AND id=?) AND active=1`, now.UTC().Format(time.RFC3339Nano), organizationID, id)
|
||||
default:
|
||||
return errors.New("push delivery outcome is invalid")
|
||||
}
|
||||
if err != nil {
|
||||
return errors.New("record push delivery result")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return errors.New("push subscription not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanPushSubscription(row rowScanner) (PushSubscription, error) {
|
||||
var subscription PushSubscription
|
||||
var createdAt, updatedAt string
|
||||
var lastSent sql.NullString
|
||||
if err := row.Scan(&subscription.OrganizationID, &subscription.ID, &subscription.UserID, &subscription.Endpoint, &subscription.P256DH, &subscription.Auth, &subscription.FailureCount, &createdAt, &updatedAt, &lastSent); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return PushSubscription{}, errors.New("push subscription not found")
|
||||
}
|
||||
return PushSubscription{}, errors.New("read push subscription")
|
||||
}
|
||||
var err error
|
||||
if subscription.CreatedAt, err = time.Parse(time.RFC3339Nano, createdAt); err != nil {
|
||||
return PushSubscription{}, errors.New("read push subscription")
|
||||
}
|
||||
if subscription.UpdatedAt, err = time.Parse(time.RFC3339Nano, updatedAt); err != nil {
|
||||
return PushSubscription{}, errors.New("read push subscription")
|
||||
}
|
||||
if lastSent.Valid {
|
||||
parsed, parseErr := time.Parse(time.RFC3339Nano, lastSent.String)
|
||||
if parseErr != nil {
|
||||
return PushSubscription{}, errors.New("read push subscription")
|
||||
}
|
||||
subscription.LastSentAt = &parsed
|
||||
}
|
||||
return subscription, nil
|
||||
}
|
||||
|
||||
func validatePushSubscription(input PushSubscriptionInput) error {
|
||||
if model.ValidateSourceID(input.OrganizationID) != nil || model.ValidateSourceID(input.UserID) != nil {
|
||||
return errors.New("push subscription scope is invalid")
|
||||
}
|
||||
if !utf8.ValidString(input.Endpoint) || len(input.Endpoint) < 1 || len(input.Endpoint) > 2048 || len(input.P256DH) != 65 || len(input.Auth) != 16 {
|
||||
return errors.New("push subscription material is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPushSubscriptionLifecycleAndOwnership(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC)
|
||||
input := pushInput("organization-a", "user-a", "https://push.example.test/send/one")
|
||||
created, err := store.SavePushSubscription(context.Background(), input, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if created.OrganizationID != input.OrganizationID || created.UserID != input.UserID || created.Endpoint != input.Endpoint || created.FailureCount != 0 {
|
||||
t.Fatalf("created=%+v", created)
|
||||
}
|
||||
input.P256DH[10] ^= 0xff
|
||||
updated, err := store.SavePushSubscription(context.Background(), input, now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if updated.ID != created.ID || updated.P256DH[10] != input.P256DH[10] {
|
||||
t.Fatalf("updated=%+v", updated)
|
||||
}
|
||||
claimed := input
|
||||
claimed.UserID = "user-b"
|
||||
if _, err = store.SavePushSubscription(context.Background(), claimed, now); err == nil {
|
||||
t.Fatal("another user claimed an existing endpoint")
|
||||
}
|
||||
if _, err = store.DeletePushSubscription(context.Background(), input.OrganizationID, "user-b", input.Endpoint); err == nil {
|
||||
t.Fatal("another user deleted an existing endpoint")
|
||||
}
|
||||
otherOrganization := input
|
||||
otherOrganization.OrganizationID = "organization-b"
|
||||
other, err := store.SavePushSubscription(context.Background(), otherOrganization, now.Add(2*time.Minute))
|
||||
if err != nil || other.ID == created.ID {
|
||||
t.Fatalf("other organization=%+v err=%v", other, err)
|
||||
}
|
||||
for _, organizationID := range []string{input.OrganizationID, otherOrganization.OrganizationID} {
|
||||
if subscribed, statusErr := store.HasPushSubscription(context.Background(), organizationID, input.UserID, input.Endpoint); statusErr != nil || !subscribed {
|
||||
t.Fatalf("organization=%s subscribed=%t err=%v", organizationID, subscribed, statusErr)
|
||||
}
|
||||
}
|
||||
remaining, err := store.DeletePushSubscription(context.Background(), input.OrganizationID, input.UserID, input.Endpoint)
|
||||
if err != nil || !remaining {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if subscriptions, listErr := store.PushSubscriptions(context.Background(), input.OrganizationID); listErr != nil || len(subscriptions) != 0 {
|
||||
t.Fatalf("subscriptions=%+v err=%v", subscriptions, listErr)
|
||||
}
|
||||
remaining, err = store.DeletePushSubscription(context.Background(), otherOrganization.OrganizationID, input.UserID, input.Endpoint)
|
||||
if err != nil || remaining {
|
||||
t.Fatalf("remaining=%t err=%v", remaining, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPushDeliveryResultsAreBounded(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC)
|
||||
created, err := store.SavePushSubscription(context.Background(), pushInput("organization-a", "user-a", "https://push.example.test/send/result"), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for attempt := 1; attempt <= 4; attempt++ {
|
||||
if err = store.RecordPushResult(context.Background(), created.OrganizationID, created.ID, "failed", now.Add(time.Duration(attempt)*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, currentErr := store.PushSubscription(context.Background(), created.OrganizationID, created.ID)
|
||||
if currentErr != nil || current.FailureCount != attempt {
|
||||
t.Fatalf("attempt=%d current=%+v err=%v", attempt, current, currentErr)
|
||||
}
|
||||
}
|
||||
if err = store.RecordPushResult(context.Background(), created.OrganizationID, created.ID, "sent", now.Add(5*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := store.PushSubscription(context.Background(), created.OrganizationID, created.ID)
|
||||
if err != nil || current.FailureCount != 0 || current.LastSentAt == nil {
|
||||
t.Fatalf("sent current=%+v err=%v", current, err)
|
||||
}
|
||||
for attempt := 0; attempt < 5; attempt++ {
|
||||
if err = store.RecordPushResult(context.Background(), created.OrganizationID, created.ID, "failed", now.Add(time.Duration(6+attempt)*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err = store.PushSubscription(context.Background(), created.OrganizationID, created.ID); err == nil {
|
||||
t.Fatal("five delivery failures did not disable the subscription")
|
||||
}
|
||||
if subscriptions, listErr := store.PushSubscriptions(context.Background(), created.OrganizationID); listErr != nil || len(subscriptions) != 0 {
|
||||
t.Fatalf("subscriptions=%+v err=%v", subscriptions, listErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPushSubscriptionLimit(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
var err error
|
||||
for index := 0; index < MaxPushSubscriptionsPerUser; index++ {
|
||||
input := pushInput("organization-a", "user-a", "https://push.example.test/send/limit"+string(rune('a'+index)))
|
||||
if _, err = store.SavePushSubscription(context.Background(), input, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err = store.SavePushSubscription(context.Background(), pushInput("organization-a", "user-a", "https://push.example.test/send/overflow"), time.Now()); err == nil {
|
||||
t.Fatal("subscription limit was not enforced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlSchemaFiveMigratesToPushSchema(t *testing.T) {
|
||||
database := openSchemaDatabase(t, 5)
|
||||
defer database.Close()
|
||||
if err := migrateControl(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var version int
|
||||
if err := database.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != controlSchema {
|
||||
t.Fatalf("version=%d err=%v", version, err)
|
||||
}
|
||||
for _, name := range []string{"push_endpoints", "push_subscriptions"} {
|
||||
var table int
|
||||
if err := database.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, name).Scan(&table); err != nil || table != 1 {
|
||||
t.Fatalf("table=%s count=%d err=%v", name, table, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pushInput(organizationID, userID, endpoint string) PushSubscriptionInput {
|
||||
key := make([]byte, 65)
|
||||
auth := make([]byte, 16)
|
||||
_, _ = rand.Read(key)
|
||||
_, _ = rand.Read(auth)
|
||||
key[0] = 4
|
||||
return PushSubscriptionInput{OrganizationID: organizationID, UserID: userID, Endpoint: endpoint, P256DH: key, Auth: auth}
|
||||
}
|
||||
@@ -0,0 +1,862 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
type projectedRecord struct {
|
||||
projectID, environmentID, serviceID string
|
||||
sourceID, streamID string
|
||||
sequence uint64
|
||||
recordIndex int
|
||||
signal model.Signal
|
||||
timestamp time.Time
|
||||
name, severity, body string
|
||||
value *float64
|
||||
traceID, spanID, correlationID string
|
||||
attributes map[string]string
|
||||
}
|
||||
|
||||
// Query executes a validated typed AST against one organization projection.
|
||||
// Authorization scope is supplied by the server, never by stored telemetry.
|
||||
func (s *Store) Query(ctx context.Context, ast query.AST, scope query.Scope, budget query.Budget, now time.Time) (query.Result, error) {
|
||||
if now.IsZero() {
|
||||
return query.Result{}, errors.New("query time is required")
|
||||
}
|
||||
registry, activeVersion, err := s.ActiveDescriptors(ctx, scope.OrganizationID)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
coldSegments, coldEstimate, err := s.coldSegmentsForQuery(ctx, ast, scope, now)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
useMetricRollups := metricRollupQueryEligible(ast, registry) && len(coldSegments) == 0
|
||||
useLogRollups := indexedLogCountSummaryEligible(ast) && len(coldSegments) == 0
|
||||
var estimated int64
|
||||
if useMetricRollups {
|
||||
estimated, err = s.estimateMetricRollupBytes(ctx, scope, ast, now)
|
||||
} else if useLogRollups {
|
||||
estimated, err = s.estimateLogRollupBytes(ctx, scope, ast, now)
|
||||
} else {
|
||||
estimated, err = s.EstimateOrganizationBytes(scope.OrganizationID)
|
||||
if err == nil {
|
||||
// Record queries whose predicates are fully pushed into SQLite stop
|
||||
// after limit+1 rows. Their logical scan is therefore bounded by
|
||||
// the existing result-memory guard, not by the size of every signal
|
||||
// and index in the organization's projection file. Keep whole-file
|
||||
// planning for summaries, alternate sorts, regular expressions, and
|
||||
// cold-segment reads; execution continues to enforce the exact scan,
|
||||
// row, memory, and duration budgets in every case.
|
||||
if len(coldSegments) == 0 && projectionRowLimit(ast) > 0 && estimated > budget.MaxMemoryBytes {
|
||||
estimated = budget.MaxMemoryBytes
|
||||
}
|
||||
if coldEstimate > math.MaxInt64-estimated {
|
||||
return query.Result{}, errors.New("query scan estimate overflow")
|
||||
}
|
||||
estimated += coldEstimate
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
explain, err := query.Plan(ast, scope, registry, estimated, budget)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
columns, err := resultColumns(ast, registry)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
result := query.Result{Version: query.ResultVersion, Explain: explain, Columns: columns, Rows: []query.Row{}}
|
||||
if len(coldSegments) > 0 && !useMetricRollups {
|
||||
for _, source := range append([]string(nil), result.Explain.ProjectedSources...) {
|
||||
result.Explain.ProjectedSources = append(result.Explain.ProjectedSources, source+"/cold:raw")
|
||||
}
|
||||
}
|
||||
path := filepath.Join(s.root, "organizations", scope.OrganizationID, "projection.sqlite")
|
||||
info, err := os.Lstat(path)
|
||||
projectionExists := err == nil
|
||||
if (err != nil && !errors.Is(err, os.ErrNotExist)) || (projectionExists && (!info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0)) {
|
||||
return query.Result{}, errors.New("organization projection is unavailable")
|
||||
}
|
||||
if useMetricRollups {
|
||||
if !projectionExists {
|
||||
return result, nil
|
||||
}
|
||||
for index := range result.Explain.ProjectedSources {
|
||||
result.Explain.ProjectedSources[index] += "/rollup:5m"
|
||||
}
|
||||
return s.queryMetricRollups(ctx, path, ast, scope, registry, budget, now, result)
|
||||
}
|
||||
if projectionExists && useLogRollups {
|
||||
for index := range result.Explain.ProjectedSources {
|
||||
result.Explain.ProjectedSources[index] += "/rollup:http-status-route:5m"
|
||||
}
|
||||
return s.queryIndexedLogCountSummary(ctx, path, ast, scope, budget, now, result)
|
||||
}
|
||||
|
||||
runContext, cancel := context.WithTimeout(ctx, budget.MaxDuration)
|
||||
defer cancel()
|
||||
started := time.Now()
|
||||
var records []projectedRecord
|
||||
var memoryBytes int64
|
||||
earlyLimit := ast.Summary == nil && (ast.Sort == nil || query.CanonicalField(ast.Sort.Field) == "timestamp" && ast.Sort.Descending)
|
||||
if projectionExists {
|
||||
dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String()
|
||||
db, openErr := sql.Open("sqlite", dsn)
|
||||
if openErr != nil {
|
||||
return query.Result{}, errors.New("open organization projection")
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
statement, arguments, selectionErr := projectionSelection(ast, scope, registry, activeVersion, now)
|
||||
if selectionErr != nil {
|
||||
return query.Result{}, selectionErr
|
||||
}
|
||||
rows, queryErr := db.QueryContext(runContext, statement, arguments...)
|
||||
if queryErr != nil {
|
||||
return query.Result{}, queryExecutionError(runContext, queryErr)
|
||||
}
|
||||
for rows.Next() {
|
||||
record, readBytes, scanErr := scanProjected(rows)
|
||||
if scanErr != nil {
|
||||
_ = rows.Close()
|
||||
if runContext.Err() != nil {
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
return query.Result{}, errors.New("read organization projection")
|
||||
}
|
||||
if readBytes > budget.MaxScannedBytes-result.Stats.ScannedBytes {
|
||||
_ = rows.Close()
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
if result.Stats.ScannedRows == math.MaxInt64 {
|
||||
_ = rows.Close()
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
result.Stats.ScannedRows++
|
||||
result.Stats.ScannedBytes += readBytes
|
||||
matched, matchErr := matchesRecord(record, ast, registry)
|
||||
if matchErr != nil {
|
||||
_ = rows.Close()
|
||||
return query.Result{}, matchErr
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
if result.Stats.MatchedRows == math.MaxInt64 {
|
||||
_ = rows.Close()
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
result.Stats.MatchedRows++
|
||||
if readBytes+256 > budget.MaxMemoryBytes-memoryBytes {
|
||||
_ = rows.Close()
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
memoryBytes += readBytes + 256
|
||||
records = append(records, record)
|
||||
if earlyLimit && len(records) > ast.Limit {
|
||||
result.Stats.Truncated = true
|
||||
break
|
||||
}
|
||||
if err = runContext.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return query.Result{}, queryExecutionError(runContext, err)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return query.Result{}, errors.New("close organization projection query")
|
||||
}
|
||||
}
|
||||
if !(earlyLimit && result.Stats.Truncated) {
|
||||
records, memoryBytes, err = s.appendRawQuerySegments(runContext, coldSegments, ast, registry, budget, now, &result, records, memoryBytes)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
}
|
||||
if len(coldSegments) > 0 {
|
||||
sortRawQueryRecords(records)
|
||||
}
|
||||
return finishQueryResult(result, records, ast, columns, registry, memoryBytes, budget.MaxMemoryBytes, started)
|
||||
}
|
||||
|
||||
func (s *Store) appendRawQuerySegments(ctx context.Context, segments []rawQuerySegment, ast query.AST, registry query.Registry, budget query.Budget, now time.Time, result *query.Result, records []projectedRecord, memoryBytes int64) ([]projectedRecord, int64, error) {
|
||||
for _, segment := range segments {
|
||||
if segment.uncompressedBytes > budget.MaxMemoryBytes || segment.uncompressedBytes > budget.MaxScannedBytes-result.Stats.ScannedBytes {
|
||||
return nil, 0, query.ErrBudgetExceeded
|
||||
}
|
||||
batch, err := s.segments.Read(segment.path, segment.digest)
|
||||
if err != nil {
|
||||
return nil, 0, errors.New("read raw query segment")
|
||||
}
|
||||
if batch.SourceID != segment.sourceID || batch.StreamID != segment.streamID || batch.Sequence != segment.sequence || batch.Signal != ast.Signal || batch.Validate(batch.ObservedAt) != nil || validateMetricRollupCardinality(batch) != nil {
|
||||
return nil, 0, errors.New("raw query segment is invalid")
|
||||
}
|
||||
first, last := observationRange(batch)
|
||||
if !first.Equal(segment.firstObservedAt) || !last.Equal(segment.lastObservedAt) {
|
||||
return nil, 0, errors.New("raw query segment range does not match its catalog")
|
||||
}
|
||||
result.Stats.ScannedBytes += segment.uncompressedBytes
|
||||
for index := range batch.Records {
|
||||
if result.Stats.ScannedRows == math.MaxInt64 {
|
||||
return nil, 0, query.ErrBudgetExceeded
|
||||
}
|
||||
result.Stats.ScannedRows++
|
||||
record := rawRecord(segment, batch, index)
|
||||
if ast.Window > 0 && record.timestamp.Before(now.UTC().Add(-ast.Window)) {
|
||||
continue
|
||||
}
|
||||
matched, matchErr := matchesRecord(record, ast, registry)
|
||||
if matchErr != nil {
|
||||
return nil, 0, matchErr
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
if result.Stats.MatchedRows == math.MaxInt64 {
|
||||
return nil, 0, query.ErrBudgetExceeded
|
||||
}
|
||||
result.Stats.MatchedRows++
|
||||
recordMemory := rawRecordMemory(record)
|
||||
if recordMemory > budget.MaxMemoryBytes-memoryBytes {
|
||||
return nil, 0, query.ErrBudgetExceeded
|
||||
}
|
||||
memoryBytes += recordMemory
|
||||
records = append(records, record)
|
||||
if ctx.Err() != nil {
|
||||
return nil, 0, query.ErrBudgetExceeded
|
||||
}
|
||||
}
|
||||
}
|
||||
return records, memoryBytes, nil
|
||||
}
|
||||
|
||||
func sortRawQueryRecords(records []projectedRecord) {
|
||||
sort.SliceStable(records, func(left, right int) bool {
|
||||
if records[left].timestamp.Equal(records[right].timestamp) {
|
||||
if records[left].sourceID == records[right].sourceID {
|
||||
if records[left].streamID == records[right].streamID {
|
||||
if records[left].sequence == records[right].sequence {
|
||||
return records[left].recordIndex > records[right].recordIndex
|
||||
}
|
||||
return records[left].sequence > records[right].sequence
|
||||
}
|
||||
return records[left].streamID < records[right].streamID
|
||||
}
|
||||
return records[left].sourceID < records[right].sourceID
|
||||
}
|
||||
return records[left].timestamp.After(records[right].timestamp)
|
||||
})
|
||||
}
|
||||
|
||||
func finishQueryResult(result query.Result, records []projectedRecord, ast query.AST, columns []query.Column, registry query.Registry, memoryBytes, maxMemoryBytes int64, started time.Time) (query.Result, error) {
|
||||
var err error
|
||||
if ast.Summary == nil {
|
||||
result.Rows, err = materializeRecords(records, columns)
|
||||
} else {
|
||||
result.Rows, memoryBytes, err = summarizeRecords(records, ast, columns, registry, memoryBytes, maxMemoryBytes)
|
||||
}
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
if ast.Sort != nil {
|
||||
if err = sortRows(result.Rows, columns, ast.Sort.Field, ast.Sort.Descending); err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
}
|
||||
if len(result.Rows) > ast.Limit {
|
||||
result.Rows = result.Rows[:ast.Limit]
|
||||
result.Stats.Truncated = true
|
||||
}
|
||||
result.Stats.DurationNS = time.Since(started).Nanoseconds()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func projectionSelection(ast query.AST, scope query.Scope, registry query.Registry, activeVersion int, now time.Time) (string, []any, error) {
|
||||
statement := `SELECT o.project_id,o.environment_id,o.service_id,o.source_id,o.stream_id,o.sequence,o.record_index,o.signal,o.timestamp,o.name,o.severity,o.body,o.value,o.trace_id,o.span_id,o.correlation_id,o.attributes_json FROM observations o`
|
||||
if index := summaryProjectionIndex(ast); index != "" {
|
||||
statement += " INDEXED BY " + index
|
||||
}
|
||||
var joins, predicates []string
|
||||
var joinArguments, predicateArguments []any
|
||||
indexedJoin := 0
|
||||
for _, filter := range ast.Filters {
|
||||
if filter.Op == "=~" {
|
||||
continue
|
||||
}
|
||||
field := query.CanonicalField(filter.Field)
|
||||
expression, expressionArguments := sqlFieldExpression(field)
|
||||
descriptor, unknown := query.ResolveDescriptor(ast.Signal, field, registry)
|
||||
value, err := typedFilterValue(filter.Value, descriptor.Type)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
operator := map[string]string{"==": "=", "!=": "!=", ">": ">", ">=": ">=", "<": "<", "<=": "<="}[filter.Op]
|
||||
if operator == "" {
|
||||
return "", nil, query.ErrTypeMismatch
|
||||
}
|
||||
_, builtin := query.BuiltinDescriptor(ast.Signal, field)
|
||||
if !unknown && !builtin && descriptor.Index != schema.IndexNone && activeVersion > 1 {
|
||||
table, tableErr := projectionIndexTable(activeVersion)
|
||||
if tableErr != nil {
|
||||
return "", nil, tableErr
|
||||
}
|
||||
column := "value_text"
|
||||
if descriptor.Type == schema.TypeInteger || descriptor.Type == schema.TypeFloat || descriptor.Type == schema.TypeDuration {
|
||||
column = "value_number"
|
||||
}
|
||||
if descriptor.Type == schema.TypeTime {
|
||||
parsed, parseErr := time.Parse(time.RFC3339Nano, filter.Value)
|
||||
if parseErr != nil {
|
||||
return "", nil, query.ErrTypeMismatch
|
||||
}
|
||||
value = parsed.UTC().Format(indexedTimeFormat)
|
||||
}
|
||||
alias := fmt.Sprintf("idx%d", indexedJoin)
|
||||
indexedJoin++
|
||||
joins = append(joins, " JOIN "+table+" "+alias+" ON "+alias+".signal=o.signal AND "+alias+".field=? AND "+alias+".source_id=o.source_id AND "+alias+".stream_id=o.stream_id AND "+alias+".sequence=o.sequence AND "+alias+".record_index=o.record_index AND "+alias+"."+column+operator+"?")
|
||||
joinArguments = append(joinArguments, descriptor.Field, value)
|
||||
} else {
|
||||
predicates = append(predicates, expression+operator+"?")
|
||||
predicateArguments = append(predicateArguments, expressionArguments...)
|
||||
predicateArguments = append(predicateArguments, value)
|
||||
}
|
||||
}
|
||||
statement += strings.Join(joins, "") + ` WHERE o.organization_id=? AND o.signal=?`
|
||||
arguments := append(joinArguments, scope.OrganizationID, string(ast.Signal))
|
||||
for _, selected := range []struct {
|
||||
column, value string
|
||||
}{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} {
|
||||
if selected.value != "" {
|
||||
statement += " AND o." + selected.column + "=?"
|
||||
arguments = append(arguments, selected.value)
|
||||
}
|
||||
}
|
||||
if ast.Window > 0 {
|
||||
statement += " AND o.timestamp>=?"
|
||||
arguments = append(arguments, now.UTC().Add(-ast.Window).Format(time.RFC3339Nano))
|
||||
}
|
||||
for _, predicate := range predicates {
|
||||
statement += " AND " + predicate
|
||||
}
|
||||
arguments = append(arguments, predicateArguments...)
|
||||
if ast.Summary == nil {
|
||||
statement += ` ORDER BY o.timestamp DESC,o.source_id,o.stream_id,o.sequence DESC,o.record_index DESC`
|
||||
}
|
||||
if limit := projectionRowLimit(ast); limit > 0 {
|
||||
statement += ` LIMIT ?`
|
||||
arguments = append(arguments, limit)
|
||||
}
|
||||
return statement, arguments, nil
|
||||
}
|
||||
|
||||
// projectionRowLimit returns the number of rows SQLite may return for a
|
||||
// record query whose final result can be decided in timestamp order. The
|
||||
// extra row preserves the result's truncated signal. Regular expressions are
|
||||
// evaluated in Go, so they cannot safely use a pre-match SQL limit.
|
||||
func projectionRowLimit(ast query.AST) int {
|
||||
if ast.Summary != nil || ast.Sort != nil && (query.CanonicalField(ast.Sort.Field) != "timestamp" || !ast.Sort.Descending) {
|
||||
return 0
|
||||
}
|
||||
for _, filter := range ast.Filters {
|
||||
if filter.Op == "=~" {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return ast.Limit + 1
|
||||
}
|
||||
|
||||
// summaryProjectionIndex keeps selective built-in filters on their reviewed
|
||||
// projection index. Summary execution is independent of input order and later
|
||||
// sorts its deterministic result rows, so it must not trade the selective
|
||||
// filter path for the record-level timestamp order used by ordinary queries.
|
||||
func summaryProjectionIndex(ast query.AST) string {
|
||||
if ast.Summary == nil {
|
||||
return ""
|
||||
}
|
||||
for _, filter := range ast.Filters {
|
||||
if filter.Op == "=~" || filter.Op == "!=" {
|
||||
continue
|
||||
}
|
||||
switch query.CanonicalField(filter.Field) {
|
||||
case "http.status_code":
|
||||
return "observations_http_status"
|
||||
case "http.route":
|
||||
return "observations_http_route"
|
||||
case "duration_ns":
|
||||
return "observations_duration"
|
||||
case "name":
|
||||
return "observations_name"
|
||||
case "severity":
|
||||
return "observations_severity"
|
||||
case "value":
|
||||
return "observations_value"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sqlFieldExpression(field string) (string, []any) {
|
||||
switch query.CanonicalField(field) {
|
||||
case "project.id":
|
||||
return "o.project_id", nil
|
||||
case "environment.id":
|
||||
return "o.environment_id", nil
|
||||
case "service.id":
|
||||
return "o.service_id", nil
|
||||
case "source.id":
|
||||
return "o.source_id", nil
|
||||
case "stream.id":
|
||||
return "o.stream_id", nil
|
||||
case "timestamp", "name", "severity", "body", "value", "trace_id", "span_id", "correlation_id":
|
||||
return "o." + query.CanonicalField(field), nil
|
||||
case "http.route":
|
||||
return `json_extract(o.attributes_json,'$."http.route"')`, nil
|
||||
case "http.status_code":
|
||||
return `CAST(json_extract(o.attributes_json,'$."http.status_code"') AS INTEGER)`, nil
|
||||
case "duration_ns":
|
||||
return `CAST(json_extract(o.attributes_json,'$."duration_ns"') AS REAL)`, nil
|
||||
default:
|
||||
return "json_extract(o.attributes_json,?)", []any{`$."` + query.CanonicalField(field) + `"`}
|
||||
}
|
||||
}
|
||||
|
||||
func typedFilterValue(value string, valueType schema.Type) (any, error) {
|
||||
switch valueType {
|
||||
case schema.TypeInteger:
|
||||
parsed, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil {
|
||||
return nil, query.ErrTypeMismatch
|
||||
}
|
||||
return parsed, nil
|
||||
case schema.TypeFloat, schema.TypeDuration:
|
||||
parsed, err := strconv.ParseFloat(value, 64)
|
||||
if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) {
|
||||
return nil, query.ErrTypeMismatch
|
||||
}
|
||||
return parsed, nil
|
||||
case schema.TypeBoolean:
|
||||
parsed, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return nil, query.ErrTypeMismatch
|
||||
}
|
||||
return strconv.FormatBool(parsed), nil
|
||||
case schema.TypeTime:
|
||||
parsed, err := time.Parse(time.RFC3339Nano, value)
|
||||
if err != nil {
|
||||
return nil, query.ErrTypeMismatch
|
||||
}
|
||||
return parsed.UTC().Format(time.RFC3339Nano), nil
|
||||
default:
|
||||
return value, nil
|
||||
}
|
||||
}
|
||||
|
||||
func scanProjected(rows *sql.Rows) (projectedRecord, int64, error) {
|
||||
var record projectedRecord
|
||||
var timestamp, signal, attributes string
|
||||
var severity, body, traceID, spanID, correlationID sql.NullString
|
||||
var value sql.NullFloat64
|
||||
err := rows.Scan(&record.projectID, &record.environmentID, &record.serviceID, &record.sourceID, &record.streamID, &record.sequence, &record.recordIndex, &signal, ×tamp, &record.name, &severity, &body, &value, &traceID, &spanID, &correlationID, &attributes)
|
||||
if err != nil {
|
||||
return projectedRecord{}, 0, err
|
||||
}
|
||||
record.signal = model.Signal(signal)
|
||||
record.timestamp, err = time.Parse(time.RFC3339Nano, timestamp)
|
||||
if err != nil {
|
||||
return projectedRecord{}, 0, err
|
||||
}
|
||||
record.severity, record.body = severity.String, body.String
|
||||
record.traceID, record.spanID, record.correlationID = traceID.String, spanID.String, correlationID.String
|
||||
if value.Valid {
|
||||
record.value = &value.Float64
|
||||
}
|
||||
if err = json.Unmarshal([]byte(attributes), &record.attributes); err != nil {
|
||||
return projectedRecord{}, 0, errors.New("invalid projected attributes")
|
||||
}
|
||||
if record.attributes == nil {
|
||||
record.attributes = map[string]string{}
|
||||
}
|
||||
readBytes := int64(len(record.projectID) + len(record.environmentID) + len(record.serviceID) + len(record.sourceID) + len(record.streamID) + len(signal) + len(timestamp) + len(record.name) + len(record.severity) + len(record.body) + len(record.traceID) + len(record.spanID) + len(record.correlationID) + len(attributes) + 64)
|
||||
return record, readBytes, nil
|
||||
}
|
||||
|
||||
func matchesRecord(record projectedRecord, ast query.AST, registry query.Registry) (bool, error) {
|
||||
return query.MatchesFilters(ast, registry, record.field)
|
||||
}
|
||||
|
||||
func (record projectedRecord) field(field string) (string, bool) {
|
||||
switch query.CanonicalField(field) {
|
||||
case "project.id":
|
||||
return record.projectID, record.projectID != ""
|
||||
case "environment.id":
|
||||
return record.environmentID, record.environmentID != ""
|
||||
case "service.id":
|
||||
return record.serviceID, record.serviceID != ""
|
||||
case "source.id":
|
||||
return record.sourceID, record.sourceID != ""
|
||||
case "stream.id":
|
||||
return record.streamID, record.streamID != ""
|
||||
case "timestamp":
|
||||
return record.timestamp.UTC().Format(time.RFC3339Nano), true
|
||||
case "name":
|
||||
return record.name, record.name != ""
|
||||
case "severity":
|
||||
return record.severity, record.severity != ""
|
||||
case "body":
|
||||
return record.body, record.body != ""
|
||||
case "value":
|
||||
if record.value == nil {
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatFloat(*record.value, 'g', -1, 64), true
|
||||
case "trace_id":
|
||||
return record.traceID, record.traceID != ""
|
||||
case "span_id":
|
||||
return record.spanID, record.spanID != ""
|
||||
case "correlation_id":
|
||||
return record.correlationID, record.correlationID != ""
|
||||
default:
|
||||
value, ok := record.attributes[query.CanonicalField(field)]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
|
||||
func resultColumns(ast query.AST, registry query.Registry) ([]query.Column, error) {
|
||||
if ast.Summary != nil {
|
||||
var columns []query.Column
|
||||
if ast.Bucket > 0 {
|
||||
columns = append(columns, query.Column{Field: "window_start", Type: schema.TypeTime, Unit: "s"})
|
||||
}
|
||||
for _, field := range ast.Summary.GroupBy {
|
||||
canonical := query.CanonicalField(field)
|
||||
descriptor, _ := query.ResolveDescriptor(ast.Signal, canonical, registry)
|
||||
columns = append(columns, query.Column{Field: canonical, Type: descriptor.Type, Unit: descriptor.Unit})
|
||||
}
|
||||
for _, aggregate := range ast.Summary.Aggregates {
|
||||
valueType := schema.TypeFloat
|
||||
if aggregate.Function == "count" {
|
||||
valueType = schema.TypeInteger
|
||||
}
|
||||
unit := ""
|
||||
if aggregate.Field != "" {
|
||||
descriptor, _ := query.ResolveDescriptor(ast.Signal, aggregate.Field, registry)
|
||||
unit = descriptor.Unit
|
||||
}
|
||||
columns = append(columns, query.Column{Field: aggregate.Alias, Type: valueType, Unit: unit})
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
fields := []string{"timestamp", "service.id", "name"}
|
||||
switch ast.Signal {
|
||||
case model.SignalLogs:
|
||||
fields = append(fields, "severity")
|
||||
case model.SignalMetrics:
|
||||
fields = append(fields, "value")
|
||||
case model.SignalTraces:
|
||||
fields = append(fields, "trace_id", "span_id")
|
||||
case model.SignalDeployments:
|
||||
fields = append(fields, "correlation_id")
|
||||
}
|
||||
fields = append(fields, query.ReferencedFields(ast)...)
|
||||
seen := map[string]bool{}
|
||||
columns := make([]query.Column, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
canonical := query.CanonicalField(field)
|
||||
if seen[canonical] {
|
||||
continue
|
||||
}
|
||||
seen[canonical] = true
|
||||
descriptor, _ := query.ResolveDescriptor(ast.Signal, canonical, registry)
|
||||
columns = append(columns, query.Column{Field: canonical, Type: descriptor.Type, Unit: descriptor.Unit})
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func materializeRecords(records []projectedRecord, columns []query.Column) ([]query.Row, error) {
|
||||
result := make([]query.Row, 0, len(records))
|
||||
for _, record := range records {
|
||||
row := query.Row{Values: make([]*string, len(columns))}
|
||||
for index, column := range columns {
|
||||
if value, ok := record.field(column.Field); ok {
|
||||
if canonical, valid := canonicalResultValue(value, column.Type); valid {
|
||||
row.Values[index] = stringPointer(canonical)
|
||||
}
|
||||
}
|
||||
}
|
||||
result = append(result, row)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type aggregateState struct {
|
||||
function string
|
||||
count int64
|
||||
sum, min, max float64
|
||||
values []float64
|
||||
}
|
||||
|
||||
type summaryGroup struct {
|
||||
key string
|
||||
values []*string
|
||||
aggregates []aggregateState
|
||||
}
|
||||
|
||||
func summarizeRecords(records []projectedRecord, ast query.AST, columns []query.Column, registry query.Registry, memoryBytes, maxMemory int64) ([]query.Row, int64, error) {
|
||||
groups := map[string]*summaryGroup{}
|
||||
for _, record := range records {
|
||||
var values []*string
|
||||
if ast.Bucket > 0 {
|
||||
bucket := record.timestamp.UTC().Truncate(ast.Bucket).Format(time.RFC3339Nano)
|
||||
values = append(values, stringPointer(bucket))
|
||||
}
|
||||
for _, field := range ast.Summary.GroupBy {
|
||||
value, ok := record.field(field)
|
||||
if ok {
|
||||
column := columns[len(values)]
|
||||
if canonical, valid := canonicalResultValue(value, column.Type); valid {
|
||||
values = append(values, stringPointer(canonical))
|
||||
} else {
|
||||
values = append(values, nil)
|
||||
}
|
||||
} else {
|
||||
values = append(values, nil)
|
||||
}
|
||||
}
|
||||
key := groupKey(values)
|
||||
group := groups[key]
|
||||
if group == nil {
|
||||
group = &summaryGroup{key: key, values: values, aggregates: make([]aggregateState, len(ast.Summary.Aggregates))}
|
||||
for index, aggregate := range ast.Summary.Aggregates {
|
||||
group.aggregates[index].function = aggregate.Function
|
||||
}
|
||||
groups[key] = group
|
||||
memoryBytes += int64(len(key) + len(values)*16 + len(group.aggregates)*64)
|
||||
}
|
||||
for index, aggregate := range ast.Summary.Aggregates {
|
||||
state := &group.aggregates[index]
|
||||
if aggregate.Function == "count" {
|
||||
state.count++
|
||||
continue
|
||||
}
|
||||
value, ok := record.field(aggregate.Field)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
descriptor, _ := query.ResolveDescriptor(ast.Signal, aggregate.Field, registry)
|
||||
if descriptor.Type != schema.TypeInteger && descriptor.Type != schema.TypeFloat && descriptor.Type != schema.TypeDuration {
|
||||
return nil, memoryBytes, query.ErrTypeMismatch
|
||||
}
|
||||
number, err := strconv.ParseFloat(value, 64)
|
||||
if err != nil || math.IsNaN(number) || math.IsInf(number, 0) {
|
||||
continue
|
||||
}
|
||||
if state.count == 0 {
|
||||
state.min, state.max = number, number
|
||||
} else {
|
||||
state.min = math.Min(state.min, number)
|
||||
state.max = math.Max(state.max, number)
|
||||
}
|
||||
state.count++
|
||||
state.sum += number
|
||||
if aggregate.Function == "p50" || aggregate.Function == "p95" || aggregate.Function == "p99" {
|
||||
state.values = append(state.values, number)
|
||||
memoryBytes += 8
|
||||
}
|
||||
}
|
||||
if memoryBytes > maxMemory {
|
||||
return nil, memoryBytes, query.ErrBudgetExceeded
|
||||
}
|
||||
}
|
||||
ordered := make([]*summaryGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
ordered = append(ordered, group)
|
||||
}
|
||||
sort.Slice(ordered, func(i, j int) bool { return ordered[i].key < ordered[j].key })
|
||||
rows := make([]query.Row, 0, len(ordered))
|
||||
for _, group := range ordered {
|
||||
row := query.Row{Values: append([]*string(nil), group.values...)}
|
||||
for _, state := range group.aggregates {
|
||||
value, ok := aggregateValue(state)
|
||||
if ok {
|
||||
row.Values = append(row.Values, stringPointer(value))
|
||||
} else {
|
||||
row.Values = append(row.Values, nil)
|
||||
}
|
||||
}
|
||||
if len(row.Values) != len(columns) {
|
||||
return nil, memoryBytes, errors.New("summary result shape is invalid")
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows, memoryBytes, nil
|
||||
}
|
||||
|
||||
func aggregateValue(state aggregateState) (string, bool) {
|
||||
if state.function == "count" {
|
||||
return strconv.FormatInt(state.count, 10), true
|
||||
}
|
||||
if state.count == 0 {
|
||||
return "", false
|
||||
}
|
||||
var value float64
|
||||
switch state.function {
|
||||
case "min":
|
||||
value = state.min
|
||||
case "max":
|
||||
value = state.max
|
||||
case "sum":
|
||||
value = state.sum
|
||||
case "avg":
|
||||
value = state.sum / float64(state.count)
|
||||
case "p50", "p95", "p99":
|
||||
sort.Float64s(state.values)
|
||||
percentile := map[string]float64{"p50": .50, "p95": .95, "p99": .99}[state.function]
|
||||
index := max(0, int(math.Ceil(percentile*float64(len(state.values))))-1)
|
||||
value = state.values[index]
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatFloat(value, 'g', -1, 64), true
|
||||
}
|
||||
|
||||
func sortRows(rows []query.Row, columns []query.Column, field string, descending bool) error {
|
||||
canonical := query.CanonicalField(field)
|
||||
column := -1
|
||||
for index, candidate := range columns {
|
||||
if candidate.Field == canonical || candidate.Field == field {
|
||||
column = index
|
||||
break
|
||||
}
|
||||
}
|
||||
if column < 0 {
|
||||
return errors.New("query sort field is unavailable")
|
||||
}
|
||||
sort.SliceStable(rows, func(i, j int) bool {
|
||||
left, right := rows[i].Values[column], rows[j].Values[column]
|
||||
if left == nil {
|
||||
return false
|
||||
}
|
||||
if right == nil {
|
||||
return true
|
||||
}
|
||||
comparison, leftValid, rightValid := compareTyped(*left, *right, columns[column].Type)
|
||||
if !leftValid {
|
||||
return false
|
||||
}
|
||||
if !rightValid {
|
||||
return true
|
||||
}
|
||||
if descending {
|
||||
return comparison > 0
|
||||
}
|
||||
return comparison < 0
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func compareTyped(left, right string, valueType schema.Type) (int, bool, bool) {
|
||||
if valueType == schema.TypeInteger || valueType == schema.TypeFloat || valueType == schema.TypeDuration {
|
||||
leftNumber, leftErr := strconv.ParseFloat(left, 64)
|
||||
rightNumber, rightErr := strconv.ParseFloat(right, 64)
|
||||
leftValid := leftErr == nil && !math.IsNaN(leftNumber) && !math.IsInf(leftNumber, 0)
|
||||
rightValid := rightErr == nil && !math.IsNaN(rightNumber) && !math.IsInf(rightNumber, 0)
|
||||
return compareFloat(leftNumber, rightNumber), leftValid, rightValid
|
||||
}
|
||||
if valueType == schema.TypeTime {
|
||||
leftTime, leftErr := time.Parse(time.RFC3339Nano, left)
|
||||
rightTime, rightErr := time.Parse(time.RFC3339Nano, right)
|
||||
return leftTime.Compare(rightTime), leftErr == nil, rightErr == nil
|
||||
}
|
||||
return strings.Compare(left, right), true, true
|
||||
}
|
||||
|
||||
func canonicalResultValue(value string, valueType schema.Type) (string, bool) {
|
||||
switch valueType {
|
||||
case schema.TypeInteger:
|
||||
parsed, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatInt(parsed, 10), true
|
||||
case schema.TypeFloat, schema.TypeDuration:
|
||||
parsed, err := strconv.ParseFloat(value, 64)
|
||||
if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) {
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatFloat(parsed, 'g', -1, 64), true
|
||||
case schema.TypeTime:
|
||||
parsed, err := time.Parse(time.RFC3339Nano, value)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return parsed.UTC().Format(time.RFC3339Nano), true
|
||||
case schema.TypeBoolean:
|
||||
parsed, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatBool(parsed), true
|
||||
default:
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
|
||||
func compareFloat(left, right float64) int {
|
||||
if left < right {
|
||||
return -1
|
||||
}
|
||||
if left > right {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func groupKey(values []*string) string {
|
||||
var builder strings.Builder
|
||||
for _, value := range values {
|
||||
if value == nil {
|
||||
builder.WriteString("-1:")
|
||||
continue
|
||||
}
|
||||
builder.WriteString(strconv.Itoa(len(*value)))
|
||||
builder.WriteByte(':')
|
||||
builder.WriteString(*value)
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func stringPointer(value string) *string {
|
||||
copy := value
|
||||
return ©
|
||||
}
|
||||
|
||||
func queryExecutionError(ctx context.Context, err error) error {
|
||||
if ctx.Err() != nil {
|
||||
return query.ErrBudgetExceeded
|
||||
}
|
||||
return fmt.Errorf("query projection: %w", err)
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
func TestQueryExecutesScopedTypedFiltersAndSorting(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 7, 0, 0, 0, time.UTC)
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records := []model.Observation{
|
||||
requestObservation(now.Add(-time.Minute), "/ok", 200, 50),
|
||||
requestObservation(now.Add(-2*time.Minute), "/broken", 503, 300),
|
||||
requestObservation(now.Add(-3*time.Minute), "/slow", 500, 200),
|
||||
{Timestamp: now.Add(-4 * time.Minute), Name: "application.http.request", Attributes: map[string]string{"http.route": "/invalid", "http.status_code": "not-a-number", "duration_ns": "invalid"}},
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: records}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
otherToken, err := store.CreateSource(ctx, "source-b", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-b"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := model.Batch{Version: model.BatchVersion, SourceID: "source-b", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{requestObservation(now, "/other", 599, 999)}}
|
||||
if _, err = store.Ingest(ctx, otherToken, other, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
projectionPath := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite")
|
||||
projectionBefore, err := os.ReadFile(projectionPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
infoBefore, err := os.Stat(projectionPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ast, err := query.Parse(`logs | where status >= 500 | window 1h | sort duration desc | limit 1`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}, testQueryBudget(), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Rows) != 1 || result.Stats.MatchedRows != 2 || !result.Stats.Truncated {
|
||||
t.Fatalf("result=%+v", result)
|
||||
}
|
||||
duration := columnValue(t, result, 0, "duration_ns")
|
||||
service := columnValue(t, result, 0, "service.id")
|
||||
if duration != "300" || service != "service-a" {
|
||||
t.Fatalf("duration=%q service=%q result=%+v", duration, service, result)
|
||||
}
|
||||
ast, err = query.Parse(`logs | sort duration desc | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a", ServiceID: "service-a"}, testQueryBudget(), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index, column := range result.Columns {
|
||||
if column.Field == "duration_ns" && (len(result.Rows) != 4 || result.Rows[3].Values[index] != nil) {
|
||||
t.Fatalf("invalid numeric value was not sorted last: %+v", result)
|
||||
}
|
||||
}
|
||||
projectionAfter, err := os.ReadFile(projectionPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
infoAfter, err := os.Stat(projectionPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(projectionBefore, projectionAfter) || !infoBefore.ModTime().Equal(infoAfter.ModTime()) {
|
||||
t.Fatal("query execution modified the organization projection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuerySummarizesThroughSharedBucketAST(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 7, 2, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{
|
||||
requestObservation(now.Add(-time.Minute), "/items", 200, 100),
|
||||
requestObservation(now.Add(-2*time.Minute), "/items", 500, 300),
|
||||
requestObservation(now.Add(-time.Minute), "/about", 200, 20),
|
||||
}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
ast, err := query.Parse(`logs | summarize count(), p95(duration) by route, window(5m) | sort count desc | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ast.Window != 0 || ast.Bucket != 5*time.Minute {
|
||||
t.Fatalf("window=%s bucket=%s", ast.Window, ast.Bucket)
|
||||
}
|
||||
result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Rows) != 2 || columnValue(t, result, 0, "http.route") != "/items" || columnValue(t, result, 0, "count") != "2" || columnValue(t, result, 0, "p95_duration") != "300" {
|
||||
t.Fatalf("result=%+v", result)
|
||||
}
|
||||
filtered, err := query.Parse(`logs | where status >= 500 | summarize count() by route, window(5m) | sort count desc | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err = store.Query(ctx, filtered, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Rows) != 1 || columnValue(t, result, 0, "http.route") != "/items" || columnValue(t, result, 0, "count") != "1" {
|
||||
t.Fatalf("filtered result=%+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummaryProjectionUsesSelectiveIndexWithoutRecordOrder(t *testing.T) {
|
||||
now := time.Date(2026, 8, 17, 7, 2, 0, 0, time.UTC)
|
||||
ast, err := query.Parse(`logs | where status >= 500 | window 24h | summarize count() by route, window(5m) | sort count desc | limit 50`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
statement, _, err := projectionSelection(ast, query.Scope{OrganizationID: "organization-a"}, nil, 1, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(statement, "FROM observations o INDEXED BY observations_http_status") {
|
||||
t.Fatalf("summary did not pin the selective status index: %s", statement)
|
||||
}
|
||||
if strings.Contains(statement, " ORDER BY o.timestamp") {
|
||||
t.Fatalf("summary retained unnecessary record ordering: %s", statement)
|
||||
}
|
||||
|
||||
ordinary, err := query.Parse(`logs | where status >= 500 | window 24h | limit 50`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
statement, arguments, err := projectionSelection(ordinary, query.Scope{OrganizationID: "organization-a"}, nil, 1, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(statement, " INDEXED BY ") || !strings.Contains(statement, " ORDER BY o.timestamp") || !strings.HasSuffix(statement, " LIMIT ?") || arguments[len(arguments)-1] != 51 {
|
||||
t.Fatalf("ordinary record query changed its ordered plan: %s", statement)
|
||||
}
|
||||
|
||||
regularExpression, err := query.Parse(`logs | where route =~ "^/items" | window 24h | limit 50`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
statement, _, err = projectionSelection(regularExpression, query.Scope{OrganizationID: "organization-a"}, nil, 1, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(statement, " LIMIT ?") {
|
||||
t.Fatalf("regular-expression query limited rows before Go evaluation: %s", statement)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryPlansBoundedRecordLimitIndependentlyOfProjectionSize(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 18, 12, 0, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{
|
||||
requestObservation(now.Add(-time.Minute), "/one", 200, 10),
|
||||
requestObservation(now.Add(-2*time.Minute), "/two", 200, 20),
|
||||
requestObservation(now.Add(-3*time.Minute), "/three", 200, 30),
|
||||
}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
projectionBytes, err := store.EstimateOrganizationBytes("organization-a")
|
||||
if err != nil || projectionBytes < 4 {
|
||||
t.Fatalf("projection bytes=%d err=%v", projectionBytes, err)
|
||||
}
|
||||
budget := testQueryBudget()
|
||||
budget.MaxScannedBytes = projectionBytes - 1
|
||||
budget.MaxMemoryBytes = min(projectionBytes/2, 1<<20)
|
||||
ast, err := query.Parse(`logs | window 1h | limit 1`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, budget, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Rows) != 1 || result.Stats.ScannedRows != 2 || !result.Stats.Truncated || result.Explain.EstimatedScanBytes != budget.MaxMemoryBytes {
|
||||
t.Fatalf("result=%+v", result)
|
||||
}
|
||||
|
||||
regex, err := query.Parse(`logs | where route =~ "^/" | window 1h | limit 1`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.Query(ctx, regex, query.Scope{OrganizationID: "organization-a"}, budget, now); err == nil || !strings.Contains(err.Error(), "estimated query scan exceeds budget") {
|
||||
t.Fatalf("unbounded regex planning err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryEnforcesSensitiveAndExecutionBudgets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 7, 0, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.http.request", Body: "private evidence"}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
ast, _ := query.Parse(`logs | where body == "private evidence" | limit 10`, 100)
|
||||
if _, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now); !errors.Is(err, query.ErrSensitivePermissionRequired) {
|
||||
t.Fatalf("sensitive err=%v", err)
|
||||
}
|
||||
ast, _ = query.Parse(`logs | limit 10`, 100)
|
||||
budget := testQueryBudget()
|
||||
budget.MaxMemoryBytes = 1
|
||||
if _, err = store.Query(ctx, ast, query.Scope{OrganizationID: "organization-a", Sensitive: true}, budget, now); !errors.Is(err, query.ErrBudgetExceeded) {
|
||||
t.Fatalf("budget err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func requestObservation(timestamp time.Time, route string, status int, duration int) model.Observation {
|
||||
return model.Observation{Timestamp: timestamp, Name: "application.http.request", Attributes: map[string]string{"http.route": route, "http.status_code": strconv.Itoa(status), "duration_ns": strconv.Itoa(duration)}}
|
||||
}
|
||||
|
||||
func testQueryBudget() query.Budget {
|
||||
return query.Budget{MaxDuration: 5 * time.Second, MaxRows: 100, MaxScannedBytes: 100 << 20, MaxMemoryBytes: 10 << 20}
|
||||
}
|
||||
|
||||
func columnValue(t *testing.T, result query.Result, row int, field string) string {
|
||||
t.Helper()
|
||||
for index, column := range result.Columns {
|
||||
if column.Field == field {
|
||||
if result.Rows[row].Values[index] == nil {
|
||||
t.Fatalf("field %s is nil", field)
|
||||
}
|
||||
return *result.Rows[row].Values[index]
|
||||
}
|
||||
}
|
||||
t.Fatalf("field %s is absent: %+v", field, result.Columns)
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
// queryRawCandidate executes one bounded log query directly from the complete
|
||||
// retained raw-segment catalogue. It is deliberately not a public Store or
|
||||
// HTTP surface: the current projection path remains the production oracle
|
||||
// while differential tests establish the adaptive read path's semantics.
|
||||
//
|
||||
// The organization lock prevents in-process retention from moving or deleting
|
||||
// a selected object during this first proof. A future leased materialization
|
||||
// needs an explicit catalogue snapshot/high-watermark rather than holding this
|
||||
// lock across a potentially long historical scan.
|
||||
func (s *Store) queryRawCandidate(ctx context.Context, ast query.AST, scope query.Scope, budget query.Budget, now time.Time) (query.Result, error) {
|
||||
if now.IsZero() {
|
||||
return query.Result{}, errors.New("query time is required")
|
||||
}
|
||||
if ast.Signal != model.SignalLogs {
|
||||
return query.Result{}, errors.New("raw query candidate supports logs only")
|
||||
}
|
||||
lock := s.namedLock("organization:" + scope.OrganizationID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
registry, _, err := s.ActiveDescriptors(ctx, scope.OrganizationID)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
segments, estimated, err := s.allRawSegmentsForQuery(ctx, ast, scope, now)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
explain, err := query.Plan(ast, scope, registry, estimated, budget)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
for index := range explain.ProjectedSources {
|
||||
explain.ProjectedSources[index] += "/raw:catalog"
|
||||
}
|
||||
columns, err := resultColumns(ast, registry)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
result := query.Result{Version: query.ResultVersion, Explain: explain, Columns: columns, Rows: []query.Row{}}
|
||||
runContext, cancel := context.WithTimeout(ctx, budget.MaxDuration)
|
||||
defer cancel()
|
||||
started := time.Now()
|
||||
records, memoryBytes, err := s.appendRawQuerySegments(runContext, segments, ast, registry, budget, now, &result, nil, 0)
|
||||
if err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
sortRawQueryRecords(records)
|
||||
return finishQueryResult(result, records, ast, columns, registry, memoryBytes, budget.MaxMemoryBytes, started)
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
func TestRawQueryCandidateMatchesProjectionOracle(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 18, 22, 0, 0, 0, time.UTC)
|
||||
for _, source := range []struct {
|
||||
id string
|
||||
service string
|
||||
batches []model.Batch
|
||||
}{
|
||||
{id: "source-a", service: "service-a", batches: []model.Batch{
|
||||
{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now.Add(-2 * time.Minute), Signal: model.SignalLogs, Records: []model.Observation{
|
||||
requestObservation(now.Add(-3*time.Minute), "/items", 503, 300),
|
||||
requestObservation(now.Add(-4*time.Minute), "/items", 200, 80),
|
||||
}},
|
||||
{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 2, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{
|
||||
requestObservation(now.Add(-time.Minute), "/checkout", 500, 900),
|
||||
{Timestamp: now.Add(-30 * time.Second), Name: "application.note", Body: "safe", Severity: "information"},
|
||||
}},
|
||||
}},
|
||||
{id: "source-b", service: "service-b", batches: []model.Batch{
|
||||
{Version: model.BatchVersion, SourceID: "source-b", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{
|
||||
requestObservation(now.Add(-90*time.Second), "/items", 502, 450),
|
||||
}},
|
||||
}},
|
||||
} {
|
||||
token, err := store.CreateSource(ctx, source.id, model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: source.service})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, batch := range source.batches {
|
||||
if _, err = store.Ingest(ctx, token, batch, batch.ObservedAt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
projectAll(t, store)
|
||||
|
||||
queries := []struct {
|
||||
text string
|
||||
scope query.Scope
|
||||
}{
|
||||
{`logs | where status >= 500 | window 1h | sort duration desc | limit 2`, query.Scope{OrganizationID: "organization-a"}},
|
||||
{`logs | where status >= 500 | summarize count(), p95(duration) by route | sort count desc | limit 10`, query.Scope{OrganizationID: "organization-a"}},
|
||||
{`logs | where route =~ "^/item" | window 1h | limit 10`, query.Scope{OrganizationID: "organization-a", ServiceID: "service-a"}},
|
||||
}
|
||||
for _, candidate := range queries {
|
||||
ast, err := query.Parse(candidate.text, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("parse %q: %v", candidate.text, err)
|
||||
}
|
||||
projected, err := store.Query(ctx, ast, candidate.scope, testQueryBudget(), now)
|
||||
if err != nil {
|
||||
t.Fatalf("projected %q: %v", candidate.text, err)
|
||||
}
|
||||
raw, err := store.queryRawCandidate(ctx, ast, candidate.scope, testQueryBudget(), now)
|
||||
if err != nil {
|
||||
t.Fatalf("raw %q: %v", candidate.text, err)
|
||||
}
|
||||
if !reflect.DeepEqual(projected.Columns, raw.Columns) || !reflect.DeepEqual(projected.Rows, raw.Rows) || projected.Stats.Truncated != raw.Stats.Truncated || projected.Stats.MatchedRows != raw.Stats.MatchedRows {
|
||||
t.Fatalf("query %q diverged\nprojected=%+v\nraw=%+v", candidate.text, projected, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawQueryCandidateReadsDurableUnprojectedBatchesWithoutWriting(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 18, 22, 15, 0, 0, time.UTC)
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{requestObservation(now, "/ready", 200, 25)}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectionPath := filepath.Join(store.root, "organizations", "organization-a", "projection.sqlite")
|
||||
if _, err = os.Lstat(projectionPath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("projection unexpectedly exists: %v", err)
|
||||
}
|
||||
ast, err := query.Parse(`logs | where route == "/ready" | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := store.queryRawCandidate(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now)
|
||||
if err != nil || len(result.Rows) != 1 || columnValue(t, result, 0, "http.route") != "/ready" {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
if _, err = os.Lstat(projectionPath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("raw query created a projection: %v", err)
|
||||
}
|
||||
var projected int
|
||||
if err = store.control.QueryRow(`SELECT COUNT(projected_at) FROM segments`).Scan(&projected); err != nil || projected != 0 {
|
||||
t.Fatalf("projected=%d err=%v", projected, err)
|
||||
}
|
||||
budget := testQueryBudget()
|
||||
budget.MaxScannedBytes = 1
|
||||
if _, err = store.queryRawCandidate(ctx, ast, query.Scope{OrganizationID: "organization-a"}, budget, now); err == nil {
|
||||
t.Fatal("raw query ignored the scan budget")
|
||||
}
|
||||
if _, err = store.control.Exec(`UPDATE segments SET archiving_at=?,archive_path=path WHERE source_id='source-a'`, now.Format(time.RFC3339Nano)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.queryRawCandidate(ctx, ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), now); err == nil || err.Error() != "raw query segment transition is incomplete" {
|
||||
t.Fatalf("transition error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawQueryCandidateRejectsNonLogSignals(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ast, err := query.Parse(`metrics | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.queryRawCandidate(t.Context(), ast, query.Scope{OrganizationID: "organization-a"}, testQueryBudget(), time.Now().UTC()); err == nil {
|
||||
t.Fatal("non-log raw query candidate was accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
const maxRebuildSegments = 1_000_000
|
||||
|
||||
type RebuildReport struct {
|
||||
OrganizationID string `json:"organization_id"`
|
||||
Segments int `json:"segments"`
|
||||
Observations int64 `json:"observations"`
|
||||
ActiveVersion int `json:"active_projection_version"`
|
||||
IndexedRows int64 `json:"indexed_rows"`
|
||||
}
|
||||
|
||||
type rebuildSegment struct {
|
||||
digest string
|
||||
path string
|
||||
}
|
||||
|
||||
// RebuildOrganization reconstructs one disposable organization projection
|
||||
// from checksummed raw truth beside the live database, then atomically replaces
|
||||
// it. The caller must hold the data directory's exclusive process lock.
|
||||
func (s *Store) RebuildOrganization(ctx context.Context, organizationID string, now time.Time) (RebuildReport, error) {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil || now.IsZero() {
|
||||
return RebuildReport{}, errors.New("projection rebuild input is invalid")
|
||||
}
|
||||
lock := s.namedLock("organization:" + organizationID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
dir := filepath.Join(s.root, "organizations", organizationID)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return RebuildReport{}, errors.New("create projection rebuild directory")
|
||||
}
|
||||
live := filepath.Join(dir, "projection.sqlite")
|
||||
if err := requireProjectionTarget(live); err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
exists, err := s.organizationHasSource(ctx, organizationID)
|
||||
if err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
if !exists {
|
||||
return RebuildReport{}, errors.New("projection rebuild organization has no enrolled sources")
|
||||
}
|
||||
segments, err := s.rebuildSegments(ctx, organizationID)
|
||||
if err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
temporary, err := os.CreateTemp(dir, ".projection-rebuild-*.sqlite")
|
||||
if err != nil {
|
||||
return RebuildReport{}, errors.New("create projection rebuild target")
|
||||
}
|
||||
stage := temporary.Name()
|
||||
if err = temporary.Close(); err != nil {
|
||||
_ = os.Remove(stage)
|
||||
return RebuildReport{}, errors.New("close projection rebuild target")
|
||||
}
|
||||
if err = os.Remove(stage); err != nil {
|
||||
return RebuildReport{}, errors.New("prepare projection rebuild target")
|
||||
}
|
||||
defer removeProjectionFiles(stage)
|
||||
|
||||
db, err := openProjection(ctx, stage)
|
||||
if err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
if err = db.Close(); err != nil {
|
||||
return RebuildReport{}, errors.New("close empty projection rebuild target")
|
||||
}
|
||||
report := RebuildReport{OrganizationID: organizationID, ActiveVersion: 1}
|
||||
for _, entry := range segments {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
batch, readErr := s.segments.Read(entry.path, entry.digest)
|
||||
if readErr != nil {
|
||||
return RebuildReport{}, fmt.Errorf("read projection rebuild segment: %w", readErr)
|
||||
}
|
||||
if err = batch.Validate(batch.ObservedAt); err != nil {
|
||||
return RebuildReport{}, errors.New("projection rebuild segment is invalid")
|
||||
}
|
||||
source, sourceErr := s.sourceByID(ctx, batch.SourceID)
|
||||
if sourceErr != nil {
|
||||
return RebuildReport{}, sourceErr
|
||||
}
|
||||
if source.Scope.OrganizationID != organizationID {
|
||||
return RebuildReport{}, errors.New("projection rebuild segment organization mismatch")
|
||||
}
|
||||
if err = projectAt(ctx, stage, source.Scope, batch, entry.digest); err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
if int64(len(batch.Records)) > math.MaxInt64-report.Observations {
|
||||
return RebuildReport{}, errors.New("projection rebuild observation count overflow")
|
||||
}
|
||||
report.Observations += int64(len(batch.Records))
|
||||
report.Segments++
|
||||
}
|
||||
descriptors, err := s.activatedDescriptors(ctx, organizationID)
|
||||
if err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
if len(descriptors) > 0 {
|
||||
report.ActiveVersion, report.IndexedRows, err = activateRebuiltDescriptors(ctx, stage, descriptors, now)
|
||||
if err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
}
|
||||
if err = finalizeProjection(stage); err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
if err = s.closeProjection(organizationID); err != nil {
|
||||
return RebuildReport{}, errors.New("close live projection before replacement")
|
||||
}
|
||||
if err = requireProjectionTarget(live); err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
if err = removeProjectionSidecars(live); err != nil {
|
||||
return RebuildReport{}, err
|
||||
}
|
||||
if err = os.Rename(stage, live); err != nil {
|
||||
return RebuildReport{}, errors.New("activate rebuilt projection")
|
||||
}
|
||||
if err = syncProjectionDirectory(dir); err != nil {
|
||||
return RebuildReport{}, errors.New("sync rebuilt projection directory")
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (s *Store) organizationHasSource(ctx context.Context, organizationID string) (bool, error) {
|
||||
var exists int
|
||||
err := s.control.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM sources WHERE organization_id=? LIMIT 1)`, organizationID).Scan(&exists)
|
||||
if err != nil {
|
||||
return false, errors.New("verify projection rebuild organization")
|
||||
}
|
||||
return exists == 1, nil
|
||||
}
|
||||
|
||||
func (s *Store) rebuildSegments(ctx context.Context, organizationID string) ([]rebuildSegment, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT digest,path FROM segments WHERE organization_id=? AND tier='hot' AND retiring_at IS NULL ORDER BY source_id,stream_id,sequence`, organizationID)
|
||||
if err != nil {
|
||||
return nil, errors.New("list projection rebuild segments")
|
||||
}
|
||||
defer rows.Close()
|
||||
segments := make([]rebuildSegment, 0)
|
||||
for rows.Next() {
|
||||
if len(segments) >= maxRebuildSegments {
|
||||
return nil, errors.New("projection rebuild segment limit exceeded")
|
||||
}
|
||||
var entry rebuildSegment
|
||||
if err = rows.Scan(&entry.digest, &entry.path); err != nil {
|
||||
return nil, errors.New("read projection rebuild segment")
|
||||
}
|
||||
segments = append(segments, entry)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, errors.New("list projection rebuild segments")
|
||||
}
|
||||
return segments, nil
|
||||
}
|
||||
|
||||
func (s *Store) activatedDescriptors(ctx context.Context, organizationID string) ([]schema.Descriptor, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT descriptor_json FROM descriptor_proposals WHERE organization_id=? AND status='activated' ORDER BY signal,field`, organizationID)
|
||||
if err != nil {
|
||||
return nil, errors.New("list activated descriptors for rebuild")
|
||||
}
|
||||
defer rows.Close()
|
||||
descriptors := make([]schema.Descriptor, 0)
|
||||
seen := map[string]bool{}
|
||||
for rows.Next() {
|
||||
if len(descriptors) >= model.MaxDistinctFields {
|
||||
return nil, errors.New("activated descriptor rebuild limit exceeded")
|
||||
}
|
||||
var encoded string
|
||||
var descriptor schema.Descriptor
|
||||
if err = rows.Scan(&encoded); err != nil || json.Unmarshal([]byte(encoded), &descriptor) != nil || descriptor.Validate() != nil {
|
||||
return nil, errors.New("activated descriptor rebuild data is invalid")
|
||||
}
|
||||
key := string(descriptor.Signal) + ":" + query.CanonicalField(descriptor.Field)
|
||||
if seen[key] {
|
||||
return nil, errors.New("activated descriptor rebuild data is duplicated")
|
||||
}
|
||||
seen[key] = true
|
||||
descriptor.ProjectionVersion = 2
|
||||
descriptors = append(descriptors, descriptor)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, errors.New("list activated descriptors for rebuild")
|
||||
}
|
||||
sort.Slice(descriptors, func(left, right int) bool {
|
||||
if descriptors[left].Signal == descriptors[right].Signal {
|
||||
return descriptors[left].Field < descriptors[right].Field
|
||||
}
|
||||
return descriptors[left].Signal < descriptors[right].Signal
|
||||
})
|
||||
return descriptors, nil
|
||||
}
|
||||
|
||||
func activateRebuiltDescriptors(ctx context.Context, path string, descriptors []schema.Descriptor, now time.Time) (int, int64, error) {
|
||||
db, err := openProjection(ctx, path)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defer db.Close()
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, errors.New("begin rebuilt descriptor activation")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
timestamp := now.UTC().Format(time.RFC3339Nano)
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO projection_versions(version,created_at,activated_at) VALUES(2,?,?)`, timestamp, timestamp); err != nil {
|
||||
return 0, 0, errors.New("create rebuilt projection version")
|
||||
}
|
||||
for _, descriptor := range descriptors {
|
||||
encoded, marshalErr := json.Marshal(descriptor)
|
||||
if marshalErr != nil {
|
||||
return 0, 0, errors.New("encode rebuilt active descriptor")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO projection_descriptors(version,signal,field,descriptor_json) VALUES(2,?,?,?)`, descriptor.Signal, descriptor.Field, string(encoded)); err != nil {
|
||||
return 0, 0, errors.New("store rebuilt active descriptor")
|
||||
}
|
||||
}
|
||||
indexed, err := buildProjectionIndex(ctx, tx, 2, descriptors)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE projection_state SET active_version=2 WHERE id=1 AND active_version=1`); err != nil {
|
||||
return 0, 0, errors.New("activate rebuilt projection version")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return 0, 0, errors.New("commit rebuilt descriptor activation")
|
||||
}
|
||||
return 2, indexed, nil
|
||||
}
|
||||
|
||||
func requireProjectionTarget(path string) error {
|
||||
if info, err := os.Lstat(path); err == nil {
|
||||
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return errors.New("organization projection must be a regular non-symlink file")
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return errors.New("inspect organization projection")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeProjectionSidecars(path string) error {
|
||||
for _, suffix := range []string{"-wal", "-shm"} {
|
||||
sidecar := path + suffix
|
||||
info, err := os.Lstat(sidecar)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return errors.New("projection sidecar is not a regular non-symlink file")
|
||||
}
|
||||
if err = os.Remove(sidecar); err != nil {
|
||||
return errors.New("remove inactive projection sidecar")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func finalizeProjection(path string) error {
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return errors.New("open rebuilt projection for finalization")
|
||||
}
|
||||
if _, err = db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err == nil {
|
||||
_, err = db.Exec(`PRAGMA journal_mode=DELETE`)
|
||||
}
|
||||
closeErr := db.Close()
|
||||
if err != nil || closeErr != nil {
|
||||
return errors.New("finalize rebuilt projection")
|
||||
}
|
||||
if err = os.Chmod(path, 0o600); err != nil {
|
||||
return errors.New("set rebuilt projection mode")
|
||||
}
|
||||
for _, suffix := range []string{"-wal", "-shm"} {
|
||||
if _, err = os.Lstat(path + suffix); !errors.Is(err, os.ErrNotExist) {
|
||||
return errors.New("rebuilt projection retained a sidecar")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeProjectionFiles(path string) {
|
||||
_ = os.Remove(path)
|
||||
_ = os.Remove(path + "-wal")
|
||||
_ = os.Remove(path + "-shm")
|
||||
}
|
||||
|
||||
func syncProjectionDirectory(path string) error {
|
||||
directory, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer directory.Close()
|
||||
return directory.Sync()
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
func TestProjectionRebuildRestoresRawTruthAndActivatedDescriptorsAtomically(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC)
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{
|
||||
{Timestamp: now.Add(-time.Minute), Name: "queue.depth", Value: floatPointer(1), Attributes: map[string]string{"workshop.queue_depth": "1"}},
|
||||
{Timestamp: now, Name: "queue.depth", Value: floatPointer(2), Attributes: map[string]string{"workshop.queue_depth": "2"}},
|
||||
}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logBatch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{
|
||||
requestObservation(now.Add(-time.Minute), "/broken", 503, 10),
|
||||
requestObservation(now, "/healthy", 200, 5),
|
||||
}}
|
||||
if _, err = store.Ingest(ctx, token, logBatch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
reviewed := schema.Descriptor{Version: schema.DescriptorVersion, Signal: model.SignalMetrics, Field: "workshop.queue_depth", Type: schema.TypeInteger, Meaning: "Number of work items waiting in the selected service queue.", Sensitivity: schema.SensitivityInternal, Cardinality: schema.CardinalityLow, Index: schema.IndexRange, Retention: schema.RetentionRaw, ProjectionVersion: 1}
|
||||
if _, err = store.ActivateDescriptor(ctx, scope.OrganizationID, reviewed, now.Add(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ast, err := query.Parse(`metrics | where workshop.queue_depth >= 1 | sort workshop.queue_depth desc | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logAST, err := query.Parse(`logs | where status >= 500 | window 1h | summarize count() by route, window(5m) | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logBefore, err := store.Query(ctx, logAST, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
otherScope := model.Scope{OrganizationID: "organization-b", ProjectID: "project-b", EnvironmentID: "production", ServiceID: "service-b"}
|
||||
otherToken, err := store.CreateSource(ctx, "source-b", otherScope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
otherBatch := model.Batch{Version: model.BatchVersion, SourceID: "source-b", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "other.request"}}}
|
||||
if _, err = store.Ingest(ctx, otherToken, otherBatch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
otherPath := filepath.Join(store.root, "organizations", otherScope.OrganizationID, "projection.sqlite")
|
||||
otherBefore, otherInfo := readFileAndInfo(t, otherPath)
|
||||
|
||||
live := filepath.Join(store.root, "organizations", scope.OrganizationID, "projection.sqlite")
|
||||
if err = os.WriteFile(live, []byte("corrupt disposable projection"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err := store.RebuildOrganization(ctx, scope.OrganizationID, now.Add(2*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Segments != 2 || report.Observations != 4 || report.ActiveVersion != 2 || report.IndexedRows != 2 {
|
||||
t.Fatalf("report=%+v", report)
|
||||
}
|
||||
after, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(before.Columns, after.Columns) || !reflect.DeepEqual(before.Rows, after.Rows) || before.Stats.ScannedRows != after.Stats.ScannedRows || before.Stats.MatchedRows != after.Stats.MatchedRows {
|
||||
t.Fatalf("before=%+v after=%+v", before, after)
|
||||
}
|
||||
logAfter, err := store.Query(ctx, logAST, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(logBefore.Columns, logAfter.Columns) || !reflect.DeepEqual(logBefore.Rows, logAfter.Rows) || logBefore.Stats.ScannedRows != logAfter.Stats.ScannedRows || logBefore.Stats.MatchedRows != logAfter.Stats.MatchedRows {
|
||||
t.Fatalf("log before=%+v after=%+v", logBefore, logAfter)
|
||||
}
|
||||
otherAfter, otherAfterInfo := readFileAndInfo(t, otherPath)
|
||||
if !bytes.Equal(otherBefore, otherAfter) || !otherInfo.ModTime().Equal(otherAfterInfo.ModTime()) {
|
||||
t.Fatal("rebuilding one organization modified another organization projection")
|
||||
}
|
||||
if _, err = os.Lstat(live + "-wal"); !os.IsNotExist(err) {
|
||||
t.Fatalf("rebuilt projection retained WAL: %v", err)
|
||||
}
|
||||
store.projectionMu.Lock()
|
||||
_, cached := store.projections[scope.OrganizationID]
|
||||
store.projectionMu.Unlock()
|
||||
if cached {
|
||||
t.Fatal("rebuilt projection retained its replaced database handle")
|
||||
}
|
||||
batch.Sequence = 2
|
||||
batch.ObservedAt = now.Add(time.Minute)
|
||||
batch.Records = []model.Observation{{Timestamp: now.Add(time.Minute), Name: "queue.depth", Value: floatPointer(3), Attributes: map[string]string{"workshop.queue_depth": "3"}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
store.projectionMu.Lock()
|
||||
refreshed := store.projections[scope.OrganizationID]
|
||||
store.projectionMu.Unlock()
|
||||
if refreshed.db == nil {
|
||||
t.Fatal("rebuilt projection did not receive a fresh database handle")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectionRebuildFailurePreservesLiveProjection(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 12, 30, 0, 0, time.UTC)
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
live := filepath.Join(store.root, "organizations", scope.OrganizationID, "projection.sqlite")
|
||||
liveBefore, liveInfo := readFileAndInfo(t, live)
|
||||
var rawPath string
|
||||
if err = store.control.QueryRow(`SELECT path FROM segments WHERE organization_id=?`, scope.OrganizationID).Scan(&rawPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(rawPath, []byte("corrupt raw truth"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.RebuildOrganization(ctx, scope.OrganizationID, now.Add(time.Minute)); err == nil {
|
||||
t.Fatal("corrupt raw segment was accepted")
|
||||
}
|
||||
liveAfter, liveAfterInfo := readFileAndInfo(t, live)
|
||||
if !bytes.Equal(liveBefore, liveAfter) || !liveInfo.ModTime().Equal(liveAfterInfo.ModTime()) {
|
||||
t.Fatal("failed rebuild modified the live projection")
|
||||
}
|
||||
stages, err := filepath.Glob(filepath.Join(filepath.Dir(live), ".projection-rebuild-*"))
|
||||
if err != nil || len(stages) != 0 {
|
||||
t.Fatalf("stages=%v err=%v", stages, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectionRebuildRejectsUnknownOrganization(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
if _, err := store.RebuildOrganization(context.Background(), "unknown-organization", time.Now().UTC()); err == nil {
|
||||
t.Fatal("unknown organization projection was created")
|
||||
}
|
||||
path := filepath.Join(store.root, "organizations", "unknown-organization", "projection.sqlite")
|
||||
if _, err := os.Lstat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("unknown organization projection exists: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectionRebuildRefusesSymlinkProjectionOrSidecar(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 17, 13, 0, 0, 0, time.UTC)
|
||||
dir := filepath.Join(store.root, "organizations", "organization-a")
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
live := filepath.Join(dir, "projection.sqlite")
|
||||
if err := os.WriteFile(live, []byte("placeholder"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := filepath.Join(dir, "target.sqlite")
|
||||
if err := os.WriteFile(target, []byte("target"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(target, live+"-wal"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.RebuildOrganization(ctx, "organization-a", now); err == nil {
|
||||
t.Fatal("symlink projection sidecar was accepted")
|
||||
}
|
||||
if err := os.Remove(live + "-wal"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(target, live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.RebuildOrganization(ctx, "organization-a", now); err == nil {
|
||||
t.Fatal("symlink projection was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func readFileAndInfo(t *testing.T, path string) ([]byte, os.FileInfo) {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return body, info
|
||||
}
|
||||
@@ -0,0 +1,855 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/segment"
|
||||
)
|
||||
|
||||
const maximumRetentionDays = 3650
|
||||
|
||||
var ErrOrganizationStorageQuotaExceeded = errors.New("organization storage quota exceeded")
|
||||
|
||||
// RetentionPolicy is the complete storage lifecycle for one organization.
|
||||
// Raw metric samples may expire before their five-minute aggregate projection.
|
||||
type RetentionPolicy struct {
|
||||
RawLogsDays int `json:"raw_logs_days"`
|
||||
RawTracesDays int `json:"raw_traces_days"`
|
||||
RawMetricsDays int `json:"raw_metrics_days"`
|
||||
ColdRawDays int `json:"cold_raw_days"`
|
||||
DeleteColdRaw bool `json:"delete_cold_raw"`
|
||||
MetricRollupsDays int `json:"metric_rollups_days"`
|
||||
EvidenceDays int `json:"evidence_days"`
|
||||
}
|
||||
|
||||
func (policy RetentionPolicy) Validate() error {
|
||||
for _, days := range []int{policy.RawLogsDays, policy.RawTracesDays, policy.RawMetricsDays, policy.ColdRawDays, policy.MetricRollupsDays, policy.EvidenceDays} {
|
||||
if days < 1 || days > maximumRetentionDays {
|
||||
return errors.New("retention values must be between 1 and 3650 days")
|
||||
}
|
||||
}
|
||||
if policy.MetricRollupsDays < policy.RawMetricsDays {
|
||||
return errors.New("metric rollup retention cannot be shorter than raw metric retention")
|
||||
}
|
||||
if policy.ColdRawDays < policy.RawLogsDays || policy.ColdRawDays < policy.RawTracesDays || policy.ColdRawDays < policy.RawMetricsDays || policy.ColdRawDays < policy.EvidenceDays {
|
||||
return errors.New("cold raw retention cannot be shorter than a hot raw or evidence retention window")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type OrganizationRetention struct {
|
||||
OrganizationID string `json:"organization_id"`
|
||||
Policy RetentionPolicy `json:"policy"`
|
||||
QuotaBytes int64 `json:"quota_bytes,omitempty"`
|
||||
ExtensionApproved bool `json:"extension_approved"`
|
||||
ExtensionApprovedBy string `json:"extension_approved_by,omitempty"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type SetRetentionInput struct {
|
||||
OrganizationID string
|
||||
Policy RetentionPolicy
|
||||
Defaults RetentionPolicy
|
||||
ActorUserID string
|
||||
ApproveExtensionFor string
|
||||
QuotaBytes int64
|
||||
}
|
||||
|
||||
type RetentionReport struct {
|
||||
Version int `json:"version"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
Organizations int `json:"organizations"`
|
||||
RawSegmentsRemoved int `json:"raw_segments_removed"`
|
||||
RawBytesRemoved int64 `json:"raw_bytes_removed"`
|
||||
RawSegmentsArchived int `json:"raw_segments_archived"`
|
||||
RawBytesArchived int64 `json:"raw_bytes_archived"`
|
||||
ProjectedObservationsRemoved int64 `json:"projected_observations_removed"`
|
||||
MetricRollupsRemoved int64 `json:"metric_rollups_removed"`
|
||||
LogRollupsRemoved int64 `json:"log_rollups_removed"`
|
||||
ResolvedIncidentsRemoved int64 `json:"resolved_incidents_removed"`
|
||||
PolicyEventsRemoved int64 `json:"policy_events_removed"`
|
||||
}
|
||||
|
||||
func migrateControlRetention(db *sql.DB) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin retention migration: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var segmentTable int
|
||||
if err = tx.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='segments'`).Scan(&segmentTable); err != nil {
|
||||
return fmt.Errorf("inspect segment schema: %w", err)
|
||||
}
|
||||
if segmentTable == 0 {
|
||||
if _, err = tx.Exec(`CREATE TABLE segments (
|
||||
digest TEXT PRIMARY KEY,
|
||||
organization_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
stream_id TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
compressed_bytes INTEGER NOT NULL,
|
||||
uncompressed_bytes INTEGER NOT NULL,
|
||||
committed_at TEXT NOT NULL,
|
||||
projected_at TEXT,
|
||||
signal TEXT NOT NULL DEFAULT '',
|
||||
first_observed_at TEXT NOT NULL DEFAULT '',
|
||||
last_observed_at TEXT NOT NULL DEFAULT '',
|
||||
tier TEXT NOT NULL DEFAULT 'hot' CHECK(tier IN ('hot','cold')),
|
||||
archiving_at TEXT,
|
||||
archive_path TEXT,
|
||||
cold_at TEXT,
|
||||
retiring_at TEXT,
|
||||
UNIQUE(source_id,stream_id,sequence)
|
||||
)`); err != nil {
|
||||
return fmt.Errorf("create retained segment schema: %w", err)
|
||||
}
|
||||
} else {
|
||||
columns, columnErr := sqliteColumns(tx, "segments")
|
||||
if columnErr != nil {
|
||||
return columnErr
|
||||
}
|
||||
for _, column := range []struct{ name, definition string }{
|
||||
{"signal", `TEXT NOT NULL DEFAULT ''`},
|
||||
{"first_observed_at", `TEXT NOT NULL DEFAULT ''`},
|
||||
{"last_observed_at", `TEXT NOT NULL DEFAULT ''`},
|
||||
{"tier", `TEXT NOT NULL DEFAULT 'hot' CHECK(tier IN ('hot','cold'))`},
|
||||
{"archiving_at", `TEXT`},
|
||||
{"archive_path", `TEXT`},
|
||||
{"cold_at", `TEXT`},
|
||||
{"retiring_at", `TEXT`},
|
||||
} {
|
||||
if columns[column.name] {
|
||||
continue
|
||||
}
|
||||
if _, err = tx.Exec(`ALTER TABLE segments ADD COLUMN ` + column.name + ` ` + column.definition); err != nil {
|
||||
return fmt.Errorf("add segment retention column: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, statement := range []string{
|
||||
`CREATE INDEX IF NOT EXISTS segments_retention ON segments(organization_id,tier,signal,last_observed_at,retiring_at)`,
|
||||
`CREATE INDEX IF NOT EXISTS segments_archiving ON segments(archiving_at,organization_id)`,
|
||||
`CREATE TABLE IF NOT EXISTS organization_retention_policies (
|
||||
organization_id TEXT PRIMARY KEY,
|
||||
raw_logs_days INTEGER NOT NULL CHECK(raw_logs_days BETWEEN 1 AND 3650),
|
||||
raw_traces_days INTEGER NOT NULL CHECK(raw_traces_days BETWEEN 1 AND 3650),
|
||||
raw_metrics_days INTEGER NOT NULL CHECK(raw_metrics_days BETWEEN 1 AND 3650),
|
||||
cold_raw_days INTEGER NOT NULL CHECK(cold_raw_days BETWEEN 1 AND 3650),
|
||||
delete_cold_raw INTEGER NOT NULL DEFAULT 0 CHECK(delete_cold_raw IN (0,1)),
|
||||
metric_rollups_days INTEGER NOT NULL CHECK(metric_rollups_days BETWEEN 1 AND 3650),
|
||||
evidence_days INTEGER NOT NULL CHECK(evidence_days BETWEEN 1 AND 3650),
|
||||
quota_bytes INTEGER CHECK(quota_bytes IS NULL OR quota_bytes > 0),
|
||||
extension_approved_by TEXT,
|
||||
updated_by TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS retention_policy_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
organization_id TEXT NOT NULL,
|
||||
actor_user_id TEXT NOT NULL,
|
||||
action TEXT NOT NULL CHECK(action IN ('created','updated')),
|
||||
summary TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS retention_policy_events_age ON retention_policy_events(organization_id,created_at)`,
|
||||
`UPDATE schema_version SET version=7 WHERE version=6`,
|
||||
} {
|
||||
if _, err = tx.Exec(statement); err != nil {
|
||||
return fmt.Errorf("migrate retention schema: %w", err)
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit retention migration: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateControlForensicRetention(db *sql.DB) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin forensic retention migration: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
columns, err := sqliteColumns(tx, "organization_retention_policies")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !columns["delete_cold_raw"] {
|
||||
if _, err = tx.Exec(`ALTER TABLE organization_retention_policies ADD COLUMN delete_cold_raw INTEGER NOT NULL DEFAULT 0 CHECK(delete_cold_raw IN (0,1))`); err != nil {
|
||||
return fmt.Errorf("add forensic retention policy: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(`UPDATE schema_version SET version=8 WHERE version=7`); err != nil {
|
||||
return fmt.Errorf("advance forensic retention schema: %w", err)
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit forensic retention migration: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type columnQuery interface {
|
||||
Query(string, ...any) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
func sqliteColumns(db columnQuery, table string) (map[string]bool, error) {
|
||||
rows, err := db.Query(`PRAGMA table_info(` + table + `)`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect SQLite columns: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
columns := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var index, notNull, primaryKey int
|
||||
var name, valueType string
|
||||
var defaultValue any
|
||||
if err = rows.Scan(&index, &name, &valueType, ¬Null, &defaultValue, &primaryKey); err != nil {
|
||||
return nil, fmt.Errorf("read SQLite columns: %w", err)
|
||||
}
|
||||
columns[name] = true
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read SQLite columns: %w", err)
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func retentionExtended(policy, defaults RetentionPolicy) bool {
|
||||
return policy.RawLogsDays > defaults.RawLogsDays ||
|
||||
policy.RawTracesDays > defaults.RawTracesDays ||
|
||||
policy.RawMetricsDays > defaults.RawMetricsDays ||
|
||||
(policy.DeleteColdRaw && defaults.DeleteColdRaw && policy.ColdRawDays > defaults.ColdRawDays) ||
|
||||
(!policy.DeleteColdRaw && defaults.DeleteColdRaw) ||
|
||||
policy.MetricRollupsDays > defaults.MetricRollupsDays ||
|
||||
policy.EvidenceDays > defaults.EvidenceDays
|
||||
}
|
||||
|
||||
// SetOrganizationRetention records a policy selected by an organization
|
||||
// owner. Extending a server default additionally requires an exact approval
|
||||
// string and a quota larger than current organization storage.
|
||||
func (s *Store) SetOrganizationRetention(ctx context.Context, input SetRetentionInput, now time.Time) (OrganizationRetention, error) {
|
||||
if err := model.ValidateSourceID(input.OrganizationID); err != nil || model.ValidateSourceID(input.ActorUserID) != nil || input.Policy.Validate() != nil || input.Defaults.Validate() != nil || now.IsZero() {
|
||||
return OrganizationRetention{}, errors.New("retention policy input is invalid")
|
||||
}
|
||||
extended := retentionExtended(input.Policy, input.Defaults)
|
||||
if extended {
|
||||
if input.ApproveExtensionFor != input.OrganizationID || input.QuotaBytes <= 0 {
|
||||
return OrganizationRetention{}, errors.New("retention extension requires exact organization approval and a positive quota")
|
||||
}
|
||||
used, err := s.organizationStorageBytes(input.OrganizationID)
|
||||
if err != nil {
|
||||
return OrganizationRetention{}, err
|
||||
}
|
||||
if used >= input.QuotaBytes {
|
||||
return OrganizationRetention{}, errors.New("organization storage already exceeds the approved quota")
|
||||
}
|
||||
} else if input.QuotaBytes != 0 || input.ApproveExtensionFor != "" {
|
||||
return OrganizationRetention{}, errors.New("retention extension approval is not applicable")
|
||||
}
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return OrganizationRetention{}, errors.New("begin retention policy update")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var existed int
|
||||
if err = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM organization_retention_policies WHERE organization_id=?)`, input.OrganizationID).Scan(&existed); err != nil {
|
||||
return OrganizationRetention{}, errors.New("inspect retention policy")
|
||||
}
|
||||
var quota any
|
||||
var approvedBy any
|
||||
if extended {
|
||||
quota = input.QuotaBytes
|
||||
approvedBy = input.ActorUserID
|
||||
}
|
||||
timestamp := now.UTC().Format(time.RFC3339Nano)
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO organization_retention_policies(organization_id,raw_logs_days,raw_traces_days,raw_metrics_days,cold_raw_days,delete_cold_raw,metric_rollups_days,evidence_days,quota_bytes,extension_approved_by,updated_by,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(organization_id) DO UPDATE SET raw_logs_days=excluded.raw_logs_days,raw_traces_days=excluded.raw_traces_days,raw_metrics_days=excluded.raw_metrics_days,cold_raw_days=excluded.cold_raw_days,delete_cold_raw=excluded.delete_cold_raw,metric_rollups_days=excluded.metric_rollups_days,evidence_days=excluded.evidence_days,quota_bytes=excluded.quota_bytes,extension_approved_by=excluded.extension_approved_by,updated_by=excluded.updated_by,updated_at=excluded.updated_at`, input.OrganizationID, input.Policy.RawLogsDays, input.Policy.RawTracesDays, input.Policy.RawMetricsDays, input.Policy.ColdRawDays, input.Policy.DeleteColdRaw, input.Policy.MetricRollupsDays, input.Policy.EvidenceDays, quota, approvedBy, input.ActorUserID, timestamp)
|
||||
if err != nil {
|
||||
return OrganizationRetention{}, errors.New("store retention policy")
|
||||
}
|
||||
action := "created"
|
||||
if existed == 1 {
|
||||
action = "updated"
|
||||
}
|
||||
summary := fmt.Sprintf("Organization retention policy changed; delete_cold_raw=%t", input.Policy.DeleteColdRaw)
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO retention_policy_events(organization_id,actor_user_id,action,summary,created_at) VALUES(?,?,?,?,?)`, input.OrganizationID, input.ActorUserID, action, summary, timestamp); err != nil {
|
||||
return OrganizationRetention{}, errors.New("record retention policy event")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return OrganizationRetention{}, errors.New("commit retention policy update")
|
||||
}
|
||||
return OrganizationRetention{OrganizationID: input.OrganizationID, Policy: input.Policy, QuotaBytes: input.QuotaBytes, ExtensionApproved: extended, ExtensionApprovedBy: func() string {
|
||||
if extended {
|
||||
return input.ActorUserID
|
||||
}
|
||||
return ""
|
||||
}(), UpdatedBy: input.ActorUserID, UpdatedAt: now.UTC()}, nil
|
||||
}
|
||||
|
||||
func (s *Store) organizationStorageBytes(organizationID string) (int64, error) {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil {
|
||||
return 0, errors.New("invalid organization identifier")
|
||||
}
|
||||
var raw int64
|
||||
if err := s.control.QueryRow(`SELECT COALESCE(SUM(compressed_bytes),0) FROM segments WHERE organization_id=?`, organizationID).Scan(&raw); err != nil {
|
||||
return 0, errors.New("measure organization raw storage")
|
||||
}
|
||||
projected, err := s.EstimateOrganizationBytes(organizationID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if raw > int64(^uint64(0)>>1)-projected {
|
||||
return 0, errors.New("organization storage size overflow")
|
||||
}
|
||||
return raw + projected, nil
|
||||
}
|
||||
|
||||
func (s *Store) checkCommittedQuota(ctx context.Context, organizationID string, committed segment.Committed) error {
|
||||
var recorded int
|
||||
if err := s.control.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM segments WHERE digest=?)`, committed.Digest).Scan(&recorded); err != nil {
|
||||
return errors.New("inspect committed segment quota state")
|
||||
}
|
||||
if recorded == 1 {
|
||||
return nil
|
||||
}
|
||||
var quota sql.NullInt64
|
||||
err := s.control.QueryRowContext(ctx, `SELECT quota_bytes FROM organization_retention_policies WHERE organization_id=?`, organizationID).Scan("a)
|
||||
if errors.Is(err, sql.ErrNoRows) || !quota.Valid {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.New("read organization storage quota")
|
||||
}
|
||||
used, err := s.organizationStorageBytes(organizationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
addition := committed.Compressed + committed.Uncompressed
|
||||
if addition < 0 || used > quota.Int64-addition {
|
||||
return ErrOrganizationStorageQuotaExceeded
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// admitCommitted serializes quota admission across every source belonging to
|
||||
// one organization. The raw object is already durable at this point; a quota
|
||||
// rejection removes that exact checksummed object before returning. The
|
||||
// segment catalog row and stream watermark are committed together before an
|
||||
// acknowledgement can be returned. Projection is intentionally independent.
|
||||
func (s *Store) admitCommitted(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, committedAt time.Time) error {
|
||||
return s.admitCommittedEnvelope(ctx, scope, batch, committed, nil, committedAt)
|
||||
}
|
||||
|
||||
func (s *Store) admitCommittedEnvelope(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, envelope *model.BatchEnvelope, committedAt time.Time) error {
|
||||
lock := s.namedLock("quota:" + scope.OrganizationID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
if err := s.checkCommittedQuota(ctx, scope.OrganizationID, committed); err != nil {
|
||||
if deleteErr := s.segments.Delete(committed.Path, committed.Digest); deleteErr != nil {
|
||||
return fmt.Errorf("%w; remove rejected segment: %v", err, deleteErr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return s.recordCommittedAtEnvelope(ctx, scope, batch, committed, envelope, committedAt)
|
||||
}
|
||||
|
||||
func (s *Store) effectiveRetention(ctx context.Context, organizationID string, defaults RetentionPolicy) (RetentionPolicy, error) {
|
||||
if err := defaults.Validate(); err != nil {
|
||||
return RetentionPolicy{}, err
|
||||
}
|
||||
policy := defaults
|
||||
err := s.control.QueryRowContext(ctx, `SELECT raw_logs_days,raw_traces_days,raw_metrics_days,cold_raw_days,delete_cold_raw,metric_rollups_days,evidence_days FROM organization_retention_policies WHERE organization_id=?`, organizationID).Scan(&policy.RawLogsDays, &policy.RawTracesDays, &policy.RawMetricsDays, &policy.ColdRawDays, &policy.DeleteColdRaw, &policy.MetricRollupsDays, &policy.EvidenceDays)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return defaults, nil
|
||||
}
|
||||
if err != nil || policy.Validate() != nil {
|
||||
return RetentionPolicy{}, errors.New("organization retention policy is invalid")
|
||||
}
|
||||
return policy, nil
|
||||
}
|
||||
|
||||
// ApplyRetention materializes five-minute metric rollups before removing
|
||||
// expired raw projections, then retires only raw segments whose newest record
|
||||
// is outside the applicable window. Segment retirement is crash-recoverable.
|
||||
func (s *Store) ApplyRetention(ctx context.Context, defaults RetentionPolicy, now time.Time) (RetentionReport, error) {
|
||||
if defaults.Validate() != nil || now.IsZero() {
|
||||
return RetentionReport{}, errors.New("retention run input is invalid")
|
||||
}
|
||||
report := RetentionReport{Version: 1, StartedAt: now.UTC()}
|
||||
organizations, err := s.retentionOrganizations(ctx)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
for _, organizationID := range organizations {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
policy, policyErr := s.effectiveRetention(ctx, organizationID, defaults)
|
||||
if policyErr != nil {
|
||||
return report, policyErr
|
||||
}
|
||||
lock := s.namedLock("organization:" + organizationID)
|
||||
lock.Lock()
|
||||
organizationReport, applyErr := s.applyOrganizationRetention(ctx, organizationID, policy, now.UTC())
|
||||
lock.Unlock()
|
||||
if applyErr != nil {
|
||||
return report, applyErr
|
||||
}
|
||||
report.Organizations++
|
||||
report.RawSegmentsRemoved += organizationReport.RawSegmentsRemoved
|
||||
report.RawBytesRemoved += organizationReport.RawBytesRemoved
|
||||
report.RawSegmentsArchived += organizationReport.RawSegmentsArchived
|
||||
report.RawBytesArchived += organizationReport.RawBytesArchived
|
||||
report.ProjectedObservationsRemoved += organizationReport.ProjectedObservationsRemoved
|
||||
report.MetricRollupsRemoved += organizationReport.MetricRollupsRemoved
|
||||
report.LogRollupsRemoved += organizationReport.LogRollupsRemoved
|
||||
report.ResolvedIncidentsRemoved += organizationReport.ResolvedIncidentsRemoved
|
||||
report.PolicyEventsRemoved += organizationReport.PolicyEventsRemoved
|
||||
}
|
||||
report.CompletedAt = time.Now().UTC()
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (s *Store) retentionOrganizations(ctx context.Context) ([]string, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT organization_id FROM sources UNION SELECT organization_id FROM segments UNION SELECT organization_id FROM organization_retention_policies ORDER BY organization_id`)
|
||||
if err != nil {
|
||||
return nil, errors.New("list retention organizations")
|
||||
}
|
||||
defer rows.Close()
|
||||
var organizations []string
|
||||
for rows.Next() {
|
||||
var organizationID string
|
||||
if err = rows.Scan(&organizationID); err != nil || model.ValidateSourceID(organizationID) != nil {
|
||||
return nil, errors.New("retention organization is invalid")
|
||||
}
|
||||
organizations = append(organizations, organizationID)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, errors.New("list retention organizations")
|
||||
}
|
||||
return organizations, nil
|
||||
}
|
||||
|
||||
func retentionCutoff(now time.Time, days int) string {
|
||||
return now.Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
func (s *Store) applyOrganizationRetention(ctx context.Context, organizationID string, policy RetentionPolicy, now time.Time) (RetentionReport, error) {
|
||||
report := RetentionReport{Version: 1}
|
||||
path := s.organizationProjectionPath(organizationID)
|
||||
if info, err := os.Lstat(path); err == nil {
|
||||
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return report, errors.New("organization projection is unavailable")
|
||||
}
|
||||
db, openErr := openProjection(ctx, path)
|
||||
if openErr != nil {
|
||||
return report, openErr
|
||||
}
|
||||
tx, beginErr := db.BeginTx(ctx, nil)
|
||||
if beginErr != nil {
|
||||
_ = db.Close()
|
||||
return report, errors.New("begin organization retention")
|
||||
}
|
||||
for _, expiration := range []struct {
|
||||
signal model.Signal
|
||||
days int
|
||||
}{
|
||||
{model.SignalLogs, policy.RawLogsDays},
|
||||
{model.SignalTraces, policy.RawTracesDays},
|
||||
{model.SignalMetrics, policy.RawMetricsDays},
|
||||
{model.SignalDeployments, policy.EvidenceDays},
|
||||
} {
|
||||
result, deleteErr := tx.ExecContext(ctx, `DELETE FROM observations WHERE organization_id=? AND signal=? AND timestamp<?`, organizationID, expiration.signal, retentionCutoff(now, expiration.days))
|
||||
if deleteErr != nil {
|
||||
_ = tx.Rollback()
|
||||
_ = db.Close()
|
||||
return report, errors.New("remove expired observation projections")
|
||||
}
|
||||
removed, _ := result.RowsAffected()
|
||||
report.ProjectedObservationsRemoved += removed
|
||||
}
|
||||
result, deleteErr := tx.ExecContext(ctx, `DELETE FROM metric_rollups_5m WHERE organization_id=? AND bucket_start<?`, organizationID, retentionCutoff(now, policy.MetricRollupsDays))
|
||||
if deleteErr != nil {
|
||||
_ = tx.Rollback()
|
||||
_ = db.Close()
|
||||
return report, errors.New("remove expired metric rollups")
|
||||
}
|
||||
report.MetricRollupsRemoved, _ = result.RowsAffected()
|
||||
logCutoff := now.Add(-time.Duration(policy.RawLogsDays) * 24 * time.Hour).UTC().Truncate(logRollupWindow).Unix()
|
||||
result, deleteErr = tx.ExecContext(ctx, `DELETE FROM log_status_route_rollups_5m WHERE organization_id=? AND bucket_start<?`, organizationID, logCutoff)
|
||||
if deleteErr != nil {
|
||||
_ = tx.Rollback()
|
||||
_ = db.Close()
|
||||
return report, errors.New("remove expired log rollups")
|
||||
}
|
||||
report.LogRollupsRemoved, _ = result.RowsAffected()
|
||||
if err = tx.Commit(); err != nil {
|
||||
_ = db.Close()
|
||||
return report, errors.New("commit organization retention")
|
||||
}
|
||||
if report.ProjectedObservationsRemoved > 0 || report.MetricRollupsRemoved > 0 || report.LogRollupsRemoved > 0 {
|
||||
if _, err = db.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil {
|
||||
_ = db.Close()
|
||||
return report, errors.New("checkpoint retained organization projection")
|
||||
}
|
||||
if _, err = db.ExecContext(ctx, `VACUUM`); err != nil {
|
||||
_ = db.Close()
|
||||
return report, errors.New("compact retained organization projection")
|
||||
}
|
||||
}
|
||||
if err = db.Close(); err != nil {
|
||||
return report, errors.New("close retained organization projection")
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return report, errors.New("inspect organization projection for retention")
|
||||
}
|
||||
|
||||
evidenceCutoff := retentionCutoff(now, policy.EvidenceDays)
|
||||
result, err := s.control.ExecContext(ctx, `DELETE FROM incidents WHERE organization_id=? AND state='resolved' AND updated_at<?`, organizationID, evidenceCutoff)
|
||||
if err != nil {
|
||||
return report, errors.New("remove expired resolved incidents")
|
||||
}
|
||||
report.ResolvedIncidentsRemoved, _ = result.RowsAffected()
|
||||
result, err = s.control.ExecContext(ctx, `DELETE FROM retention_policy_events WHERE organization_id=? AND created_at<?`, organizationID, evidenceCutoff)
|
||||
if err != nil {
|
||||
return report, errors.New("remove expired retention policy events")
|
||||
}
|
||||
report.PolicyEventsRemoved, _ = result.RowsAffected()
|
||||
for _, expiration := range []struct {
|
||||
signal model.Signal
|
||||
days int
|
||||
}{
|
||||
{model.SignalLogs, policy.RawLogsDays},
|
||||
{model.SignalTraces, policy.RawTracesDays},
|
||||
{model.SignalMetrics, policy.RawMetricsDays},
|
||||
{model.SignalDeployments, policy.EvidenceDays},
|
||||
} {
|
||||
archived, bytes, archiveErr := s.archiveHotSegments(ctx, organizationID, expiration.signal, retentionCutoff(now, expiration.days), now)
|
||||
if archiveErr != nil {
|
||||
return report, archiveErr
|
||||
}
|
||||
report.RawSegmentsArchived += archived
|
||||
report.RawBytesArchived += bytes
|
||||
}
|
||||
if policy.DeleteColdRaw {
|
||||
if _, err = s.control.ExecContext(ctx, `UPDATE segments SET retiring_at=? WHERE organization_id=? AND tier='cold' AND projected_at IS NOT NULL AND retiring_at IS NULL AND last_observed_at<?`, now.Format(time.RFC3339Nano), organizationID, retentionCutoff(now, policy.ColdRawDays)); err != nil {
|
||||
return report, errors.New("mark expired cold segments")
|
||||
}
|
||||
}
|
||||
removed, bytes, err := s.finalizeRetiring(ctx, organizationID)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.RawSegmentsRemoved, report.RawBytesRemoved = removed, bytes
|
||||
return report, nil
|
||||
}
|
||||
|
||||
type archivingSegment struct {
|
||||
digest, path, archivePath, sourceID, streamID string
|
||||
signal model.Signal
|
||||
bytes int64
|
||||
}
|
||||
|
||||
func (s *Store) archiveHotSegments(ctx context.Context, organizationID string, signal model.Signal, cutoff string, now time.Time) (int, int64, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT digest,path,source_id,stream_id,compressed_bytes FROM segments WHERE organization_id=? AND signal=? AND tier='hot' AND projected_at IS NOT NULL AND retiring_at IS NULL AND archiving_at IS NULL AND last_observed_at<? ORDER BY last_observed_at,digest`, organizationID, signal, cutoff)
|
||||
if err != nil {
|
||||
return 0, 0, errors.New("list hot segments for archival")
|
||||
}
|
||||
var pending []archivingSegment
|
||||
for rows.Next() {
|
||||
var candidate archivingSegment
|
||||
candidate.signal = signal
|
||||
if err = rows.Scan(&candidate.digest, &candidate.path, &candidate.sourceID, &candidate.streamID, &candidate.bytes); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, 0, errors.New("read hot segment for archival")
|
||||
}
|
||||
candidate.archivePath, err = s.coldArchivePath(organizationID, candidate)
|
||||
if err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, 0, err
|
||||
}
|
||||
pending = append(pending, candidate)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return 0, 0, errors.New("close hot segments for archival")
|
||||
}
|
||||
stamp := now.UTC().Format(time.RFC3339Nano)
|
||||
for _, candidate := range pending {
|
||||
result, updateErr := s.control.ExecContext(ctx, `UPDATE segments SET archiving_at=?,archive_path=? WHERE digest=? AND organization_id=? AND tier='hot' AND archiving_at IS NULL AND retiring_at IS NULL`, stamp, candidate.archivePath, candidate.digest, organizationID)
|
||||
if updateErr != nil {
|
||||
return 0, 0, errors.New("mark hot segment for archival")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return 0, 0, errors.New("hot segment archival state changed")
|
||||
}
|
||||
}
|
||||
return s.finalizeArchiving(ctx, organizationID)
|
||||
}
|
||||
|
||||
func (s *Store) coldArchivePath(organizationID string, candidate archivingSegment) (string, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(candidate.sourceID) != nil || model.ValidateStreamID(candidate.streamID) != nil {
|
||||
return "", errors.New("cold archive identity is invalid")
|
||||
}
|
||||
switch candidate.signal {
|
||||
case model.SignalLogs, model.SignalMetrics, model.SignalTraces, model.SignalDeployments:
|
||||
default:
|
||||
return "", errors.New("cold archive signal is invalid")
|
||||
}
|
||||
name := filepath.Base(filepath.Clean(candidate.path))
|
||||
if name == "." || name == string(os.PathSeparator) || !strings.HasSuffix(name, "-"+candidate.digest+".zst") {
|
||||
return "", errors.New("cold archive segment name is invalid")
|
||||
}
|
||||
return filepath.Join(s.root, "cold", organizationID, string(candidate.signal), candidate.sourceID, candidate.streamID, name), nil
|
||||
}
|
||||
|
||||
func (s *Store) finalizeArchiving(ctx context.Context, organizationID string) (int, int64, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT digest,path,archive_path,source_id,stream_id,signal,compressed_bytes FROM segments WHERE organization_id=? AND archiving_at IS NOT NULL ORDER BY archiving_at,digest`, organizationID)
|
||||
if err != nil {
|
||||
return 0, 0, errors.New("list interrupted cold archives")
|
||||
}
|
||||
var pending []archivingSegment
|
||||
for rows.Next() {
|
||||
var candidate archivingSegment
|
||||
if err = rows.Scan(&candidate.digest, &candidate.path, &candidate.archivePath, &candidate.sourceID, &candidate.streamID, &candidate.signal, &candidate.bytes); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, 0, errors.New("read interrupted cold archive")
|
||||
}
|
||||
pending = append(pending, candidate)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return 0, 0, errors.New("close interrupted cold archives")
|
||||
}
|
||||
archived := 0
|
||||
var archivedBytes int64
|
||||
for _, candidate := range pending {
|
||||
expected, pathErr := s.coldArchivePath(organizationID, candidate)
|
||||
if pathErr != nil || expected != candidate.archivePath {
|
||||
return archived, archivedBytes, errors.New("cold archive target is invalid")
|
||||
}
|
||||
if err = s.segments.MoveToCold(candidate.path, candidate.archivePath, candidate.digest); err != nil {
|
||||
return archived, archivedBytes, err
|
||||
}
|
||||
result, updateErr := s.control.ExecContext(ctx, `UPDATE segments SET path=archive_path,tier='cold',cold_at=archiving_at,archiving_at=NULL,archive_path=NULL WHERE digest=? AND organization_id=? AND tier='hot' AND archiving_at IS NOT NULL AND archive_path=?`, candidate.digest, organizationID, candidate.archivePath)
|
||||
if updateErr != nil {
|
||||
return archived, archivedBytes, errors.New("complete cold segment archival")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return archived, archivedBytes, errors.New("cold segment archival state changed")
|
||||
}
|
||||
archived++
|
||||
if candidate.bytes > math.MaxInt64-archivedBytes {
|
||||
return archived, archivedBytes, errors.New("cold archive byte count overflow")
|
||||
}
|
||||
archivedBytes += candidate.bytes
|
||||
removeEmptyPrivateDirectory(filepath.Dir(candidate.path))
|
||||
removeEmptyPrivateDirectory(filepath.Dir(filepath.Dir(candidate.path)))
|
||||
}
|
||||
return archived, archivedBytes, nil
|
||||
}
|
||||
|
||||
func (s *Store) finalizeRetiring(ctx context.Context, organizationID string) (int, int64, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT digest,path,compressed_bytes FROM segments WHERE organization_id=? AND retiring_at IS NOT NULL ORDER BY retiring_at,digest`, organizationID)
|
||||
if err != nil {
|
||||
return 0, 0, errors.New("list retiring raw segments")
|
||||
}
|
||||
type retiringSegment struct {
|
||||
digest, path string
|
||||
bytes int64
|
||||
}
|
||||
var pending []retiringSegment
|
||||
for rows.Next() {
|
||||
var segment retiringSegment
|
||||
if err = rows.Scan(&segment.digest, &segment.path, &segment.bytes); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, 0, errors.New("read retiring raw segment")
|
||||
}
|
||||
pending = append(pending, segment)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return 0, 0, errors.New("close retiring raw segments")
|
||||
}
|
||||
path := s.organizationProjectionPath(organizationID)
|
||||
var projection *sql.DB
|
||||
if _, err = os.Lstat(path); err == nil {
|
||||
projection, err = openProjection(ctx, path)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defer projection.Close()
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return 0, 0, errors.New("inspect retiring segment projection")
|
||||
}
|
||||
removed := 0
|
||||
var removedBytes int64
|
||||
for _, segment := range pending {
|
||||
if projection != nil {
|
||||
tx, beginErr := projection.BeginTx(ctx, nil)
|
||||
if beginErr != nil {
|
||||
return removed, removedBytes, errors.New("begin retiring segment projection")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM observations WHERE organization_id=? AND segment_digest=?`, organizationID, segment.digest); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("remove retiring segment projection")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM metric_rollup_segments WHERE segment_digest=?`, segment.digest); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("remove retiring segment rollup ledger")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM log_rollup_segments WHERE segment_digest=?`, segment.digest); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("remove retiring segment log rollup ledger")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return removed, removedBytes, errors.New("commit retiring segment projection")
|
||||
}
|
||||
}
|
||||
if err = s.segments.Delete(segment.path, segment.digest); err != nil {
|
||||
return removed, removedBytes, err
|
||||
}
|
||||
tx, beginErr := s.control.BeginTx(ctx, nil)
|
||||
if beginErr != nil {
|
||||
return removed, removedBytes, errors.New("begin retiring segment acknowledgement")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM descriptor_proposal_segments WHERE segment_digest=?`, segment.digest); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("remove retired segment proposal evidence")
|
||||
}
|
||||
result, deleteErr := tx.ExecContext(ctx, `DELETE FROM segments WHERE digest=? AND organization_id=? AND retiring_at IS NOT NULL`, segment.digest, organizationID)
|
||||
if deleteErr != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("acknowledge retired segment")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("retired segment state changed")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return removed, removedBytes, errors.New("commit retired segment acknowledgement")
|
||||
}
|
||||
removed++
|
||||
removedBytes += segment.bytes
|
||||
removeEmptyPrivateDirectory(filepath.Dir(segment.path))
|
||||
removeEmptyPrivateDirectory(filepath.Dir(filepath.Dir(segment.path)))
|
||||
}
|
||||
return removed, removedBytes, nil
|
||||
}
|
||||
|
||||
func (s *Store) finishInterruptedRetention(ctx context.Context) error {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT DISTINCT organization_id FROM segments WHERE retiring_at IS NOT NULL ORDER BY organization_id`)
|
||||
if err != nil {
|
||||
return errors.New("list interrupted retention organizations")
|
||||
}
|
||||
var organizations []string
|
||||
for rows.Next() {
|
||||
var organizationID string
|
||||
if err = rows.Scan(&organizationID); err != nil {
|
||||
_ = rows.Close()
|
||||
return errors.New("read interrupted retention organization")
|
||||
}
|
||||
organizations = append(organizations, organizationID)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return errors.New("close interrupted retention organizations")
|
||||
}
|
||||
for _, organizationID := range organizations {
|
||||
lock := s.namedLock("organization:" + organizationID)
|
||||
lock.Lock()
|
||||
_, _, finalizeErr := s.finalizeRetiring(ctx, organizationID)
|
||||
lock.Unlock()
|
||||
if finalizeErr != nil {
|
||||
return finalizeErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) finishInterruptedArchival(ctx context.Context) error {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT DISTINCT organization_id FROM segments WHERE archiving_at IS NOT NULL ORDER BY organization_id`)
|
||||
if err != nil {
|
||||
return errors.New("list interrupted archive organizations")
|
||||
}
|
||||
var organizations []string
|
||||
for rows.Next() {
|
||||
var organizationID string
|
||||
if err = rows.Scan(&organizationID); err != nil || model.ValidateSourceID(organizationID) != nil {
|
||||
_ = rows.Close()
|
||||
return errors.New("read interrupted archive organization")
|
||||
}
|
||||
organizations = append(organizations, organizationID)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return errors.New("close interrupted archive organizations")
|
||||
}
|
||||
for _, organizationID := range organizations {
|
||||
lock := s.namedLock("organization:" + organizationID)
|
||||
lock.Lock()
|
||||
_, _, finalizeErr := s.finalizeArchiving(ctx, organizationID)
|
||||
lock.Unlock()
|
||||
if finalizeErr != nil {
|
||||
return finalizeErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) organizationProjectionPath(organizationID string) string {
|
||||
return filepath.Join(s.root, "organizations", organizationID, "projection.sqlite")
|
||||
}
|
||||
|
||||
func (s *Store) backfillSegmentRetentionMetadata(ctx context.Context) error {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT digest,path FROM segments WHERE signal='' OR first_observed_at='' OR last_observed_at='' ORDER BY committed_at,digest`)
|
||||
if err != nil {
|
||||
return errors.New("list legacy segment metadata")
|
||||
}
|
||||
type legacy struct{ digest, path string }
|
||||
var pending []legacy
|
||||
for rows.Next() {
|
||||
var item legacy
|
||||
if err = rows.Scan(&item.digest, &item.path); err != nil {
|
||||
_ = rows.Close()
|
||||
return errors.New("read legacy segment metadata")
|
||||
}
|
||||
pending = append(pending, item)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return errors.New("close legacy segment metadata")
|
||||
}
|
||||
for _, item := range pending {
|
||||
batch, readErr := s.segments.Read(item.path, item.digest)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("backfill segment metadata: %w", readErr)
|
||||
}
|
||||
if len(batch.Records) == 0 {
|
||||
return errors.New("backfill segment metadata: segment has no records")
|
||||
}
|
||||
first, last := observationRange(batch)
|
||||
result, updateErr := s.control.ExecContext(ctx, `UPDATE segments SET signal=?,first_observed_at=?,last_observed_at=? WHERE digest=? AND (signal='' OR first_observed_at='' OR last_observed_at='')`, batch.Signal, first.Format(time.RFC3339Nano), last.Format(time.RFC3339Nano), item.digest)
|
||||
if updateErr != nil {
|
||||
return errors.New("backfill segment metadata")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return errors.New("segment metadata changed during backfill")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeEmptyPrivateDirectory(path string) {
|
||||
info, err := os.Lstat(path)
|
||||
if err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,564 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/segment"
|
||||
)
|
||||
|
||||
var previewRetention = RetentionPolicy{RawLogsDays: 30, RawTracesDays: 30, RawMetricsDays: 14, ColdRawDays: 400, MetricRollupsDays: 400, EvidenceDays: 400}
|
||||
|
||||
func TestMetricRollupsAggregateAndBackfill(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := t.Context()
|
||||
scope := model.Scope{OrganizationID: "org-rollup", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-rollup", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 17, 4, 8, 0, 0, time.UTC)
|
||||
values := []float64{-2, 0, 3, 100}
|
||||
records := make([]model.Observation, 0, len(values))
|
||||
for index := range values {
|
||||
value := values[index]
|
||||
records = append(records, model.Observation{Timestamp: now.Add(time.Duration(index) * time.Second), Name: "http.duration", Value: &value, Attributes: map[string]string{"http.route": "/", "private.token_hint": "must-not-roll-up"}})
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-rollup", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: records}
|
||||
ack, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
if err = projectAt(ctx, store.organizationProjectionPath(scope.OrganizationID), scope, batch, ack.Digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db := openTestProjection(t, store.organizationProjectionPath(scope.OrganizationID))
|
||||
defer db.Close()
|
||||
var count int64
|
||||
var sum, minimum, maximum float64
|
||||
var histogram, dimensions string
|
||||
if err = db.QueryRow(`SELECT sample_count,value_sum,value_min,value_max,histogram_json,attributes_json FROM metric_rollups_5m`).Scan(&count, &sum, &minimum, &maximum, &histogram, &dimensions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 4 || sum != 101 || minimum != -2 || maximum != 100 {
|
||||
t.Fatalf("count=%d sum=%g min=%g max=%g", count, sum, minimum, maximum)
|
||||
}
|
||||
var ledger int
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM metric_rollup_segments WHERE segment_digest=?`, ack.Digest).Scan(&ledger); err != nil || ledger != 1 {
|
||||
t.Fatalf("rollup ledger=%d err=%v", ledger, err)
|
||||
}
|
||||
if dimensions != `{"http.route":"/"}` {
|
||||
t.Fatalf("rollup dimensions=%s", dimensions)
|
||||
}
|
||||
bins, err := decodeHistogram(histogram)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p95, ok := histogramPercentile(bins, .95)
|
||||
if !ok || math.Abs(p95-100) > 2 {
|
||||
t.Fatalf("p95=%g ok=%t", p95, ok)
|
||||
}
|
||||
|
||||
// Simulate the projection shape from a prior private preview. The first
|
||||
// open performs one transactional backfill and records its version.
|
||||
legacyRoot := filepath.Join(t.TempDir(), "data")
|
||||
if err = os.MkdirAll(filepath.Join(legacyRoot, "organizations", "legacy"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacyPath := filepath.Join(legacyRoot, "organizations", "legacy", "projection.sqlite")
|
||||
legacy, err := sql.Open("sqlite", legacyPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = legacy.Exec(`CREATE TABLE observations (
|
||||
organization_id TEXT NOT NULL, project_id TEXT NOT NULL, environment_id TEXT NOT NULL, service_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL, stream_id TEXT NOT NULL, sequence INTEGER NOT NULL, record_index INTEGER NOT NULL,
|
||||
signal TEXT NOT NULL, timestamp TEXT NOT NULL, name TEXT NOT NULL, severity TEXT, body TEXT, value REAL,
|
||||
trace_id TEXT, span_id TEXT, correlation_id TEXT, attributes_json TEXT NOT NULL, segment_digest TEXT NOT NULL,
|
||||
PRIMARY KEY(source_id,stream_id,sequence,record_index));
|
||||
INSERT INTO observations VALUES('legacy','project','prod','web','source','metrics',1,0,'metrics','2026-08-17T04:01:00Z','queue',NULL,NULL,5,NULL,NULL,NULL,'{}','digest')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = legacy.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err = openProjection(ctx, legacyPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer legacy.Close()
|
||||
if err = legacy.QueryRow(`SELECT sample_count,value_sum FROM metric_rollups_5m`).Scan(&count, &sum); err != nil || count != 1 || sum != 5 {
|
||||
t.Fatalf("backfilled count=%d sum=%g err=%v", count, sum, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionPreservesRollupsAndRetiresRawSegments(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := t.Context()
|
||||
scope := model.Scope{OrganizationID: "org-retain", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-retain", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 17, 5, 0, 0, 0, time.UTC)
|
||||
ingest := func(stream string, signal model.Signal, timestamp time.Time, value *float64) string {
|
||||
t.Helper()
|
||||
observation := model.Observation{Timestamp: timestamp, Name: "sample", Value: value}
|
||||
if signal == model.SignalLogs {
|
||||
observation = requestObservation(timestamp, "/retained", 503, 1)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-retain", StreamID: stream, Sequence: 1, ObservedAt: now, Signal: signal, Records: []model.Observation{observation}}
|
||||
ack, ingestErr := store.Ingest(ctx, token, batch, now)
|
||||
if ingestErr != nil {
|
||||
t.Fatal(ingestErr)
|
||||
}
|
||||
var path string
|
||||
if ingestErr = store.control.QueryRow(`SELECT path FROM segments WHERE digest=?`, ack.Digest).Scan(&path); ingestErr != nil {
|
||||
t.Fatal(ingestErr)
|
||||
}
|
||||
return path
|
||||
}
|
||||
metricValue := 42.0
|
||||
logPath := ingest("logs", model.SignalLogs, now.Add(-31*24*time.Hour), nil)
|
||||
metricPath := ingest("metrics", model.SignalMetrics, now.Add(-15*24*time.Hour), &metricValue)
|
||||
deploymentPath := ingest("deployments", model.SignalDeployments, now.Add(-399*24*time.Hour), nil)
|
||||
projectAll(t, store)
|
||||
report, err := store.ApplyRetention(ctx, previewRetention, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.RawSegmentsRemoved != 0 || report.RawSegmentsArchived != 2 || report.ProjectedObservationsRemoved != 2 || report.MetricRollupsRemoved != 0 || report.LogRollupsRemoved != 1 {
|
||||
t.Fatalf("report=%+v", report)
|
||||
}
|
||||
for _, hot := range []string{logPath, metricPath} {
|
||||
if _, err = os.Lstat(hot); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("hot path still exists: %s err=%v", hot, err)
|
||||
}
|
||||
}
|
||||
for _, signal := range []model.Signal{model.SignalLogs, model.SignalMetrics} {
|
||||
var path, tier string
|
||||
if err = store.control.QueryRow(`SELECT path,tier FROM segments WHERE organization_id=? AND signal=?`, scope.OrganizationID, signal).Scan(&path, &tier); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tier != "cold" || !strings.Contains(filepath.ToSlash(path), "/cold/") {
|
||||
t.Fatalf("signal=%s tier=%s path=%s", signal, tier, path)
|
||||
}
|
||||
if _, err = os.Stat(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err = os.Lstat(deploymentPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metricAST, err := query.Parse(`metrics | window 720h | summarize count(), p95(value) | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metricResult, err := store.Query(ctx, metricAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if metricResult.Stats.Approximate || len(metricResult.Rows) != 1 || metricResult.Rows[0].Values[0] == nil || *metricResult.Rows[0].Values[0] != "1" || metricResult.Rows[0].Values[1] == nil || *metricResult.Rows[0].Values[1] != "42" || len(metricResult.Explain.ProjectedSources) != 2 || !strings.HasSuffix(metricResult.Explain.ProjectedSources[1], "/cold:raw") {
|
||||
t.Fatalf("cold metric result=%+v", metricResult)
|
||||
}
|
||||
logAST, err := query.Parse(`logs | window 960h | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logResult, err := store.Query(ctx, logAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now)
|
||||
if err != nil || len(logResult.Rows) != 1 || len(logResult.Explain.ProjectedSources) != 2 || !strings.HasSuffix(logResult.Explain.ProjectedSources[1], "/cold:raw") {
|
||||
t.Fatalf("cold log result=%+v err=%v", logResult, err)
|
||||
}
|
||||
if _, err = store.Query(ctx, logAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1}, now); !errors.Is(err, query.ErrBudgetExceeded) {
|
||||
t.Fatalf("cold query memory budget err=%v", err)
|
||||
}
|
||||
db := openTestProjection(t, store.organizationProjectionPath(scope.OrganizationID))
|
||||
var observations, rollups, logRollups int
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&observations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM metric_rollups_5m`).Scan(&rollups); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM log_status_route_rollups_5m`).Scan(&logRollups); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if observations != 1 || rollups != 1 || logRollups != 0 {
|
||||
t.Fatalf("observations=%d metric_rollups=%d log_rollups=%d", observations, rollups, logRollups)
|
||||
}
|
||||
info, statErr := os.Stat(store.organizationProjectionPath(scope.OrganizationID))
|
||||
if statErr != nil {
|
||||
t.Fatal(statErr)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("projection mode=%v", info.Mode().Perm())
|
||||
}
|
||||
var rawSegments, ledger int
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&rawSegments); err != nil || rawSegments != 3 {
|
||||
t.Fatalf("raw_segments=%d err=%v", rawSegments, err)
|
||||
}
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM metric_rollup_segments`).Scan(&ledger); err != nil || ledger != 1 {
|
||||
t.Fatalf("rollup ledger=%d err=%v", ledger, err)
|
||||
}
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM log_rollup_segments`).Scan(&ledger); err != nil || ledger != 1 {
|
||||
t.Fatalf("log rollup ledger=%d err=%v", ledger, err)
|
||||
}
|
||||
if err = db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finalReport, err := store.ApplyRetention(ctx, previewRetention, now.Add(401*24*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if finalReport.RawSegmentsRemoved != 0 || finalReport.MetricRollupsRemoved != 1 {
|
||||
t.Fatalf("final report=%+v", finalReport)
|
||||
}
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&rawSegments); err != nil || rawSegments != 3 {
|
||||
t.Fatalf("preserved raw_segments=%d err=%v", rawSegments, err)
|
||||
}
|
||||
deletingPolicy := previewRetention
|
||||
deletingPolicy.DeleteColdRaw = true
|
||||
deletionReport, err := store.ApplyRetention(ctx, deletingPolicy, now.Add(401*24*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if deletionReport.RawSegmentsRemoved != 3 {
|
||||
t.Fatalf("deletion report=%+v", deletionReport)
|
||||
}
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&rawSegments); err != nil || rawSegments != 0 {
|
||||
t.Fatalf("final raw_segments=%d err=%v", rawSegments, err)
|
||||
}
|
||||
db = openTestProjection(t, store.organizationProjectionPath(scope.OrganizationID))
|
||||
defer db.Close()
|
||||
if err = db.QueryRow(`SELECT COUNT(*) FROM log_rollup_segments`).Scan(&ledger); err != nil || ledger != 0 {
|
||||
t.Fatalf("final log rollup ledger=%d err=%v", ledger, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetainingColdRawBeyondDefaultRequiresApprovalOnlyWhenServerDeletes(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
defaults := previewRetention
|
||||
defaults.DeleteColdRaw = true
|
||||
policy := defaults
|
||||
policy.DeleteColdRaw = false
|
||||
input := SetRetentionInput{OrganizationID: "org-forensic", Policy: policy, Defaults: defaults, ActorUserID: "owner"}
|
||||
if _, err := store.SetOrganizationRetention(t.Context(), input, time.Now().UTC()); err == nil {
|
||||
t.Fatal("indefinite forensic retention without extension approval was accepted")
|
||||
}
|
||||
input.ApproveExtensionFor = input.OrganizationID
|
||||
input.QuotaBytes = 1
|
||||
retained, err := store.SetOrganizationRetention(t.Context(), input, time.Now().UTC())
|
||||
if err != nil || !retained.ExtensionApproved || retained.Policy.DeleteColdRaw {
|
||||
t.Fatalf("retained=%+v err=%v", retained, err)
|
||||
}
|
||||
var summary string
|
||||
if err = store.control.QueryRow(`SELECT summary FROM retention_policy_events WHERE organization_id=?`, input.OrganizationID).Scan(&summary); err != nil || !strings.Contains(summary, "delete_cold_raw=false") {
|
||||
t.Fatalf("summary=%q err=%v", summary, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestColdCutoffDoesNotExtendAnIndefinitePolicy(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
policy := previewRetention
|
||||
policy.ColdRawDays++
|
||||
retained, err := store.SetOrganizationRetention(t.Context(), SetRetentionInput{
|
||||
OrganizationID: "org-indefinite", Policy: policy, Defaults: previewRetention, ActorUserID: "owner",
|
||||
}, time.Now().UTC())
|
||||
if err != nil || retained.ExtensionApproved {
|
||||
t.Fatalf("retained=%+v err=%v", retained, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricRollupRejectsOutOfRangeValueBeforeRawCommit(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := t.Context()
|
||||
scope := model.Scope{OrganizationID: "org-range", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-range", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
value := maxMetricMagnitude * 2
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-range", StreamID: "metrics", Sequence: 1, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: now, Name: "sample", Value: &value}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err == nil {
|
||||
t.Fatal("out-of-range metric value was accepted")
|
||||
}
|
||||
entries, err := store.segments.List()
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatalf("raw entries=%d err=%v", len(entries), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestColdQueryCombinesHotAndColdWithoutDuplication(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := t.Context()
|
||||
scope := model.Scope{OrganizationID: "org-hot-cold", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-hot-cold", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 17, 6, 0, 0, 0, time.UTC)
|
||||
for index, sample := range []struct {
|
||||
stream string
|
||||
at time.Time
|
||||
}{{"old", now.Add(-31 * 24 * time.Hour)}, {"new", now.Add(-time.Hour)}} {
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-hot-cold", StreamID: sample.stream, Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: sample.at, Name: "sample", Body: string(rune('a' + index))}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
projectAll(t, store)
|
||||
if _, err = store.ApplyRetention(ctx, previewRetention, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ast, err := query.Parse(`logs | window 960h | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Rows) != 2 || result.Stats.MatchedRows != 2 || len(result.Explain.ProjectedSources) != 2 || !strings.HasSuffix(result.Explain.ProjectedSources[1], "/cold:raw") {
|
||||
t.Fatalf("combined result=%+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionExtensionRequiresApprovalAndEnforcesQuota(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := t.Context()
|
||||
now := time.Date(2026, 8, 17, 5, 0, 0, 0, time.UTC)
|
||||
scope := model.Scope{OrganizationID: "org-quota", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-quota", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extended := previewRetention
|
||||
extended.RawLogsDays++
|
||||
input := SetRetentionInput{OrganizationID: scope.OrganizationID, Policy: extended, Defaults: previewRetention, ActorUserID: "owner"}
|
||||
if _, err = store.SetOrganizationRetention(ctx, input, now); err == nil {
|
||||
t.Fatal("retention extension without approval was accepted")
|
||||
}
|
||||
used, err := store.organizationStorageBytes(scope.OrganizationID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input.ApproveExtensionFor, input.QuotaBytes = scope.OrganizationID, used+1
|
||||
policy, err := store.SetOrganizationRetention(ctx, input, now)
|
||||
if err != nil || !policy.ExtensionApproved {
|
||||
t.Fatalf("policy=%+v err=%v", policy, err)
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-quota", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err == nil || err.Error() != "organization storage quota exceeded" {
|
||||
t.Fatalf("quota ingest err=%v", err)
|
||||
}
|
||||
var segments int
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&segments); err != nil || segments != 0 {
|
||||
t.Fatalf("segments=%d err=%v", segments, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuotaAdmissionSerializesSourcesAndRecoveryCannotBypassQuota(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := t.Context()
|
||||
now := time.Date(2026, 8, 17, 5, 30, 0, 0, time.UTC)
|
||||
scope := model.Scope{OrganizationID: "org-quota-race", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
for _, sourceID := range []string{"source-race-a", "source-race-b"} {
|
||||
if _, err := store.CreateSource(ctx, sourceID, scope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
batches := []model.Batch{
|
||||
{Version: model.BatchVersion, SourceID: "source-race-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}},
|
||||
{Version: model.BatchVersion, SourceID: "source-race-b", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}},
|
||||
}
|
||||
committed := make([]segment.Committed, len(batches))
|
||||
var err error
|
||||
for index := range batches {
|
||||
committed[index], err = store.segments.Commit(scope, batches[index])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
quota := committed[0].Compressed + committed[0].Uncompressed
|
||||
other := committed[1].Compressed + committed[1].Uncompressed
|
||||
if other > quota {
|
||||
quota = other
|
||||
}
|
||||
extended := previewRetention
|
||||
extended.RawLogsDays++
|
||||
if _, err = store.SetOrganizationRetention(ctx, SetRetentionInput{OrganizationID: scope.OrganizationID, Policy: extended, Defaults: previewRetention, ActorUserID: "owner", ApproveExtensionFor: scope.OrganizationID, QuotaBytes: quota}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
errorsBySource := make(chan error, len(batches))
|
||||
var group sync.WaitGroup
|
||||
for index := range batches {
|
||||
group.Add(1)
|
||||
go func(index int) {
|
||||
defer group.Done()
|
||||
errorsBySource <- store.admitCommitted(ctx, scope, batches[index], committed[index], time.Now().UTC())
|
||||
}(index)
|
||||
}
|
||||
group.Wait()
|
||||
close(errorsBySource)
|
||||
accepted, rejected := 0, 0
|
||||
for admissionErr := range errorsBySource {
|
||||
switch {
|
||||
case admissionErr == nil:
|
||||
accepted++
|
||||
case errors.Is(admissionErr, ErrOrganizationStorageQuotaExceeded):
|
||||
rejected++
|
||||
default:
|
||||
t.Fatalf("admission error=%v", admissionErr)
|
||||
}
|
||||
}
|
||||
if accepted != 1 || rejected != 1 {
|
||||
t.Fatalf("accepted=%d rejected=%d", accepted, rejected)
|
||||
}
|
||||
var recorded int
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments WHERE organization_id=?`, scope.OrganizationID).Scan(&recorded); err != nil || recorded != 1 {
|
||||
t.Fatalf("recorded=%d err=%v", recorded, err)
|
||||
}
|
||||
|
||||
recoveryScope := model.Scope{OrganizationID: "org-quota-recovery", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
if _, err = store.CreateSource(ctx, "source-recovery", recoveryScope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recoveryBatch := model.Batch{Version: model.BatchVersion, SourceID: "source-recovery", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}}
|
||||
recoverySegment, err := store.segments.Commit(recoveryScope, recoveryBatch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.SetOrganizationRetention(ctx, SetRetentionInput{OrganizationID: recoveryScope.OrganizationID, Policy: extended, Defaults: previewRetention, ActorUserID: "owner", ApproveExtensionFor: recoveryScope.OrganizationID, QuotaBytes: 1}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.Recover(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = os.Lstat(recoverySegment.Path); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("over-quota recovery segment still exists: %v", err)
|
||||
}
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments WHERE organization_id=?`, recoveryScope.OrganizationID).Scan(&recorded); err != nil || recorded != 0 {
|
||||
t.Fatalf("recovered records=%d err=%v", recorded, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverFinishesInterruptedRetention(t *testing.T) {
|
||||
store := testStore(t)
|
||||
ctx := context.Background()
|
||||
scope := model.Scope{OrganizationID: "org-recover", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-recover", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-recover", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}}
|
||||
ack, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var path string
|
||||
if _, err = store.control.Exec(`UPDATE segments SET retiring_at=? WHERE digest=?`, now.Format(time.RFC3339Nano), ack.Digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.control.QueryRow(`SELECT path FROM segments WHERE digest=?`, ack.Digest).Scan(&path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.Recover(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("retiring segment exists after recovery: %v", err)
|
||||
}
|
||||
var segments int
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&segments); err != nil || segments != 0 {
|
||||
t.Fatalf("segments=%d err=%v", segments, err)
|
||||
}
|
||||
if err = store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverFinishesInterruptedColdArchivesBeforeProjectionRecovery(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := t.Context()
|
||||
scope := model.Scope{OrganizationID: "org-archive-recover", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-archive", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for sequence, stream := range []string{"before-move", "after-move"} {
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-archive", StreamID: stream, Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "sample"}}}
|
||||
ack, ingestErr := store.Ingest(ctx, token, batch, now)
|
||||
if ingestErr != nil {
|
||||
t.Fatal(ingestErr)
|
||||
}
|
||||
var candidate archivingSegment
|
||||
candidate.digest, candidate.sourceID, candidate.streamID, candidate.signal = ack.Digest, batch.SourceID, batch.StreamID, batch.Signal
|
||||
if ingestErr = store.control.QueryRow(`SELECT path,compressed_bytes FROM segments WHERE digest=?`, ack.Digest).Scan(&candidate.path, &candidate.bytes); ingestErr != nil {
|
||||
t.Fatal(ingestErr)
|
||||
}
|
||||
candidate.archivePath, ingestErr = store.coldArchivePath(scope.OrganizationID, candidate)
|
||||
if ingestErr != nil {
|
||||
t.Fatal(ingestErr)
|
||||
}
|
||||
if _, ingestErr = store.control.Exec(`UPDATE segments SET archiving_at=?,archive_path=? WHERE digest=?`, now.Format(time.RFC3339Nano), candidate.archivePath, ack.Digest); ingestErr != nil {
|
||||
t.Fatal(ingestErr)
|
||||
}
|
||||
if sequence == 1 {
|
||||
if ingestErr = store.segments.MoveToCold(candidate.path, candidate.archivePath, candidate.digest); ingestErr != nil {
|
||||
t.Fatal(ingestErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err = store.Recover(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows, err := store.control.Query(`SELECT path,tier,archiving_at,archive_path FROM segments WHERE organization_id=? ORDER BY stream_id`, scope.OrganizationID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
count := 0
|
||||
for rows.Next() {
|
||||
var path, tier string
|
||||
var archivingAt, archivePath sql.NullString
|
||||
if err = rows.Scan(&path, &tier, &archivingAt, &archivePath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tier != "cold" || archivingAt.Valid || archivePath.Valid || !strings.Contains(filepath.ToSlash(path), "/cold/") {
|
||||
t.Fatalf("path=%s tier=%s archiving=%+v archive_path=%+v", path, tier, archivingAt, archivePath)
|
||||
}
|
||||
if _, err = os.Stat(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count++
|
||||
}
|
||||
if err = rows.Err(); err != nil || count != 2 {
|
||||
t.Fatalf("archives=%d err=%v", count, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
metricRollupVersion = 1
|
||||
metricRollupWindow = 5 * time.Minute
|
||||
maxMetricRollupGroupsBatch = 1024
|
||||
maxHistogramBins = 8192
|
||||
maxHistogramJSON = 256 << 10
|
||||
metricHistogramScale = 64.0
|
||||
maxMetricMagnitude = 1e25
|
||||
)
|
||||
|
||||
type histogramBin struct {
|
||||
Bucket int64 `json:"bucket"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
type metricRollup struct {
|
||||
bucket, projectID, environmentID, serviceID, name string
|
||||
dimensionsDigest, attributesJSON string
|
||||
sampleCount, valueCount int64
|
||||
sum, minimum, maximum, lastValue float64
|
||||
lastTimestamp string
|
||||
bins map[int64]int64
|
||||
}
|
||||
|
||||
func validateMetricRollupCardinality(batch model.Batch) error {
|
||||
if batch.Signal != model.SignalMetrics {
|
||||
return nil
|
||||
}
|
||||
groups := map[string]struct{}{}
|
||||
sums := map[string]float64{}
|
||||
for _, observation := range batch.Records {
|
||||
if observation.Value == nil || math.Abs(*observation.Value) > maxMetricMagnitude {
|
||||
return errors.New("metric value exceeds rollup numeric range")
|
||||
}
|
||||
dimensions, err := retainedMetricDimensions(observation.Attributes, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attributes, err := json.Marshal(dimensions)
|
||||
if err != nil {
|
||||
return errors.New("encode metric rollup dimensions")
|
||||
}
|
||||
bucket := observation.Timestamp.UTC().Truncate(metricRollupWindow).Format(time.RFC3339Nano)
|
||||
key := bucket + "\x00" + observation.Name + "\x00" + string(attributes)
|
||||
groups[key] = struct{}{}
|
||||
sums[key] += *observation.Value
|
||||
if math.IsNaN(sums[key]) || math.IsInf(sums[key], 0) {
|
||||
return errors.New("metric batch sum exceeds rollup numeric range")
|
||||
}
|
||||
if len(groups) > maxMetricRollupGroupsBatch {
|
||||
return fmt.Errorf("metric batch exceeds %d rollup groups", maxMetricRollupGroupsBatch)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureMetricRollups(ctx context.Context, db *sql.DB) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin metric rollup migration: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, statement := range []string{
|
||||
`CREATE TABLE IF NOT EXISTS storage_projection_state (id INTEGER PRIMARY KEY CHECK(id=1), metric_rollup_version INTEGER NOT NULL CHECK(metric_rollup_version BETWEEN 0 AND 1), base_index_version INTEGER NOT NULL DEFAULT 0 CHECK(base_index_version BETWEEN 0 AND 1))`,
|
||||
// Keep this insert compatible with preview databases whose state table
|
||||
// predates the independently versioned base-index migration. That
|
||||
// migration adds and initializes its own column transactionally.
|
||||
`INSERT OR IGNORE INTO storage_projection_state(id,metric_rollup_version) VALUES(1,0)`,
|
||||
`CREATE TABLE IF NOT EXISTS metric_rollups_5m (
|
||||
organization_id TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
environment_id TEXT NOT NULL,
|
||||
service_id TEXT NOT NULL,
|
||||
bucket_start TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
dimensions_digest TEXT NOT NULL,
|
||||
attributes_json TEXT NOT NULL,
|
||||
sample_count INTEGER NOT NULL CHECK(sample_count > 0),
|
||||
value_count INTEGER NOT NULL CHECK(value_count > 0),
|
||||
value_sum REAL NOT NULL,
|
||||
value_min REAL NOT NULL,
|
||||
value_max REAL NOT NULL,
|
||||
last_value REAL NOT NULL,
|
||||
last_timestamp TEXT NOT NULL,
|
||||
histogram_json TEXT NOT NULL,
|
||||
PRIMARY KEY(organization_id,project_id,environment_id,service_id,bucket_start,name,dimensions_digest)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS metric_rollup_segments (
|
||||
segment_digest TEXT PRIMARY KEY
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS metric_rollups_time ON metric_rollups_5m(organization_id,bucket_start)`,
|
||||
`CREATE INDEX IF NOT EXISTS metric_rollups_scope ON metric_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start)`,
|
||||
`CREATE INDEX IF NOT EXISTS metric_rollups_name ON metric_rollups_5m(organization_id,name,bucket_start)`,
|
||||
} {
|
||||
if _, err = tx.ExecContext(ctx, statement); err != nil {
|
||||
return fmt.Errorf("migrate metric rollups: %w", err)
|
||||
}
|
||||
}
|
||||
var version int
|
||||
if err = tx.QueryRowContext(ctx, `SELECT metric_rollup_version FROM storage_projection_state WHERE id=1`).Scan(&version); err != nil {
|
||||
return errors.New("read metric rollup migration state")
|
||||
}
|
||||
if version == 0 {
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM metric_rollups_5m`); err != nil {
|
||||
return errors.New("clear incomplete metric rollup migration")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM metric_rollup_segments`); err != nil {
|
||||
return errors.New("clear incomplete metric rollup segment ledger")
|
||||
}
|
||||
_, registry, _, registryErr := activeProjection(ctx, tx)
|
||||
if registryErr != nil {
|
||||
return registryErr
|
||||
}
|
||||
if err = backfillMetricRollups(ctx, tx, registry); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO metric_rollup_segments(segment_digest) SELECT DISTINCT segment_digest FROM observations WHERE signal=?`, model.SignalMetrics); err != nil {
|
||||
return errors.New("record backfilled metric rollup segments")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE storage_projection_state SET metric_rollup_version=? WHERE id=1`, metricRollupVersion); err != nil {
|
||||
return errors.New("complete metric rollup migration")
|
||||
}
|
||||
} else if version != metricRollupVersion {
|
||||
return errors.New("unsupported metric rollup projection version")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit metric rollup migration: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func backfillMetricRollups(ctx context.Context, tx *sql.Tx, registry query.Registry) error {
|
||||
rows, err := tx.QueryContext(ctx, `SELECT organization_id,project_id,environment_id,service_id,timestamp,name,value,attributes_json FROM observations WHERE signal=? ORDER BY timestamp,organization_id,project_id,environment_id,service_id,name,attributes_json`, model.SignalMetrics)
|
||||
if err != nil {
|
||||
return errors.New("read metrics for rollup migration")
|
||||
}
|
||||
defer rows.Close()
|
||||
groups := map[string]*metricRollup{}
|
||||
currentBucket := ""
|
||||
for rows.Next() {
|
||||
var organizationID, projectID, environmentID, serviceID, timestampText, name, attributesJSON string
|
||||
var value float64
|
||||
if err = rows.Scan(&organizationID, &projectID, &environmentID, &serviceID, ×tampText, &name, &value, &attributesJSON); err != nil {
|
||||
return errors.New("read metric rollup migration row")
|
||||
}
|
||||
timestamp, parseErr := time.Parse(time.RFC3339Nano, timestampText)
|
||||
if parseErr != nil || math.IsNaN(value) || math.IsInf(value, 0) || math.Abs(value) > maxMetricMagnitude {
|
||||
return errors.New("metric rollup migration row is invalid")
|
||||
}
|
||||
var attributes map[string]string
|
||||
if err = json.Unmarshal([]byte(attributesJSON), &attributes); err != nil {
|
||||
return errors.New("metric rollup migration dimensions are invalid")
|
||||
}
|
||||
retainedAttributes, retainErr := retainedMetricDimensions(attributes, registry)
|
||||
if retainErr != nil {
|
||||
return retainErr
|
||||
}
|
||||
retainedJSON, marshalErr := json.Marshal(retainedAttributes)
|
||||
if marshalErr != nil {
|
||||
return errors.New("encode metric rollup migration dimensions")
|
||||
}
|
||||
attributesJSON = string(retainedJSON)
|
||||
bucket := timestamp.UTC().Truncate(metricRollupWindow).Format(time.RFC3339Nano)
|
||||
if currentBucket != "" && bucket != currentBucket {
|
||||
if err = flushMetricRollups(ctx, tx, groups); err != nil {
|
||||
return err
|
||||
}
|
||||
groups = map[string]*metricRollup{}
|
||||
}
|
||||
currentBucket = bucket
|
||||
rollupKey, digest := metricRollupKey(projectID, environmentID, serviceID, bucket, name, attributesJSON)
|
||||
key := organizationID + "\x00" + rollupKey
|
||||
rollup := groups[key]
|
||||
if rollup == nil {
|
||||
rollup = newMetricRollup(projectID, environmentID, serviceID, bucket, name, attributesJSON, digest)
|
||||
groups[key] = rollup
|
||||
}
|
||||
addMetricValue(rollup, timestamp, value)
|
||||
if len(groups) >= maxMetricRollupGroupsBatch {
|
||||
if err = flushMetricRollups(ctx, tx, groups); err != nil {
|
||||
return err
|
||||
}
|
||||
groups = map[string]*metricRollup{}
|
||||
}
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return errors.New("read metric rollup migration rows")
|
||||
}
|
||||
return flushMetricRollups(ctx, tx, groups)
|
||||
}
|
||||
|
||||
func projectMetricRollups(ctx context.Context, tx *sql.Tx, scope model.Scope, batch model.Batch, segmentDigest string, registry query.Registry) error {
|
||||
if batch.Signal != model.SignalMetrics {
|
||||
return nil
|
||||
}
|
||||
ledger, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO metric_rollup_segments(segment_digest) VALUES(?)`, segmentDigest)
|
||||
if err != nil {
|
||||
return errors.New("record metric rollup segment")
|
||||
}
|
||||
inserted, err := ledger.RowsAffected()
|
||||
if err != nil {
|
||||
return errors.New("inspect metric rollup segment")
|
||||
}
|
||||
if inserted == 0 {
|
||||
return nil
|
||||
}
|
||||
groups := map[string]*metricRollup{}
|
||||
for _, observation := range batch.Records {
|
||||
retained, err := retainedMetricDimensions(observation.Attributes, registry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attributes, err := json.Marshal(retained)
|
||||
if err != nil {
|
||||
return errors.New("encode metric rollup dimensions")
|
||||
}
|
||||
bucket := observation.Timestamp.UTC().Truncate(metricRollupWindow).Format(time.RFC3339Nano)
|
||||
key, digest := metricRollupKey(scope.ProjectID, scope.EnvironmentID, scope.ServiceID, bucket, observation.Name, string(attributes))
|
||||
rollup := groups[key]
|
||||
if rollup == nil {
|
||||
rollup = newMetricRollup(scope.ProjectID, scope.EnvironmentID, scope.ServiceID, bucket, observation.Name, string(attributes), digest)
|
||||
groups[key] = rollup
|
||||
if len(groups) > maxMetricRollupGroupsBatch {
|
||||
return fmt.Errorf("metric batch exceeds %d projected rollup groups", maxMetricRollupGroupsBatch)
|
||||
}
|
||||
}
|
||||
addMetricValue(rollup, observation.Timestamp.UTC(), *observation.Value)
|
||||
}
|
||||
for _, rollup := range groups {
|
||||
if err := mergeMetricRollup(ctx, tx, scope.OrganizationID, rollup); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retainedMetricDimensions(attributes map[string]string, registry query.Registry) (map[string]string, error) {
|
||||
retained := map[string]string{}
|
||||
for field, value := range attributes {
|
||||
canonical := query.CanonicalField(field)
|
||||
descriptor, unknown := query.ResolveDescriptor(model.SignalMetrics, canonical, registry)
|
||||
if unknown || descriptor.Retention != schema.RetentionMetric || descriptor.Sensitivity == schema.SensitivitySensitive || descriptor.Cardinality == schema.CardinalityHigh {
|
||||
continue
|
||||
}
|
||||
if len(retained) >= model.MaxAttributes {
|
||||
return nil, errors.New("metric rollup dimension limit exceeded")
|
||||
}
|
||||
if _, exists := retained[canonical]; exists {
|
||||
return nil, errors.New("metric rollup dimensions contain a canonical alias collision")
|
||||
}
|
||||
retained[canonical] = value
|
||||
}
|
||||
return retained, nil
|
||||
}
|
||||
|
||||
func newMetricRollup(projectID, environmentID, serviceID, bucket, name, attributesJSON, digest string) *metricRollup {
|
||||
return &metricRollup{bucket: bucket, projectID: projectID, environmentID: environmentID, serviceID: serviceID, name: name, dimensionsDigest: digest, attributesJSON: attributesJSON, bins: map[int64]int64{}}
|
||||
}
|
||||
|
||||
func addMetricValue(rollup *metricRollup, timestamp time.Time, value float64) {
|
||||
rollup.sampleCount++
|
||||
rollup.valueCount++
|
||||
rollup.sum += value
|
||||
if rollup.valueCount == 1 || value < rollup.minimum {
|
||||
rollup.minimum = value
|
||||
}
|
||||
if rollup.valueCount == 1 || value > rollup.maximum {
|
||||
rollup.maximum = value
|
||||
}
|
||||
stamp := timestamp.UTC().Format(time.RFC3339Nano)
|
||||
if rollup.lastTimestamp == "" || stamp >= rollup.lastTimestamp {
|
||||
rollup.lastTimestamp, rollup.lastValue = stamp, value
|
||||
}
|
||||
rollup.bins[metricHistogramBucket(value)]++
|
||||
}
|
||||
|
||||
func metricRollupKey(projectID, environmentID, serviceID, bucket, name, attributesJSON string) (string, string) {
|
||||
dimensions := sha256.Sum256([]byte(name + "\x00" + attributesJSON))
|
||||
digest := hex.EncodeToString(dimensions[:])
|
||||
return projectID + "\x00" + environmentID + "\x00" + serviceID + "\x00" + bucket + "\x00" + name + "\x00" + digest, digest
|
||||
}
|
||||
|
||||
func flushMetricRollups(ctx context.Context, tx *sql.Tx, groups map[string]*metricRollup) error {
|
||||
keys := make([]string, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
// Backfill keys prefix the organization. Split once; identifiers cannot
|
||||
// contain NUL under the model validation contract.
|
||||
separator := -1
|
||||
for index := 0; index < len(key); index++ {
|
||||
if key[index] == 0 {
|
||||
separator = index
|
||||
break
|
||||
}
|
||||
}
|
||||
if separator < 1 {
|
||||
return errors.New("metric rollup migration key is invalid")
|
||||
}
|
||||
if err := mergeMetricRollup(ctx, tx, key[:separator], groups[key]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeMetricRollup(ctx context.Context, tx *sql.Tx, organizationID string, incoming *metricRollup) error {
|
||||
var stored metricRollup
|
||||
var histogram string
|
||||
err := tx.QueryRowContext(ctx, `SELECT attributes_json,sample_count,value_count,value_sum,value_min,value_max,last_value,last_timestamp,histogram_json FROM metric_rollups_5m WHERE organization_id=? AND project_id=? AND environment_id=? AND service_id=? AND bucket_start=? AND name=? AND dimensions_digest=?`, organizationID, incoming.projectID, incoming.environmentID, incoming.serviceID, incoming.bucket, incoming.name, incoming.dimensionsDigest).Scan(&stored.attributesJSON, &stored.sampleCount, &stored.valueCount, &stored.sum, &stored.minimum, &stored.maximum, &stored.lastValue, &stored.lastTimestamp, &histogram)
|
||||
if err == nil {
|
||||
if stored.attributesJSON != incoming.attributesJSON {
|
||||
return errors.New("metric rollup dimension digest collision")
|
||||
}
|
||||
stored.bins, err = decodeHistogram(histogram)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stored.projectID, stored.environmentID, stored.serviceID, stored.bucket, stored.name, stored.dimensionsDigest = incoming.projectID, incoming.environmentID, incoming.serviceID, incoming.bucket, incoming.name, incoming.dimensionsDigest
|
||||
if stored.sampleCount > math.MaxInt64-incoming.sampleCount || stored.valueCount > math.MaxInt64-incoming.valueCount {
|
||||
return errors.New("metric rollup count exceeds numeric range")
|
||||
}
|
||||
stored.sampleCount += incoming.sampleCount
|
||||
stored.valueCount += incoming.valueCount
|
||||
stored.sum += incoming.sum
|
||||
if math.IsNaN(stored.sum) || math.IsInf(stored.sum, 0) {
|
||||
return errors.New("metric rollup sum exceeds numeric range")
|
||||
}
|
||||
stored.minimum = math.Min(stored.minimum, incoming.minimum)
|
||||
stored.maximum = math.Max(stored.maximum, incoming.maximum)
|
||||
if incoming.lastTimestamp >= stored.lastTimestamp {
|
||||
stored.lastTimestamp, stored.lastValue = incoming.lastTimestamp, incoming.lastValue
|
||||
}
|
||||
for bucket, count := range incoming.bins {
|
||||
if stored.bins[bucket] > math.MaxInt64-count {
|
||||
return errors.New("metric histogram count exceeds numeric range")
|
||||
}
|
||||
stored.bins[bucket] += count
|
||||
}
|
||||
incoming = &stored
|
||||
} else if !errors.Is(err, sql.ErrNoRows) {
|
||||
return errors.New("read metric rollup")
|
||||
}
|
||||
histogram, err = encodeHistogram(incoming.bins)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO metric_rollups_5m(organization_id,project_id,environment_id,service_id,bucket_start,name,dimensions_digest,attributes_json,sample_count,value_count,value_sum,value_min,value_max,last_value,last_timestamp,histogram_json) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(organization_id,project_id,environment_id,service_id,bucket_start,name,dimensions_digest) DO UPDATE SET sample_count=excluded.sample_count,value_count=excluded.value_count,value_sum=excluded.value_sum,value_min=excluded.value_min,value_max=excluded.value_max,last_value=excluded.last_value,last_timestamp=excluded.last_timestamp,histogram_json=excluded.histogram_json WHERE metric_rollups_5m.attributes_json=excluded.attributes_json`, organizationID, incoming.projectID, incoming.environmentID, incoming.serviceID, incoming.bucket, incoming.name, incoming.dimensionsDigest, incoming.attributesJSON, incoming.sampleCount, incoming.valueCount, incoming.sum, incoming.minimum, incoming.maximum, incoming.lastValue, incoming.lastTimestamp, histogram)
|
||||
if err != nil {
|
||||
return errors.New("store metric rollup")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func encodeHistogram(bins map[int64]int64) (string, error) {
|
||||
if len(bins) < 1 || len(bins) > maxHistogramBins {
|
||||
return "", errors.New("metric histogram bin count is invalid")
|
||||
}
|
||||
keys := make([]int64, 0, len(bins))
|
||||
for bucket, count := range bins {
|
||||
if count < 1 {
|
||||
return "", errors.New("metric histogram count is invalid")
|
||||
}
|
||||
keys = append(keys, bucket)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
|
||||
encoded := make([]histogramBin, 0, len(keys))
|
||||
for _, bucket := range keys {
|
||||
encoded = append(encoded, histogramBin{Bucket: bucket, Count: bins[bucket]})
|
||||
}
|
||||
body, err := json.Marshal(encoded)
|
||||
if err != nil || len(body) > maxHistogramJSON {
|
||||
return "", errors.New("metric histogram encoding exceeds limit")
|
||||
}
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func decodeHistogram(encoded string) (map[int64]int64, error) {
|
||||
if len(encoded) < 2 || len(encoded) > maxHistogramJSON {
|
||||
return nil, errors.New("metric histogram encoding is invalid")
|
||||
}
|
||||
var values []histogramBin
|
||||
decoder := json.NewDecoder(strings.NewReader(encoded))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&values); err != nil || len(values) < 1 || len(values) > maxHistogramBins {
|
||||
return nil, errors.New("metric histogram encoding is invalid")
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return nil, errors.New("metric histogram encoding is invalid")
|
||||
}
|
||||
bins := make(map[int64]int64, len(values))
|
||||
var previous int64
|
||||
for index, value := range values {
|
||||
if value.Count < 1 || index > 0 && value.Bucket <= previous {
|
||||
return nil, errors.New("metric histogram encoding is invalid")
|
||||
}
|
||||
bins[value.Bucket] = value.Count
|
||||
previous = value.Bucket
|
||||
}
|
||||
return bins, nil
|
||||
}
|
||||
|
||||
func metricHistogramBucket(value float64) int64 {
|
||||
if value == 0 {
|
||||
return 0
|
||||
}
|
||||
bucket := int64(math.Round(math.Log1p(math.Abs(value))*metricHistogramScale)) + 1
|
||||
if value < 0 {
|
||||
return -bucket
|
||||
}
|
||||
return bucket
|
||||
}
|
||||
|
||||
func metricHistogramValue(bucket int64) float64 {
|
||||
if bucket == 0 {
|
||||
return 0
|
||||
}
|
||||
sign := 1.0
|
||||
if bucket < 0 {
|
||||
sign, bucket = -1, -bucket
|
||||
}
|
||||
return sign * math.Expm1(float64(bucket-1)/metricHistogramScale)
|
||||
}
|
||||
|
||||
func histogramPercentile(bins map[int64]int64, percentile float64) (float64, bool) {
|
||||
var total int64
|
||||
keys := make([]int64, 0, len(bins))
|
||||
for bucket, count := range bins {
|
||||
if count < 1 || total > math.MaxInt64-count {
|
||||
return 0, false
|
||||
}
|
||||
total += count
|
||||
keys = append(keys, bucket)
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, false
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
|
||||
target := int64(math.Ceil(percentile * float64(total)))
|
||||
var seen int64
|
||||
for _, bucket := range keys {
|
||||
seen += bins[bucket]
|
||||
if seen >= target {
|
||||
return metricHistogramValue(bucket), true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func histogramCanonical(value float64) string {
|
||||
return strconv.FormatFloat(value, 'g', -1, 64)
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
func metricRollupQueryEligible(ast query.AST, registry query.Registry) bool {
|
||||
if ast.Signal != model.SignalMetrics || ast.Summary == nil {
|
||||
return false
|
||||
}
|
||||
if ast.Bucket > 0 && (ast.Bucket < metricRollupWindow || ast.Bucket%metricRollupWindow != 0) {
|
||||
return false
|
||||
}
|
||||
for _, aggregate := range ast.Summary.Aggregates {
|
||||
if aggregate.Function != "count" && query.CanonicalField(aggregate.Field) != "value" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, filter := range ast.Filters {
|
||||
field := query.CanonicalField(filter.Field)
|
||||
switch field {
|
||||
case "value", "timestamp", "source.id", "stream.id", "severity", "body", "trace_id", "span_id", "correlation_id":
|
||||
return false
|
||||
}
|
||||
if !metricRollupDimensionAvailable(field, registry) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, field := range ast.Summary.GroupBy {
|
||||
canonical := query.CanonicalField(field)
|
||||
switch canonical {
|
||||
case "value", "timestamp", "source.id", "stream.id", "severity", "body", "trace_id", "span_id", "correlation_id":
|
||||
return false
|
||||
}
|
||||
if !metricRollupDimensionAvailable(canonical, registry) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func metricRollupDimensionAvailable(field string, registry query.Registry) bool {
|
||||
switch query.CanonicalField(field) {
|
||||
case "project.id", "environment.id", "service.id", "name":
|
||||
return true
|
||||
}
|
||||
descriptor, unknown := query.ResolveDescriptor(model.SignalMetrics, field, registry)
|
||||
return !unknown && descriptor.Retention == schema.RetentionMetric && descriptor.Sensitivity != schema.SensitivitySensitive && descriptor.Cardinality != schema.CardinalityHigh
|
||||
}
|
||||
|
||||
func (s *Store) estimateMetricRollupBytes(ctx context.Context, scope query.Scope, ast query.AST, now time.Time) (int64, error) {
|
||||
path := s.organizationProjectionPath(scope.OrganizationID)
|
||||
info, err := os.Lstat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return 0, errors.New("organization projection is unavailable")
|
||||
}
|
||||
dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String()
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return 0, errors.New("open metric rollup estimate")
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
statement := `SELECT COALESCE(SUM(192+LENGTH(name)+LENGTH(attributes_json)+LENGTH(histogram_json)),0) FROM metric_rollups_5m WHERE organization_id=?`
|
||||
arguments := []any{scope.OrganizationID}
|
||||
for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} {
|
||||
if selected.value != "" {
|
||||
statement += " AND " + selected.column + "=?"
|
||||
arguments = append(arguments, selected.value)
|
||||
}
|
||||
}
|
||||
if ast.Window > 0 {
|
||||
statement += " AND bucket_start>=?"
|
||||
arguments = append(arguments, now.UTC().Add(-ast.Window).Truncate(metricRollupWindow).Format(time.RFC3339Nano))
|
||||
}
|
||||
var estimated int64
|
||||
if err = db.QueryRowContext(ctx, statement, arguments...).Scan(&estimated); err != nil || estimated < 0 {
|
||||
return 0, errors.New("estimate metric rollup scan")
|
||||
}
|
||||
return estimated, nil
|
||||
}
|
||||
|
||||
type rollupAggregate struct {
|
||||
function string
|
||||
count int64
|
||||
sum, minimum, maximum float64
|
||||
bins map[int64]int64
|
||||
}
|
||||
|
||||
type rollupSummaryGroup struct {
|
||||
key string
|
||||
values []*string
|
||||
aggregates []rollupAggregate
|
||||
}
|
||||
|
||||
func (s *Store) queryMetricRollups(ctx context.Context, path string, ast query.AST, scope query.Scope, registry query.Registry, budget query.Budget, now time.Time, result query.Result) (query.Result, error) {
|
||||
runContext, cancel := context.WithTimeout(ctx, budget.MaxDuration)
|
||||
defer cancel()
|
||||
started := time.Now()
|
||||
dsn := (&url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro"}).String()
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return query.Result{}, errors.New("open metric rollup projection")
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
statement := `SELECT project_id,environment_id,service_id,bucket_start,name,attributes_json,sample_count,value_count,value_sum,value_min,value_max,last_value,last_timestamp,histogram_json FROM metric_rollups_5m WHERE organization_id=?`
|
||||
arguments := []any{scope.OrganizationID}
|
||||
for _, selected := range []struct{ column, value string }{{"project_id", scope.ProjectID}, {"environment_id", scope.EnvironmentID}, {"service_id", scope.ServiceID}} {
|
||||
if selected.value != "" {
|
||||
statement += " AND " + selected.column + "=?"
|
||||
arguments = append(arguments, selected.value)
|
||||
}
|
||||
}
|
||||
if ast.Window > 0 {
|
||||
statement += " AND bucket_start>=?"
|
||||
arguments = append(arguments, now.UTC().Add(-ast.Window).Truncate(metricRollupWindow).Format(time.RFC3339Nano))
|
||||
}
|
||||
statement += ` ORDER BY bucket_start DESC,project_id,environment_id,service_id,name,dimensions_digest`
|
||||
rows, err := db.QueryContext(runContext, statement, arguments...)
|
||||
if err != nil {
|
||||
return query.Result{}, queryExecutionError(runContext, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
groups := map[string]*rollupSummaryGroup{}
|
||||
var memoryBytes int64
|
||||
for rows.Next() {
|
||||
var projectID, environmentID, serviceID, bucketText, name, attributesJSON, lastTimestamp, histogram string
|
||||
var sampleCount, valueCount int64
|
||||
var sum, minimum, maximum, lastValue float64
|
||||
if err = rows.Scan(&projectID, &environmentID, &serviceID, &bucketText, &name, &attributesJSON, &sampleCount, &valueCount, &sum, &minimum, &maximum, &lastValue, &lastTimestamp, &histogram); err != nil {
|
||||
return query.Result{}, errors.New("read metric rollup projection")
|
||||
}
|
||||
if result.Stats.ScannedRows == math.MaxInt64 {
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
result.Stats.ScannedRows++
|
||||
readBytes := int64(192 + len(projectID) + len(environmentID) + len(serviceID) + len(bucketText) + len(name) + len(attributesJSON) + len(lastTimestamp) + len(histogram))
|
||||
if readBytes < 0 || readBytes > budget.MaxScannedBytes-result.Stats.ScannedBytes {
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
result.Stats.ScannedBytes += readBytes
|
||||
bucket, parseErr := time.Parse(time.RFC3339Nano, bucketText)
|
||||
if parseErr != nil || sampleCount < 1 || valueCount < 1 || valueCount > sampleCount || math.IsNaN(sum) || math.IsInf(sum, 0) || math.IsNaN(minimum) || math.IsInf(minimum, 0) || math.IsNaN(maximum) || math.IsInf(maximum, 0) || math.IsNaN(lastValue) || math.IsInf(lastValue, 0) {
|
||||
return query.Result{}, errors.New("metric rollup projection is invalid")
|
||||
}
|
||||
attributes := map[string]string{}
|
||||
decoder := json.NewDecoder(strings.NewReader(attributesJSON))
|
||||
if err = decoder.Decode(&attributes); err != nil || len(attributes) > model.MaxAttributes {
|
||||
return query.Result{}, errors.New("metric rollup dimensions are invalid")
|
||||
}
|
||||
var trailing any
|
||||
if err = decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return query.Result{}, errors.New("metric rollup dimensions are invalid")
|
||||
}
|
||||
record := projectedRecord{projectID: projectID, environmentID: environmentID, serviceID: serviceID, signal: model.SignalMetrics, timestamp: bucket.UTC(), name: name, value: &lastValue, attributes: attributes}
|
||||
matched, matchErr := matchesRecord(record, ast, registry)
|
||||
if matchErr != nil {
|
||||
return query.Result{}, matchErr
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
if result.Stats.MatchedRows == math.MaxInt64 {
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
result.Stats.MatchedRows++
|
||||
bins, decodeErr := decodeHistogram(histogram)
|
||||
if decodeErr != nil {
|
||||
return query.Result{}, decodeErr
|
||||
}
|
||||
var values []*string
|
||||
if ast.Bucket > 0 {
|
||||
window := bucket.UTC().Truncate(ast.Bucket).Format(time.RFC3339Nano)
|
||||
values = append(values, stringPointer(window))
|
||||
}
|
||||
for _, field := range ast.Summary.GroupBy {
|
||||
value, present := record.field(field)
|
||||
if !present {
|
||||
values = append(values, nil)
|
||||
continue
|
||||
}
|
||||
column := result.Columns[len(values)]
|
||||
canonical, valid := canonicalResultValue(value, column.Type)
|
||||
if !valid {
|
||||
values = append(values, nil)
|
||||
continue
|
||||
}
|
||||
values = append(values, stringPointer(canonical))
|
||||
}
|
||||
key := groupKey(values)
|
||||
group := groups[key]
|
||||
if group == nil {
|
||||
group = &rollupSummaryGroup{key: key, values: values, aggregates: make([]rollupAggregate, len(ast.Summary.Aggregates))}
|
||||
for index, aggregate := range ast.Summary.Aggregates {
|
||||
group.aggregates[index] = rollupAggregate{function: aggregate.Function, bins: map[int64]int64{}}
|
||||
}
|
||||
groups[key] = group
|
||||
addition := int64(len(key) + len(values)*16 + len(group.aggregates)*96)
|
||||
if addition < 0 || addition > budget.MaxMemoryBytes-memoryBytes {
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
memoryBytes += addition
|
||||
}
|
||||
for index, aggregate := range ast.Summary.Aggregates {
|
||||
state := &group.aggregates[index]
|
||||
if aggregate.Function == "count" {
|
||||
if sampleCount > math.MaxInt64-state.count {
|
||||
return query.Result{}, errors.New("metric rollup count exceeds numeric range")
|
||||
}
|
||||
state.count += sampleCount
|
||||
continue
|
||||
}
|
||||
if state.count == 0 {
|
||||
state.minimum, state.maximum = minimum, maximum
|
||||
} else {
|
||||
state.minimum = math.Min(state.minimum, minimum)
|
||||
state.maximum = math.Max(state.maximum, maximum)
|
||||
}
|
||||
if valueCount > math.MaxInt64-state.count {
|
||||
return query.Result{}, errors.New("metric rollup count exceeds numeric range")
|
||||
}
|
||||
state.count += valueCount
|
||||
state.sum += sum
|
||||
if math.IsNaN(state.sum) || math.IsInf(state.sum, 0) {
|
||||
return query.Result{}, errors.New("metric rollup sum exceeds numeric range")
|
||||
}
|
||||
if aggregate.Function == "p50" || aggregate.Function == "p95" || aggregate.Function == "p99" {
|
||||
result.Stats.Approximate = true
|
||||
for histogramBucket, count := range bins {
|
||||
if count > math.MaxInt64-state.bins[histogramBucket] {
|
||||
return query.Result{}, errors.New("metric histogram count exceeds numeric range")
|
||||
}
|
||||
state.bins[histogramBucket] += count
|
||||
}
|
||||
addition := int64(len(bins) * 24)
|
||||
if addition < 0 || addition > budget.MaxMemoryBytes-memoryBytes {
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
memoryBytes += addition
|
||||
}
|
||||
}
|
||||
if memoryBytes > budget.MaxMemoryBytes {
|
||||
return query.Result{}, query.ErrBudgetExceeded
|
||||
}
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return query.Result{}, queryExecutionError(runContext, err)
|
||||
}
|
||||
ordered := make([]*rollupSummaryGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
ordered = append(ordered, group)
|
||||
}
|
||||
sort.Slice(ordered, func(i, j int) bool { return ordered[i].key < ordered[j].key })
|
||||
for _, group := range ordered {
|
||||
row := query.Row{Values: append([]*string(nil), group.values...)}
|
||||
for _, aggregate := range group.aggregates {
|
||||
value, present := rollupAggregateValue(aggregate)
|
||||
if present {
|
||||
row.Values = append(row.Values, stringPointer(value))
|
||||
} else {
|
||||
row.Values = append(row.Values, nil)
|
||||
}
|
||||
}
|
||||
result.Rows = append(result.Rows, row)
|
||||
}
|
||||
if ast.Sort != nil {
|
||||
if err = sortRows(result.Rows, result.Columns, ast.Sort.Field, ast.Sort.Descending); err != nil {
|
||||
return query.Result{}, err
|
||||
}
|
||||
}
|
||||
if len(result.Rows) > ast.Limit {
|
||||
result.Rows = result.Rows[:ast.Limit]
|
||||
result.Stats.Truncated = true
|
||||
}
|
||||
result.Stats.DurationNS = time.Since(started).Nanoseconds()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func rollupAggregateValue(state rollupAggregate) (string, bool) {
|
||||
if state.function == "count" {
|
||||
return strconv.FormatInt(state.count, 10), true
|
||||
}
|
||||
if state.count == 0 {
|
||||
return "", false
|
||||
}
|
||||
var value float64
|
||||
switch state.function {
|
||||
case "min":
|
||||
value = state.minimum
|
||||
case "max":
|
||||
value = state.maximum
|
||||
case "sum":
|
||||
value = state.sum
|
||||
case "avg":
|
||||
value = state.sum / float64(state.count)
|
||||
case "p50", "p95", "p99":
|
||||
percentile := map[string]float64{"p50": .50, "p95": .95, "p99": .99}[state.function]
|
||||
var ok bool
|
||||
value, ok = histogramPercentile(state.bins, percentile)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatFloat(value, 'g', -1, 64), true
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
func TestMetricSummaryQueriesUseFiveMinuteRollups(t *testing.T) {
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
ctx := t.Context()
|
||||
scope := model.Scope{OrganizationID: "org-query-rollup", ProjectID: "project", EnvironmentID: "production", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-query-rollup", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 17, 4, 9, 0, 0, time.UTC)
|
||||
sequence := uint64(1)
|
||||
for _, sample := range []struct {
|
||||
at time.Time
|
||||
value float64
|
||||
}{{now.Add(-8 * time.Minute), 1}, {now.Add(-7 * time.Minute), 2}, {now.Add(-2 * time.Minute), 100}} {
|
||||
value := sample.value
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-query-rollup", StreamID: "metrics", Sequence: sequence, ObservedAt: now, Signal: model.SignalMetrics, Records: []model.Observation{{Timestamp: sample.at, Name: "request.duration", Value: &value, Attributes: map[string]string{"http.route": "/items", "private.dimension": "secret"}}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sequence++
|
||||
}
|
||||
projectAll(t, store)
|
||||
ast, err := query.Parse(`metrics | where route == "/items" | window 1h | summarize count(), avg(value), p95(value) by name, window(10m) | limit 20`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID, ProjectID: scope.ProjectID, EnvironmentID: scope.EnvironmentID, ServiceID: scope.ServiceID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Explain.ProjectedSources) != 1 || !strings.HasSuffix(result.Explain.ProjectedSources[0], "/rollup:5m") {
|
||||
t.Fatalf("sources=%v", result.Explain.ProjectedSources)
|
||||
}
|
||||
if !result.Stats.Approximate || result.Stats.ScannedRows != 2 || result.Stats.MatchedRows != 2 || len(result.Rows) != 1 {
|
||||
t.Fatalf("stats=%+v rows=%+v", result.Stats, result.Rows)
|
||||
}
|
||||
row := result.Rows[0]
|
||||
if len(row.Values) != 5 || row.Values[2] == nil || *row.Values[2] != "3" || row.Values[3] == nil || row.Values[4] == nil {
|
||||
t.Fatalf("row=%+v", row)
|
||||
}
|
||||
average, err := strconv.ParseFloat(*row.Values[3], 64)
|
||||
if err != nil || average < 34.3 || average > 34.4 {
|
||||
t.Fatalf("average=%g err=%v", average, err)
|
||||
}
|
||||
p95, err := strconv.ParseFloat(*row.Values[4], 64)
|
||||
if err != nil || p95 < 98 || p95 > 102 {
|
||||
t.Fatalf("p95=%g err=%v", p95, err)
|
||||
}
|
||||
|
||||
rawAST, err := query.Parse(`metrics | where value >= 2 | window 1h | summarize count() | limit 20`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := store.Query(ctx, rawAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.HasSuffix(raw.Explain.ProjectedSources[0], "/rollup:5m") || len(raw.Rows) != 1 || raw.Rows[0].Values[0] == nil || *raw.Rows[0].Values[0] != "2" {
|
||||
t.Fatalf("raw=%+v", raw)
|
||||
}
|
||||
unknownAST, err := query.Parse(`metrics | where private.dimension == "secret" | window 1h | summarize count() | limit 20`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unknown, err := store.Query(ctx, unknownAST, query.Scope{OrganizationID: scope.OrganizationID, Sensitive: true}, query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.HasSuffix(unknown.Explain.ProjectedSources[0], "/rollup:5m") || len(unknown.Rows) != 1 || unknown.Rows[0].Values[0] == nil || *unknown.Rows[0].Values[0] != "3" {
|
||||
t.Fatalf("unknown=%+v", unknown)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
type SourceAlertTransitionAck struct {
|
||||
SourceID string `json:"source_id"`
|
||||
RuleID string `json:"rule_id"`
|
||||
RuleRevision int `json:"rule_revision"`
|
||||
AgentEpoch string `json:"agent_epoch"`
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Digest string `json:"digest"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
}
|
||||
|
||||
func migrateControlSourceAlertTransitions(db *sql.DB) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return errors.New("begin source alert transition migration")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, statement := range []string{
|
||||
`CREATE TABLE source_alert_transitions (
|
||||
organization_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
rule_id TEXT NOT NULL,
|
||||
rule_revision INTEGER NOT NULL CHECK(rule_revision BETWEEN 1 AND 1000000),
|
||||
agent_epoch TEXT NOT NULL CHECK(length(agent_epoch)=32),
|
||||
transition_sequence INTEGER NOT NULL CHECK(transition_sequence >= 1),
|
||||
stream_id TEXT NOT NULL,
|
||||
batch_sequence INTEGER NOT NULL CHECK(batch_sequence >= 1),
|
||||
segment_digest TEXT NOT NULL,
|
||||
window_start TEXT NOT NULL,
|
||||
window_end TEXT NOT NULL,
|
||||
state TEXT NOT NULL CHECK(state IN ('matched','clear','error')),
|
||||
observed_at TEXT NOT NULL,
|
||||
received_at TEXT NOT NULL,
|
||||
transition_digest TEXT NOT NULL CHECK(length(transition_digest)=64),
|
||||
PRIMARY KEY(source_id,rule_id,rule_revision,agent_epoch,transition_sequence),
|
||||
FOREIGN KEY(source_id) REFERENCES sources(id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY(organization_id,rule_id) REFERENCES alert_rules(organization_id,id) ON DELETE RESTRICT,
|
||||
FOREIGN KEY(segment_digest) REFERENCES segments(digest) ON DELETE RESTRICT
|
||||
)`,
|
||||
`CREATE INDEX source_alert_transitions_by_rule ON source_alert_transitions(organization_id,rule_id,observed_at,source_id,agent_epoch,transition_sequence)`,
|
||||
`UPDATE schema_version SET version=9 WHERE version=8`,
|
||||
} {
|
||||
if _, err = tx.Exec(statement); err != nil {
|
||||
return fmt.Errorf("migrate source alert transitions: %w", err)
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return errors.New("commit source alert transition migration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) RecordSourceAlertTransition(ctx context.Context, token string, transition model.AlertTransition, now time.Time) (SourceAlertTransitionAck, error) {
|
||||
if err := transition.Validate(now); err != nil {
|
||||
return SourceAlertTransitionAck{}, err
|
||||
}
|
||||
digest, err := transition.Digest()
|
||||
if err != nil {
|
||||
return SourceAlertTransitionAck{}, err
|
||||
}
|
||||
source, err := s.Authenticate(ctx, token)
|
||||
if err != nil {
|
||||
return SourceAlertTransitionAck{}, err
|
||||
}
|
||||
ack := SourceAlertTransitionAck{SourceID: source.ID, RuleID: transition.RuleID, RuleRevision: transition.RuleRevision, AgentEpoch: transition.AgentEpoch, Sequence: transition.Sequence, Digest: digest}
|
||||
lock := s.namedLock("source-alert:" + source.ID + ":" + transition.RuleID + ":" + transition.AgentEpoch)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
var existing string
|
||||
err = s.control.QueryRowContext(ctx, `SELECT transition_digest FROM source_alert_transitions WHERE source_id=? AND rule_id=? AND rule_revision=? AND agent_epoch=? AND transition_sequence=?`, source.ID, transition.RuleID, transition.RuleRevision, transition.AgentEpoch, transition.Sequence).Scan(&existing)
|
||||
if err == nil {
|
||||
if existing != digest {
|
||||
return SourceAlertTransitionAck{}, errors.New("alert transition sequence reused with different content")
|
||||
}
|
||||
ack.Duplicate = true
|
||||
return ack, nil
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return SourceAlertTransitionAck{}, errors.New("read source alert transition")
|
||||
}
|
||||
|
||||
rule, err := s.AlertRule(ctx, source.Scope.OrganizationID, transition.RuleID)
|
||||
if err != nil || !rule.Enabled || rule.Revision != transition.RuleRevision {
|
||||
return SourceAlertTransitionAck{}, errors.New("source alert rule is unavailable")
|
||||
}
|
||||
saved, err := s.SavedQuery(ctx, source.Scope.OrganizationID, rule.SavedQueryID)
|
||||
if err != nil || saved.AST.Signal != model.SignalLogs || saved.Scope.ProjectID != source.Scope.ProjectID || saved.Scope.EnvironmentID != source.Scope.EnvironmentID || saved.Scope.ServiceID != source.Scope.ServiceID {
|
||||
return SourceAlertTransitionAck{}, errors.New("source alert rule is not scoped to this source")
|
||||
}
|
||||
|
||||
var segmentOrganization, segmentSignal, firstText, lastText string
|
||||
err = s.control.QueryRowContext(ctx, `SELECT organization_id,signal,first_observed_at,last_observed_at FROM segments WHERE digest=? AND source_id=? AND stream_id=? AND sequence=? AND retiring_at IS NULL`, transition.SegmentDigest, source.ID, transition.StreamID, transition.BatchSequence).Scan(&segmentOrganization, &segmentSignal, &firstText, &lastText)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return SourceAlertTransitionAck{}, errors.New("source alert evidence is unavailable")
|
||||
}
|
||||
if err != nil {
|
||||
return SourceAlertTransitionAck{}, errors.New("read source alert evidence")
|
||||
}
|
||||
first, firstErr := time.Parse(time.RFC3339Nano, firstText)
|
||||
last, lastErr := time.Parse(time.RFC3339Nano, lastText)
|
||||
if firstErr != nil || lastErr != nil || segmentOrganization != source.Scope.OrganizationID || segmentSignal != string(model.SignalLogs) || transition.WindowStart.After(first) || transition.WindowEnd.Before(last) {
|
||||
return SourceAlertTransitionAck{}, errors.New("source alert evidence does not match transition")
|
||||
}
|
||||
|
||||
var previous uint64
|
||||
if err = s.control.QueryRowContext(ctx, `SELECT COALESCE(MAX(transition_sequence),0) FROM source_alert_transitions WHERE source_id=? AND rule_id=? AND rule_revision=? AND agent_epoch=?`, source.ID, transition.RuleID, transition.RuleRevision, transition.AgentEpoch).Scan(&previous); err != nil {
|
||||
return SourceAlertTransitionAck{}, errors.New("read source alert transition watermark")
|
||||
}
|
||||
if previous != 0 && transition.Sequence != previous+1 {
|
||||
return SourceAlertTransitionAck{}, errors.New("source alert transition sequence gap")
|
||||
}
|
||||
_, err = s.control.ExecContext(ctx, `INSERT INTO source_alert_transitions(organization_id,source_id,rule_id,rule_revision,agent_epoch,transition_sequence,stream_id,batch_sequence,segment_digest,window_start,window_end,state,observed_at,received_at,transition_digest) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, source.Scope.OrganizationID, source.ID, transition.RuleID, transition.RuleRevision, transition.AgentEpoch, transition.Sequence, transition.StreamID, transition.BatchSequence, transition.SegmentDigest, transition.WindowStart.UTC().Format(time.RFC3339Nano), transition.WindowEnd.UTC().Format(time.RFC3339Nano), transition.State, transition.ObservedAt.UTC().Format(time.RFC3339Nano), now.UTC().Format(time.RFC3339Nano), digest)
|
||||
if err != nil {
|
||||
return SourceAlertTransitionAck{}, errors.New("record source alert transition")
|
||||
}
|
||||
return ack, nil
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/query"
|
||||
)
|
||||
|
||||
func TestSourceAlertTransitionBindsAuthenticatedBatchEvidence(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 18, 23, 0, 0, 0, time.UTC)
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved, err := store.SaveQuery(ctx, SavedQueryInput{
|
||||
OrganizationID: scope.OrganizationID,
|
||||
ActorUserID: "operator-a",
|
||||
MaxRows: 100,
|
||||
Name: "Source failures",
|
||||
Description: "Exact source-scoped failures for differential evaluation.",
|
||||
Query: "logs | where status >= 500 | window 1h | limit 50",
|
||||
Scope: ResourceScope{ProjectID: scope.ProjectID, EnvironmentID: scope.EnvironmentID, ServiceID: scope.ServiceID},
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rule, err := store.SaveAlertRule(ctx, AlertRuleInput{
|
||||
OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", SavedQueryID: saved.ID,
|
||||
Name: "Source failures", Description: "Source and server comparison rule.", Severity: "warning",
|
||||
MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observed := now.Add(time.Second)
|
||||
batch := model.Batch{
|
||||
Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1,
|
||||
ObservedAt: observed, Signal: model.SignalLogs,
|
||||
Records: []model.Observation{{Timestamp: observed, Name: "http.request", Attributes: map[string]string{"http.status_code": "503", "http.route": "/failed"}}},
|
||||
}
|
||||
ingested, err := store.Ingest(ctx, token, batch, observed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
budget := query.Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 16 << 20, MaxMemoryBytes: 8 << 20}
|
||||
central, err := store.EvaluateDueAlertRules(ctx, budget, now.Add(15*time.Second))
|
||||
if err != nil || len(central) != 1 || !central[0].Matched {
|
||||
t.Fatalf("central=%+v err=%v", central, err)
|
||||
}
|
||||
|
||||
transition := model.AlertTransition{
|
||||
Version: model.AlertTransitionVersion, RuleID: rule.ID, RuleRevision: rule.Revision,
|
||||
AgentEpoch: strings.Repeat("a", 32), Sequence: 7, StreamID: batch.StreamID,
|
||||
BatchSequence: batch.Sequence, SegmentDigest: ingested.Digest,
|
||||
WindowStart: observed, WindowEnd: observed, State: "matched", ObservedAt: now.Add(16 * time.Second),
|
||||
}
|
||||
ack, err := store.RecordSourceAlertTransition(ctx, token, transition, now.Add(16*time.Second))
|
||||
if err != nil || ack.SourceID != batch.SourceID || ack.RuleID != rule.ID || ack.Sequence != 7 || ack.Digest == "" || ack.Duplicate {
|
||||
t.Fatalf("ack=%+v err=%v", ack, err)
|
||||
}
|
||||
replay, err := store.RecordSourceAlertTransition(ctx, token, transition, now.Add(17*time.Second))
|
||||
if err != nil || !replay.Duplicate || replay.Digest != ack.Digest {
|
||||
t.Fatalf("replay=%+v err=%v", replay, err)
|
||||
}
|
||||
|
||||
conflict := transition
|
||||
conflict.State = "clear"
|
||||
if _, err = store.RecordSourceAlertTransition(ctx, token, conflict, now.Add(17*time.Second)); err == nil || !strings.Contains(err.Error(), "reused with different content") {
|
||||
t.Fatalf("conflicting replay err=%v", err)
|
||||
}
|
||||
gap := transition
|
||||
gap.Sequence = 9
|
||||
if _, err = store.RecordSourceAlertTransition(ctx, token, gap, now.Add(17*time.Second)); err == nil || !strings.Contains(err.Error(), "sequence gap") {
|
||||
t.Fatalf("sequence gap err=%v", err)
|
||||
}
|
||||
wrongEvidence := transition
|
||||
wrongEvidence.Sequence = 8
|
||||
wrongEvidence.SegmentDigest = strings.Repeat("b", 64)
|
||||
if _, err = store.RecordSourceAlertTransition(ctx, token, wrongEvidence, now.Add(17*time.Second)); err == nil || !strings.Contains(err.Error(), "evidence is unavailable") {
|
||||
t.Fatalf("wrong evidence err=%v", err)
|
||||
}
|
||||
|
||||
otherToken, err := store.CreateSource(ctx, "source-b", model.Scope{OrganizationID: scope.OrganizationID, ProjectID: "project-b", EnvironmentID: "production", ServiceID: "service-b"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := transition
|
||||
other.Sequence = 8
|
||||
if _, err = store.RecordSourceAlertTransition(ctx, otherToken, other, now.Add(17*time.Second)); err == nil || !strings.Contains(err.Error(), "not scoped to this source") {
|
||||
t.Fatalf("cross-source transition err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceAlertTransitionRequiresEvidenceCoveringItsWindow(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 18, 23, 30, 0, 0, time.UTC)
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved, err := store.SaveQuery(ctx, SavedQueryInput{OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", MaxRows: 100, Name: "Logs", Query: "logs | limit 10", Scope: ResourceScope{ProjectID: scope.ProjectID, EnvironmentID: scope.EnvironmentID, ServiceID: scope.ServiceID}}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rule, err := store.SaveAlertRule(ctx, AlertRuleInput{OrganizationID: scope.OrganizationID, ActorUserID: "operator-a", SavedQueryID: saved.ID, Name: "Logs", Severity: "warning", MinimumMatches: 1, RequiredConsecutive: 1, EvaluationInterval: 15 * time.Second, Enabled: true}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, last := now.Add(-time.Minute), now
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "requests", Sequence: 1, ObservedAt: last, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: first, Name: "first"}, {Timestamp: last, Name: "last"}}}
|
||||
ingested, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transition := model.AlertTransition{Version: model.AlertTransitionVersion, RuleID: rule.ID, RuleRevision: rule.Revision, AgentEpoch: strings.Repeat("c", 32), Sequence: 1, StreamID: batch.StreamID, BatchSequence: batch.Sequence, SegmentDigest: ingested.Digest, WindowStart: first.Add(time.Second), WindowEnd: last, State: "matched", ObservedAt: now}
|
||||
if _, err = store.RecordSourceAlertTransition(ctx, token, transition, now); err == nil || !strings.Contains(err.Error(), "does not match transition") {
|
||||
t.Fatalf("partial evidence window err=%v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,1180 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/segment"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const controlSchema = 11
|
||||
|
||||
const recoveryPageSize = 128
|
||||
|
||||
type Store struct {
|
||||
root string
|
||||
control *sql.DB
|
||||
segments *segment.Store
|
||||
locks sync.Map
|
||||
projectionMu sync.Mutex
|
||||
projectorMu sync.Mutex
|
||||
projections map[string]projectionHandle
|
||||
projectionWake chan struct{}
|
||||
}
|
||||
|
||||
type projectionHandle struct {
|
||||
db *sql.DB
|
||||
device uint64
|
||||
inode uint64
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
ID string
|
||||
Scope model.Scope
|
||||
Active bool
|
||||
}
|
||||
|
||||
type Ack struct {
|
||||
SourceID string `json:"source_id"`
|
||||
StreamID string `json:"stream_id"`
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Digest string `json:"digest"`
|
||||
BatchDigest string `json:"batch_digest"`
|
||||
Duplicate bool `json:"duplicate"`
|
||||
}
|
||||
|
||||
type Enrollment struct {
|
||||
SourceID, CreatedByUserID string
|
||||
Scope model.Scope
|
||||
CreatedAt, ExpiresAt time.Time
|
||||
}
|
||||
|
||||
func Open(root string) (*Store, error) {
|
||||
if !filepath.IsAbs(root) || filepath.Clean(root) != root {
|
||||
return nil, errors.New("data root must be an absolute clean path")
|
||||
}
|
||||
if err := os.MkdirAll(root, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("create data root: %w", err)
|
||||
}
|
||||
info, err := os.Lstat(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect data root: %w", err)
|
||||
}
|
||||
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return nil, errors.New("data root must be a non-symlink directory")
|
||||
}
|
||||
if info.Mode().Perm()&0o077 != 0 {
|
||||
return nil, errors.New("data root must not grant group or other permissions")
|
||||
}
|
||||
controlPath := filepath.Join(root, "control.sqlite")
|
||||
if err := validateSQLiteFileSet(controlPath); err != nil {
|
||||
return nil, fmt.Errorf("inspect control database: %w", err)
|
||||
}
|
||||
db, err := sql.Open("sqlite", controlPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open control database: %w", err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if err := migrateControl(db); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chmod(controlPath, 0o600); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("set control database mode: %w", err)
|
||||
}
|
||||
segments, err := segment.New(root)
|
||||
if err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
store := &Store{
|
||||
root: root,
|
||||
control: db,
|
||||
segments: segments,
|
||||
projections: make(map[string]projectionHandle),
|
||||
projectionWake: make(chan struct{}, 1),
|
||||
}
|
||||
if err = store.backfillSegmentRetentionMetadata(context.Background()); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() error {
|
||||
s.projectionMu.Lock()
|
||||
handles := make([]projectionHandle, 0, len(s.projections))
|
||||
for organizationID, handle := range s.projections {
|
||||
handles = append(handles, handle)
|
||||
delete(s.projections, organizationID)
|
||||
}
|
||||
s.projectionMu.Unlock()
|
||||
errs := make([]error, 0, len(handles)+1)
|
||||
for _, handle := range handles {
|
||||
if err := handle.db.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if err := s.control.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// EstimateOrganizationBytes returns a conservative whole-projection scan
|
||||
// estimate. Query execution will refine this with index and time-window
|
||||
// statistics; planning never accepts a client-supplied cost estimate.
|
||||
func (s *Store) EstimateOrganizationBytes(organizationID string) (int64, error) {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil {
|
||||
return 0, errors.New("invalid organization identifier")
|
||||
}
|
||||
path := filepath.Join(s.root, "organizations", organizationID, "projection.sqlite")
|
||||
var total int64
|
||||
for _, candidate := range []string{path, path + "-wal"} {
|
||||
info, err := os.Lstat(candidate)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("inspect organization projection: %w", err)
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return 0, errors.New("organization projection is not a regular file")
|
||||
}
|
||||
if info.Size() > math.MaxInt64-total {
|
||||
return 0, errors.New("organization projection size overflow")
|
||||
}
|
||||
total += info.Size()
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func migrateControl(db *sql.DB) error {
|
||||
statements := []string{
|
||||
`PRAGMA journal_mode=WAL`,
|
||||
`PRAGMA synchronous=FULL`,
|
||||
`PRAGMA busy_timeout=5000`,
|
||||
`PRAGMA foreign_keys=ON`,
|
||||
`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)`,
|
||||
`INSERT INTO schema_version(version) SELECT 1 WHERE NOT EXISTS (SELECT 1 FROM schema_version)`,
|
||||
`CREATE TABLE IF NOT EXISTS sources (
|
||||
id TEXT PRIMARY KEY,
|
||||
organization_id TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
environment_id TEXT NOT NULL,
|
||||
service_id TEXT NOT NULL,
|
||||
credential_digest BLOB NOT NULL UNIQUE,
|
||||
active INTEGER NOT NULL CHECK(active IN (0,1)),
|
||||
created_at TEXT NOT NULL,
|
||||
rotated_at TEXT
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS streams (
|
||||
source_id TEXT NOT NULL REFERENCES sources(id),
|
||||
stream_id TEXT NOT NULL,
|
||||
last_sequence INTEGER NOT NULL,
|
||||
last_digest TEXT NOT NULL,
|
||||
PRIMARY KEY(source_id, stream_id)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS segments (
|
||||
digest TEXT PRIMARY KEY,
|
||||
organization_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
stream_id TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
compressed_bytes INTEGER NOT NULL,
|
||||
uncompressed_bytes INTEGER NOT NULL,
|
||||
committed_at TEXT NOT NULL,
|
||||
projected_at TEXT,
|
||||
UNIQUE(source_id, stream_id, sequence)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS source_enrollments (
|
||||
credential_digest BLOB PRIMARY KEY,
|
||||
source_id TEXT NOT NULL UNIQUE,
|
||||
organization_id TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
environment_id TEXT NOT NULL,
|
||||
service_id TEXT NOT NULL,
|
||||
created_by_user_id TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
used_at TEXT
|
||||
)`,
|
||||
`UPDATE schema_version SET version=2 WHERE version=1`,
|
||||
`CREATE TABLE IF NOT EXISTS descriptor_proposals (
|
||||
organization_id TEXT NOT NULL,
|
||||
signal TEXT NOT NULL,
|
||||
field TEXT NOT NULL,
|
||||
descriptor_json TEXT NOT NULL,
|
||||
observed_values INTEGER NOT NULL CHECK(observed_values > 0),
|
||||
estimated_bytes INTEGER NOT NULL CHECK(estimated_bytes >= 0),
|
||||
example_queries_json TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK(status IN ('pending','activated','rejected')),
|
||||
first_seen_at TEXT NOT NULL,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
PRIMARY KEY(organization_id,signal,field)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS descriptor_proposal_segments (
|
||||
segment_digest TEXT NOT NULL,
|
||||
organization_id TEXT NOT NULL,
|
||||
signal TEXT NOT NULL,
|
||||
field TEXT NOT NULL,
|
||||
observed_values INTEGER NOT NULL CHECK(observed_values > 0),
|
||||
estimated_bytes INTEGER NOT NULL CHECK(estimated_bytes >= 0),
|
||||
PRIMARY KEY(segment_digest,organization_id,field)
|
||||
)`,
|
||||
`UPDATE schema_version SET version=3 WHERE version=2`,
|
||||
`CREATE TABLE IF NOT EXISTS saved_queries (
|
||||
organization_id TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
version INTEGER NOT NULL CHECK(version=1),
|
||||
revision INTEGER NOT NULL CHECK(revision >= 1),
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
query_text TEXT NOT NULL,
|
||||
ast_json TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
environment_id TEXT NOT NULL,
|
||||
service_id TEXT NOT NULL,
|
||||
created_by TEXT NOT NULL,
|
||||
updated_by TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY(organization_id,id),
|
||||
UNIQUE(organization_id,name)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS dashboards (
|
||||
organization_id TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
version INTEGER NOT NULL CHECK(version=1),
|
||||
revision INTEGER NOT NULL CHECK(revision >= 1),
|
||||
slug TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
created_by TEXT NOT NULL,
|
||||
updated_by TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY(organization_id,id),
|
||||
UNIQUE(organization_id,slug)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS dashboard_panels (
|
||||
organization_id TEXT NOT NULL,
|
||||
dashboard_id TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
position INTEGER NOT NULL CHECK(position >= 0 AND position < 64),
|
||||
title TEXT NOT NULL,
|
||||
visualization TEXT NOT NULL CHECK(visualization IN ('table','stat','timeseries')),
|
||||
saved_query_id TEXT NOT NULL,
|
||||
PRIMARY KEY(organization_id,dashboard_id,id),
|
||||
UNIQUE(organization_id,dashboard_id,position),
|
||||
FOREIGN KEY(organization_id,dashboard_id) REFERENCES dashboards(organization_id,id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(organization_id,saved_query_id) REFERENCES saved_queries(organization_id,id) ON DELETE RESTRICT
|
||||
)`,
|
||||
`UPDATE schema_version SET version=4 WHERE version=3`,
|
||||
`CREATE TABLE IF NOT EXISTS alert_rules (
|
||||
organization_id TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
version INTEGER NOT NULL CHECK(version=1),
|
||||
revision INTEGER NOT NULL CHECK(revision >= 1),
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
saved_query_id TEXT NOT NULL,
|
||||
severity TEXT NOT NULL CHECK(severity IN ('information','warning','critical')),
|
||||
minimum_matches INTEGER NOT NULL CHECK(minimum_matches BETWEEN 1 AND 100000),
|
||||
required_consecutive INTEGER NOT NULL CHECK(required_consecutive BETWEEN 1 AND 10),
|
||||
evaluation_interval_seconds INTEGER NOT NULL CHECK(evaluation_interval_seconds BETWEEN 15 AND 86400),
|
||||
enabled INTEGER NOT NULL CHECK(enabled IN (0,1)),
|
||||
last_evaluated_at TEXT,
|
||||
next_evaluation_at TEXT NOT NULL,
|
||||
last_result INTEGER,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
created_by TEXT NOT NULL,
|
||||
updated_by TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY(organization_id,id),
|
||||
UNIQUE(organization_id,name),
|
||||
FOREIGN KEY(organization_id,saved_query_id) REFERENCES saved_queries(organization_id,id) ON DELETE RESTRICT
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS alert_rules_due ON alert_rules(enabled,next_evaluation_at,organization_id,id)`,
|
||||
`CREATE TABLE IF NOT EXISTS incidents (
|
||||
organization_id TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
version INTEGER NOT NULL CHECK(version=1),
|
||||
rule_id TEXT NOT NULL,
|
||||
state TEXT NOT NULL CHECK(state IN ('pending','firing','acknowledged','silenced','resolved')),
|
||||
severity TEXT NOT NULL CHECK(severity IN ('information','warning','critical')),
|
||||
title TEXT NOT NULL,
|
||||
consecutive_matches INTEGER NOT NULL CHECK(consecutive_matches >= 0),
|
||||
started_at TEXT NOT NULL,
|
||||
last_observed_at TEXT NOT NULL,
|
||||
acknowledged_by TEXT,
|
||||
acknowledged_at TEXT,
|
||||
silenced_by TEXT,
|
||||
silenced_until TEXT,
|
||||
resolved_at TEXT,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY(organization_id,id),
|
||||
FOREIGN KEY(organization_id,rule_id) REFERENCES alert_rules(organization_id,id) ON DELETE RESTRICT
|
||||
)`,
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS one_open_incident_per_rule ON incidents(organization_id,rule_id) WHERE state!='resolved'`,
|
||||
`CREATE INDEX IF NOT EXISTS incidents_by_state ON incidents(organization_id,state,updated_at DESC,id)`,
|
||||
`CREATE TABLE IF NOT EXISTS incident_events (
|
||||
organization_id TEXT NOT NULL,
|
||||
incident_id TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL CHECK(sequence >= 1),
|
||||
event TEXT NOT NULL CHECK(event IN ('opened','promoted','acknowledged','silenced','unsilenced','resolved')),
|
||||
actor TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
PRIMARY KEY(organization_id,incident_id,sequence),
|
||||
FOREIGN KEY(organization_id,incident_id) REFERENCES incidents(organization_id,id) ON DELETE CASCADE
|
||||
)`,
|
||||
`UPDATE schema_version SET version=5 WHERE version=4`,
|
||||
`CREATE TABLE IF NOT EXISTS push_endpoints (
|
||||
id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
endpoint TEXT NOT NULL,
|
||||
endpoint_digest BLOB NOT NULL UNIQUE,
|
||||
p256dh BLOB NOT NULL,
|
||||
auth_secret BLOB NOT NULL,
|
||||
active INTEGER NOT NULL CHECK(active IN (0,1)),
|
||||
failure_count INTEGER NOT NULL CHECK(failure_count >= 0),
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
last_sent_at TEXT,
|
||||
PRIMARY KEY(id)
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS push_subscriptions (
|
||||
organization_id TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
endpoint_id TEXT NOT NULL REFERENCES push_endpoints(id) ON DELETE CASCADE,
|
||||
created_at TEXT NOT NULL,
|
||||
PRIMARY KEY(organization_id,id),
|
||||
UNIQUE(organization_id,user_id,endpoint_id)
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS push_subscriptions_by_organization ON push_subscriptions(organization_id,user_id,id)`,
|
||||
`CREATE INDEX IF NOT EXISTS push_subscriptions_by_endpoint ON push_subscriptions(endpoint_id,organization_id,id)`,
|
||||
`UPDATE schema_version SET version=6 WHERE version=5`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := db.Exec(statement); err != nil {
|
||||
return fmt.Errorf("migrate control database: %w", err)
|
||||
}
|
||||
}
|
||||
var version int
|
||||
if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil {
|
||||
return fmt.Errorf("read control schema: %w", err)
|
||||
}
|
||||
if version == 6 {
|
||||
if err := migrateControlRetention(db); err != nil {
|
||||
return err
|
||||
}
|
||||
version = 7
|
||||
}
|
||||
if version == 7 {
|
||||
if err := migrateControlForensicRetention(db); err != nil {
|
||||
return err
|
||||
}
|
||||
version = 8
|
||||
}
|
||||
if version == 8 {
|
||||
if err := migrateControlSourceAlertTransitions(db); err != nil {
|
||||
return err
|
||||
}
|
||||
version = 9
|
||||
}
|
||||
if version == 9 {
|
||||
if err := migrateControlBatchMetadata(db); err != nil {
|
||||
return err
|
||||
}
|
||||
version = 10
|
||||
}
|
||||
if version == 10 {
|
||||
if err := migrateControlBatchEnvelopes(db); err != nil {
|
||||
return err
|
||||
}
|
||||
version = 11
|
||||
}
|
||||
if version != controlSchema {
|
||||
return fmt.Errorf("unsupported control schema %d", version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateSource(ctx context.Context, id string, scope model.Scope) (string, error) {
|
||||
if err := scope.Validate(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := model.ValidateSourceID(id); err != nil {
|
||||
return "", err
|
||||
}
|
||||
token, err := sourceCredential(id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest := sha256.Sum256([]byte(token))
|
||||
_, err = s.control.ExecContext(ctx, `INSERT INTO sources(id, organization_id, project_id, environment_id, service_id, credential_digest, active, created_at) VALUES(?,?,?,?,?,?,1,?)`, id, scope.OrganizationID, scope.ProjectID, scope.EnvironmentID, scope.ServiceID, digest[:], time.Now().UTC().Format(time.RFC3339Nano))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create source: %w", err)
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateEnrollment(ctx context.Context, id string, scope model.Scope, createdBy string, lifetime time.Duration, now time.Time) (string, Enrollment, error) {
|
||||
if err := model.ValidateSourceID(id); err != nil {
|
||||
return "", Enrollment{}, err
|
||||
}
|
||||
if err := scope.Validate(); err != nil {
|
||||
return "", Enrollment{}, err
|
||||
}
|
||||
if err := model.ValidateSourceID(createdBy); err != nil || lifetime < 5*time.Minute || lifetime > 24*time.Hour || now.IsZero() {
|
||||
return "", Enrollment{}, errors.New("invalid source enrollment")
|
||||
}
|
||||
secret := make([]byte, 32)
|
||||
if _, err := rand.Read(secret); err != nil {
|
||||
return "", Enrollment{}, errors.New("cryptographic randomness unavailable")
|
||||
}
|
||||
token := "obse1." + hex.EncodeToString(secret)
|
||||
digest := sha256.Sum256([]byte(token))
|
||||
enrollment := Enrollment{SourceID: id, Scope: scope, CreatedByUserID: createdBy, CreatedAt: now.UTC(), ExpiresAt: now.UTC().Add(lifetime)}
|
||||
_, err := s.control.ExecContext(ctx, `INSERT INTO source_enrollments(credential_digest,source_id,organization_id,project_id,environment_id,service_id,created_by_user_id,created_at,expires_at) VALUES(?,?,?,?,?,?,?,?,?)`, digest[:], id, scope.OrganizationID, scope.ProjectID, scope.EnvironmentID, scope.ServiceID, createdBy, enrollment.CreatedAt.Format(time.RFC3339Nano), enrollment.ExpiresAt.Format(time.RFC3339Nano))
|
||||
if err != nil {
|
||||
return "", Enrollment{}, fmt.Errorf("create source enrollment: %w", err)
|
||||
}
|
||||
return token, enrollment, nil
|
||||
}
|
||||
|
||||
func (s *Store) RedeemEnrollment(ctx context.Context, token string, now time.Time) (Enrollment, string, error) {
|
||||
if len(token) != len("obse1.")+64 || !strings.HasPrefix(token, "obse1.") || strings.ContainsAny(token, " \t\r\n") || now.IsZero() {
|
||||
return Enrollment{}, "", errors.New("invalid or expired source enrollment")
|
||||
}
|
||||
digest := sha256.Sum256([]byte(token))
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return Enrollment{}, "", err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var enrollment Enrollment
|
||||
var created, expires string
|
||||
err = tx.QueryRowContext(ctx, `SELECT source_id,organization_id,project_id,environment_id,service_id,created_by_user_id,created_at,expires_at FROM source_enrollments WHERE credential_digest=? AND used_at IS NULL`, digest[:]).Scan(&enrollment.SourceID, &enrollment.Scope.OrganizationID, &enrollment.Scope.ProjectID, &enrollment.Scope.EnvironmentID, &enrollment.Scope.ServiceID, &enrollment.CreatedByUserID, &created, &expires)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Enrollment{}, "", errors.New("invalid or expired source enrollment")
|
||||
}
|
||||
if err != nil {
|
||||
return Enrollment{}, "", err
|
||||
}
|
||||
enrollment.CreatedAt, err = time.Parse(time.RFC3339Nano, created)
|
||||
if err != nil {
|
||||
return Enrollment{}, "", errors.New("stored source enrollment is invalid")
|
||||
}
|
||||
enrollment.ExpiresAt, err = time.Parse(time.RFC3339Nano, expires)
|
||||
if err != nil {
|
||||
return Enrollment{}, "", errors.New("stored source enrollment is invalid")
|
||||
}
|
||||
if !now.UTC().Before(enrollment.ExpiresAt) {
|
||||
return Enrollment{}, "", errors.New("invalid or expired source enrollment")
|
||||
}
|
||||
credential, err := sourceCredential(enrollment.SourceID)
|
||||
if err != nil {
|
||||
return Enrollment{}, "", err
|
||||
}
|
||||
credentialDigest := sha256.Sum256([]byte(credential))
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO sources(id,organization_id,project_id,environment_id,service_id,credential_digest,active,created_at) VALUES(?,?,?,?,?,?,1,?)`, enrollment.SourceID, enrollment.Scope.OrganizationID, enrollment.Scope.ProjectID, enrollment.Scope.EnvironmentID, enrollment.Scope.ServiceID, credentialDigest[:], now.UTC().Format(time.RFC3339Nano)); err != nil {
|
||||
return Enrollment{}, "", errors.New("source enrollment failed")
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `UPDATE source_enrollments SET used_at=? WHERE credential_digest=? AND used_at IS NULL`, now.UTC().Format(time.RFC3339Nano), digest[:])
|
||||
if err != nil {
|
||||
return Enrollment{}, "", err
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return Enrollment{}, "", errors.New("invalid or expired source enrollment")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return Enrollment{}, "", err
|
||||
}
|
||||
return enrollment, credential, nil
|
||||
}
|
||||
|
||||
func (s *Store) CancelEnrollment(ctx context.Context, token string) error {
|
||||
if len(token) != len("obse1.")+64 || !strings.HasPrefix(token, "obse1.") || strings.ContainsAny(token, " \t\r\n") {
|
||||
return errors.New("invalid source enrollment")
|
||||
}
|
||||
digest := sha256.Sum256([]byte(token))
|
||||
result, err := s.control.ExecContext(ctx, `DELETE FROM source_enrollments WHERE credential_digest=? AND used_at IS NULL`, digest[:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return errors.New("source enrollment not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sourceCredential(id string) (string, error) {
|
||||
secret := make([]byte, 32)
|
||||
if _, err := rand.Read(secret); err != nil {
|
||||
return "", errors.New("cryptographic randomness unavailable")
|
||||
}
|
||||
return "obs1." + id + "." + hex.EncodeToString(secret), nil
|
||||
}
|
||||
|
||||
func (s *Store) Authenticate(ctx context.Context, token string) (Source, error) {
|
||||
if len(token) < 48 || len(token) > 512 || !strings.HasPrefix(token, "obs1.") {
|
||||
return Source{}, errors.New("invalid source credential")
|
||||
}
|
||||
digest := sha256.Sum256([]byte(token))
|
||||
var source Source
|
||||
var active int
|
||||
err := s.control.QueryRowContext(ctx, `SELECT id, organization_id, project_id, environment_id, service_id, active FROM sources WHERE credential_digest = ?`, digest[:]).Scan(&source.ID, &source.Scope.OrganizationID, &source.Scope.ProjectID, &source.Scope.EnvironmentID, &source.Scope.ServiceID, &active)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Source{}, errors.New("invalid source credential")
|
||||
}
|
||||
if err != nil {
|
||||
return Source{}, fmt.Errorf("authenticate source: %w", err)
|
||||
}
|
||||
source.Active = active == 1
|
||||
if !source.Active {
|
||||
return Source{}, errors.New("source credential revoked")
|
||||
}
|
||||
return source, nil
|
||||
}
|
||||
|
||||
func (s *Store) RevokeSource(ctx context.Context, id string) error {
|
||||
result, err := s.control.ExecContext(ctx, `UPDATE sources SET active=0, rotated_at=? WHERE id=? AND active=1`, time.Now().UTC().Format(time.RFC3339Nano), id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("revoke source: %w", err)
|
||||
}
|
||||
n, _ := result.RowsAffected()
|
||||
if n != 1 {
|
||||
return errors.New("active source not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Ingest(ctx context.Context, token string, batch model.Batch, now time.Time) (Ack, error) {
|
||||
source, err := s.Authenticate(ctx, token)
|
||||
if err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
if batch.SourceID != source.ID {
|
||||
return Ack{}, errors.New("batch source does not match credential")
|
||||
}
|
||||
lock := s.sourceLock(source.ID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
return s.ingestAuthenticated(ctx, source, batch, nil, now)
|
||||
}
|
||||
|
||||
// IngestNative validates the exact encoded request against its transport
|
||||
// envelope before committing a new batch. A concurrently acknowledged exact
|
||||
// replay is returned without recompressing or rewriting raw evidence.
|
||||
func (s *Store) IngestNative(ctx context.Context, token string, batch model.Batch, envelope model.BatchEnvelope, encoded []byte, now time.Time) (Ack, error) {
|
||||
if err := envelope.Match(batch, encoded); err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
source, err := s.Authenticate(ctx, token)
|
||||
if err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
if batch.SourceID != source.ID {
|
||||
return Ack{}, errors.New("batch source does not match credential")
|
||||
}
|
||||
lock := s.sourceLock(source.ID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
if ack, exact, checkErr := s.checkEnvelope(ctx, source, envelope); checkErr != nil {
|
||||
return Ack{}, checkErr
|
||||
} else if exact {
|
||||
return ack, nil
|
||||
}
|
||||
return s.ingestAuthenticated(ctx, source, batch, &envelope, now)
|
||||
}
|
||||
|
||||
func (s *Store) IngestAuto(ctx context.Context, token, streamID string, signal model.Signal, records []model.Observation, now time.Time) (Ack, error) {
|
||||
source, err := s.Authenticate(ctx, token)
|
||||
if err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
if err = model.ValidateStreamID(streamID); err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
lock := s.sourceLock(source.ID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
lastSequence, _, found, err := s.watermark(ctx, source.ID, streamID)
|
||||
if err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
if lastSequence == ^uint64(0) {
|
||||
return Ack{}, errors.New("stream sequence exhausted")
|
||||
}
|
||||
sequence := uint64(1)
|
||||
if found {
|
||||
sequence = lastSequence + 1
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return Ack{}, errors.New("automatic ingestion requires records")
|
||||
}
|
||||
// Derive the batch observation time from its records so a retry of the
|
||||
// same OTLP payload produces the same content-addressed segment even if a
|
||||
// prior attempt stopped after the atomic raw commit.
|
||||
observedAt := records[0].Timestamp
|
||||
for _, record := range records[1:] {
|
||||
if record.Timestamp.After(observedAt) {
|
||||
observedAt = record.Timestamp
|
||||
}
|
||||
}
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: source.ID, StreamID: streamID, Sequence: sequence, ObservedAt: observedAt.UTC(), Signal: signal, Records: records}
|
||||
return s.ingestAuthenticated(ctx, source, batch, nil, now)
|
||||
}
|
||||
|
||||
func (s *Store) sourceLock(sourceID string) *sync.Mutex {
|
||||
return s.namedLock("source:" + sourceID)
|
||||
}
|
||||
|
||||
func (s *Store) ingestAuthenticated(ctx context.Context, source Source, batch model.Batch, envelope *model.BatchEnvelope, now time.Time) (Ack, error) {
|
||||
if err := batch.Validate(now); err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
batchDigest, err := batch.Digest()
|
||||
if err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
if err := validateMetricRollupCardinality(batch); err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
lastSequence, lastDigest, found, err := s.watermark(ctx, source.ID, batch.StreamID)
|
||||
if err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
if found && batch.Sequence < lastSequence {
|
||||
return Ack{}, errors.New("sequence replay is older than acknowledged watermark")
|
||||
}
|
||||
if (!found && batch.Sequence != 1) || (found && batch.Sequence > lastSequence+1) {
|
||||
return Ack{}, errors.New("sequence gap")
|
||||
}
|
||||
committed, err := s.segments.Commit(source.Scope, batch)
|
||||
if err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
if found && batch.Sequence == lastSequence {
|
||||
if committed.Digest != lastDigest {
|
||||
if deleteErr := s.segments.Delete(committed.Path, committed.Digest); deleteErr != nil {
|
||||
return Ack{}, fmt.Errorf("acknowledged sequence reused with different content; remove rejected raw object: %w", deleteErr)
|
||||
}
|
||||
return Ack{}, errors.New("acknowledged sequence reused with different content")
|
||||
}
|
||||
if envelope != nil {
|
||||
if err = s.backfillAcknowledgedEnvelope(ctx, batch, committed.Digest, *envelope); err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
}
|
||||
return Ack{SourceID: source.ID, StreamID: batch.StreamID, Sequence: batch.Sequence, Digest: committed.Digest, BatchDigest: batchDigest, Duplicate: true}, nil
|
||||
}
|
||||
if err := s.admitCommittedEnvelope(ctx, source.Scope, batch, committed, envelope, now); err != nil {
|
||||
return Ack{}, err
|
||||
}
|
||||
s.notifyProjector()
|
||||
return Ack{SourceID: source.ID, StreamID: batch.StreamID, Sequence: batch.Sequence, Digest: committed.Digest, BatchDigest: batchDigest}, nil
|
||||
}
|
||||
|
||||
func (s *Store) notifyProjector() {
|
||||
select {
|
||||
case s.projectionWake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) watermark(ctx context.Context, sourceID, streamID string) (uint64, string, bool, error) {
|
||||
var sequence uint64
|
||||
var digest string
|
||||
err := s.control.QueryRowContext(ctx, `SELECT last_sequence, last_digest FROM streams WHERE source_id=? AND stream_id=?`, sourceID, streamID).Scan(&sequence, &digest)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, "", false, fmt.Errorf("read stream watermark: %w", err)
|
||||
}
|
||||
return sequence, digest, true, nil
|
||||
}
|
||||
|
||||
func (s *Store) recordCommitted(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed) error {
|
||||
return s.recordCommittedAt(ctx, scope, batch, committed, time.Now().UTC())
|
||||
}
|
||||
|
||||
func (s *Store) recordCommittedAt(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, committedAt time.Time) error {
|
||||
return s.recordCommittedAtEnvelope(ctx, scope, batch, committed, nil, committedAt)
|
||||
}
|
||||
|
||||
func (s *Store) recordCommittedAtEnvelope(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, envelope *model.BatchEnvelope, committedAt time.Time) error {
|
||||
if committedAt.IsZero() {
|
||||
return errors.New("committed segment time is required")
|
||||
}
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin committed segment record: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err = recordCommittedTx(ctx, tx, scope, batch, committed, committedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = advanceStreamTx(ctx, tx, batch, committed.Digest, envelope); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit segment record: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func recordCommittedTx(ctx context.Context, tx *sql.Tx, scope model.Scope, batch model.Batch, committed segment.Committed, committedAt time.Time) error {
|
||||
first, last := observationRange(batch)
|
||||
_, err := tx.ExecContext(ctx, `INSERT INTO segments(digest, organization_id, source_id, stream_id, sequence, path, compressed_bytes, uncompressed_bytes, committed_at, signal, first_observed_at, last_observed_at, record_count) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`, committed.Digest, scope.OrganizationID, batch.SourceID, batch.StreamID, batch.Sequence, committed.Path, committed.Compressed, committed.Uncompressed, committedAt.UTC().Format(time.RFC3339Nano), batch.Signal, first.Format(time.RFC3339Nano), last.Format(time.RFC3339Nano), len(batch.Records))
|
||||
if err != nil {
|
||||
return fmt.Errorf("record committed segment: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func observationRange(batch model.Batch) (time.Time, time.Time) {
|
||||
first, last := batch.Records[0].Timestamp.UTC(), batch.Records[0].Timestamp.UTC()
|
||||
for _, observation := range batch.Records[1:] {
|
||||
timestamp := observation.Timestamp.UTC()
|
||||
if timestamp.Before(first) {
|
||||
first = timestamp
|
||||
}
|
||||
if timestamp.After(last) {
|
||||
last = timestamp
|
||||
}
|
||||
}
|
||||
return first, last
|
||||
}
|
||||
|
||||
func (s *Store) project(ctx context.Context, scope model.Scope, batch model.Batch, digest string) error {
|
||||
lock := s.namedLock("organization:" + scope.OrganizationID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
db, err := s.projection(ctx, scope.OrganizationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return projectWithDB(ctx, db, scope, batch, digest)
|
||||
}
|
||||
|
||||
func projectAt(ctx context.Context, path string, scope model.Scope, batch model.Batch, digest string) error {
|
||||
db, err := openProjection(ctx, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
return projectWithDB(ctx, db, scope, batch, digest)
|
||||
}
|
||||
|
||||
type projectionItem struct {
|
||||
scope model.Scope
|
||||
batch model.Batch
|
||||
digest string
|
||||
}
|
||||
|
||||
func projectWithDB(ctx context.Context, db *sql.DB, scope model.Scope, batch model.Batch, digest string) error {
|
||||
return projectGroupWithDB(ctx, db, []projectionItem{{scope: scope, batch: batch, digest: digest}})
|
||||
}
|
||||
|
||||
func projectGroupWithDB(ctx context.Context, db *sql.DB, items []projectionItem) error {
|
||||
if len(items) == 0 {
|
||||
return errors.New("projection group is empty")
|
||||
}
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin projection: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
activeVersion, activeRegistry, activeDescriptors, err := activeProjection(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
insert, err := tx.PrepareContext(ctx, `INSERT OR IGNORE INTO observations(organization_id, project_id, environment_id, service_id, source_id, stream_id, sequence, record_index, signal, timestamp, name, severity, body, value, trace_id, span_id, correlation_id, attributes_json, segment_digest) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare observation projection: %w", err)
|
||||
}
|
||||
defer insert.Close()
|
||||
for _, item := range items {
|
||||
batch, scope, digest := item.batch, item.scope, item.digest
|
||||
var observationBytes []int64
|
||||
if batch.Signal == model.SignalLogs {
|
||||
observationBytes = make([]int64, 0, len(batch.Records))
|
||||
}
|
||||
for index, observation := range batch.Records {
|
||||
attributes, marshalErr := json.Marshal(observation.Attributes)
|
||||
if marshalErr != nil {
|
||||
return fmt.Errorf("encode observation attributes: %w", marshalErr)
|
||||
}
|
||||
_, execErr := insert.ExecContext(ctx, scope.OrganizationID, scope.ProjectID, scope.EnvironmentID, scope.ServiceID, batch.SourceID, batch.StreamID, batch.Sequence, index, batch.Signal, observation.Timestamp.UTC().Format(time.RFC3339Nano), observation.Name, observation.Severity, observation.Body, observation.Value, observation.TraceID, observation.SpanID, observation.CorrelationID, string(attributes), digest)
|
||||
if execErr != nil {
|
||||
return fmt.Errorf("project observation: %w", execErr)
|
||||
}
|
||||
if err = indexProjectedObservation(ctx, tx, activeVersion, activeDescriptors, batch, index, observation); err != nil {
|
||||
return err
|
||||
}
|
||||
if batch.Signal == model.SignalLogs {
|
||||
observationBytes = append(observationBytes, projectedObservationBytes(scope, batch, observation, len(attributes)))
|
||||
}
|
||||
}
|
||||
if err = projectMetricRollups(ctx, tx, scope, batch, digest, activeRegistry); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = projectLogRollups(ctx, tx, scope, batch, digest, observationBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit projection: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) projection(ctx context.Context, organizationID string) (*sql.DB, error) {
|
||||
path := filepath.Join(s.root, "organizations", organizationID, "projection.sqlite")
|
||||
s.projectionMu.Lock()
|
||||
handle, exists := s.projections[organizationID]
|
||||
s.projectionMu.Unlock()
|
||||
if exists {
|
||||
device, inode, err := projectionIdentity(path)
|
||||
if err != nil || device != handle.device || inode != handle.inode {
|
||||
return nil, errors.New("organization projection identity changed")
|
||||
}
|
||||
return handle.db, nil
|
||||
}
|
||||
db, err := openProjection(ctx, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
device, inode, err := projectionIdentity(path)
|
||||
if err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
s.projectionMu.Lock()
|
||||
s.projections[organizationID] = projectionHandle{db: db, device: device, inode: inode}
|
||||
s.projectionMu.Unlock()
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (s *Store) closeProjection(organizationID string) error {
|
||||
s.projectionMu.Lock()
|
||||
handle, exists := s.projections[organizationID]
|
||||
if exists {
|
||||
delete(s.projections, organizationID)
|
||||
}
|
||||
s.projectionMu.Unlock()
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
return handle.db.Close()
|
||||
}
|
||||
|
||||
func projectionIdentity(path string) (uint64, uint64, error) {
|
||||
if err := validateSQLiteFileSet(path); err != nil {
|
||||
return 0, 0, fmt.Errorf("inspect organization projection: %w", err)
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return 0, 0, errors.New("organization projection must be a regular non-symlink file")
|
||||
}
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok || stat.Nlink != 1 {
|
||||
return 0, 0, errors.New("organization projection identity is unavailable")
|
||||
}
|
||||
return uint64(stat.Dev), uint64(stat.Ino), nil
|
||||
}
|
||||
|
||||
func openProjection(ctx context.Context, path string) (*sql.DB, error) {
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("create organization store: %w", err)
|
||||
}
|
||||
for _, directory := range []string{filepath.Dir(dir), dir} {
|
||||
info, err := os.Lstat(directory)
|
||||
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 {
|
||||
return nil, errors.New("organization store path must contain private non-symlink directories")
|
||||
}
|
||||
}
|
||||
if err := validateSQLiteFileSet(path); err != nil {
|
||||
return nil, fmt.Errorf("inspect organization projection: %w", err)
|
||||
}
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open organization projection: %w", err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if _, err := db.ExecContext(ctx, `PRAGMA journal_mode=WAL`); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("configure organization projection: %w", err)
|
||||
}
|
||||
for _, statement := range []string{`PRAGMA synchronous=FULL`, `PRAGMA busy_timeout=5000`, `PRAGMA foreign_keys=ON`} {
|
||||
if _, err := db.ExecContext(ctx, statement); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("configure organization projection: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS observations (
|
||||
organization_id TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL,
|
||||
environment_id TEXT NOT NULL,
|
||||
service_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
stream_id TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL,
|
||||
record_index INTEGER NOT NULL,
|
||||
signal TEXT NOT NULL,
|
||||
timestamp TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
severity TEXT,
|
||||
body TEXT,
|
||||
value REAL,
|
||||
trace_id TEXT,
|
||||
span_id TEXT,
|
||||
correlation_id TEXT,
|
||||
attributes_json TEXT NOT NULL,
|
||||
segment_digest TEXT NOT NULL,
|
||||
PRIMARY KEY(source_id, stream_id, sequence, record_index)
|
||||
)`); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("migrate organization projection: %w", err)
|
||||
}
|
||||
for _, statement := range []string{
|
||||
`CREATE INDEX IF NOT EXISTS observations_signal_time ON observations(signal,timestamp)`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_scope ON observations(project_id,environment_id,service_id,signal,timestamp)`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_environment ON observations(environment_id,signal,timestamp)`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_service ON observations(service_id,signal,timestamp)`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_name ON observations(signal,name,timestamp)`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_severity ON observations(signal,severity,timestamp)`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_trace ON observations(trace_id) WHERE trace_id IS NOT NULL`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_span ON observations(span_id) WHERE span_id IS NOT NULL`,
|
||||
`CREATE INDEX IF NOT EXISTS observations_correlation ON observations(correlation_id) WHERE correlation_id IS NOT NULL`,
|
||||
} {
|
||||
if _, err := db.ExecContext(ctx, statement); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("index organization projection: %w", err)
|
||||
}
|
||||
}
|
||||
if err := ensureProjectionMetadata(ctx, db); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := ensureMetricRollups(ctx, db); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := ensureBaseIndexes(ctx, db); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := ensureLogRollups(ctx, db); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chmod(path, 0o600); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("set organization database mode: %w", err)
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func validateSQLiteFileSet(path string) error {
|
||||
for _, candidate := range []string{path, path + "-wal", path + "-shm"} {
|
||||
info, err := os.Lstat(candidate)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return errors.New("SQLite file must be a regular non-symlink file")
|
||||
}
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok || stat.Nlink != 1 {
|
||||
return errors.New("SQLite file must not have additional hard links")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) markProjected(ctx context.Context, batch model.Batch, digest string) error {
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin acknowledgement: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err = markProjectedTx(ctx, tx, batch, digest, time.Now().UTC()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit acknowledgement: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func markProjectedTx(ctx context.Context, tx *sql.Tx, batch model.Batch, digest string, projectedAt time.Time) error {
|
||||
result, err := tx.ExecContext(ctx, `UPDATE segments SET projected_at=? WHERE digest=? AND projected_at IS NULL`, projectedAt.UTC().Format(time.RFC3339Nano), digest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mark segment projected: %w", err)
|
||||
}
|
||||
n, _ := result.RowsAffected()
|
||||
if n != 1 {
|
||||
return errors.New("committed segment state changed before acknowledgement")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func advanceStreamTx(ctx context.Context, tx *sql.Tx, batch model.Batch, digest string, envelope *model.BatchEnvelope) error {
|
||||
var lastSequence uint64
|
||||
var lastDigest string
|
||||
err := tx.QueryRowContext(ctx, `SELECT last_sequence,last_digest FROM streams WHERE source_id=? AND stream_id=?`, batch.SourceID, batch.StreamID).Scan(&lastSequence, &lastDigest)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
if batch.Sequence != 1 {
|
||||
return errors.New("sequence gap while accepting committed segment")
|
||||
}
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("read stream watermark while accepting committed segment: %w", err)
|
||||
} else {
|
||||
if batch.Sequence == lastSequence && digest == lastDigest {
|
||||
return nil
|
||||
}
|
||||
if lastSequence == ^uint64(0) || batch.Sequence != lastSequence+1 {
|
||||
return errors.New("sequence gap while accepting committed segment")
|
||||
}
|
||||
}
|
||||
batchDigest, wireDigest, signal, recordCount, encodedBytes, first, last := envelopeSQL(envelope)
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO streams(source_id,stream_id,last_sequence,last_digest,last_batch_digest,last_wire_digest,last_signal,last_record_count,last_encoded_bytes,last_first_observed_at,last_last_observed_at) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_id,stream_id) DO UPDATE SET last_sequence=excluded.last_sequence,last_digest=excluded.last_digest,last_batch_digest=excluded.last_batch_digest,last_wire_digest=excluded.last_wire_digest,last_signal=excluded.last_signal,last_record_count=excluded.last_record_count,last_encoded_bytes=excluded.last_encoded_bytes,last_first_observed_at=excluded.last_first_observed_at,last_last_observed_at=excluded.last_last_observed_at`, batch.SourceID, batch.StreamID, batch.Sequence, digest, batchDigest, wireDigest, signal, recordCount, encodedBytes, first, last)
|
||||
if err != nil {
|
||||
return fmt.Errorf("advance stream watermark: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecoverRaw reconciles immutable raw objects with the control catalog. It
|
||||
// deliberately does not project telemetry, so a large projection backlog
|
||||
// cannot delay server readiness.
|
||||
func (s *Store) RecoverRaw(ctx context.Context) error {
|
||||
if err := s.finishInterruptedArchival(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.finishInterruptedRetention(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
lookup, err := s.control.PrepareContext(ctx, `SELECT organization_id,source_id,stream_id,sequence,path,compressed_bytes,tier FROM segments WHERE digest=?`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare recovered segment lookup: %w", err)
|
||||
}
|
||||
var missing []segment.Metadata
|
||||
walkErr := s.segments.WalkMetadata(func(metadata segment.Metadata) error {
|
||||
var catalog segment.Metadata
|
||||
var tier string
|
||||
catalog.Digest = metadata.Digest
|
||||
lookupErr := lookup.QueryRowContext(ctx, metadata.Digest).Scan(&catalog.OrganizationID, &catalog.SourceID, &catalog.StreamID, &catalog.Sequence, &catalog.Path, &catalog.Compressed, &tier)
|
||||
if lookupErr == nil {
|
||||
if tier != "hot" || catalog != metadata {
|
||||
return errors.New("catalogued segment metadata does not match raw object")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(lookupErr, sql.ErrNoRows) {
|
||||
return fmt.Errorf("inspect recovered segment: %w", lookupErr)
|
||||
}
|
||||
missing = append(missing, metadata)
|
||||
return nil
|
||||
})
|
||||
if closeErr := lookup.Close(); walkErr == nil && closeErr != nil {
|
||||
walkErr = fmt.Errorf("close recovered segment lookup: %w", closeErr)
|
||||
}
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
sort.Slice(missing, func(i, j int) bool {
|
||||
left, right := missing[i], missing[j]
|
||||
if left.SourceID != right.SourceID {
|
||||
return left.SourceID < right.SourceID
|
||||
}
|
||||
if left.StreamID != right.StreamID {
|
||||
return left.StreamID < right.StreamID
|
||||
}
|
||||
return left.Sequence < right.Sequence
|
||||
})
|
||||
for _, metadata := range missing {
|
||||
entry, readErr := s.segments.ReadEntry(metadata)
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
source, sourceErr := s.sourceByID(ctx, entry.Batch.SourceID)
|
||||
if sourceErr != nil {
|
||||
return sourceErr
|
||||
}
|
||||
if validateErr := entry.Batch.Validate(entry.Batch.ObservedAt); validateErr != nil {
|
||||
return fmt.Errorf("validate recovered segment: %w", validateErr)
|
||||
}
|
||||
if validateErr := validateMetricRollupCardinality(entry.Batch); validateErr != nil {
|
||||
return fmt.Errorf("validate recovered metric segment: %w", validateErr)
|
||||
}
|
||||
if source.Scope.OrganizationID != entry.OrganizationID {
|
||||
return errors.New("recovered segment organization does not match enrolled source")
|
||||
}
|
||||
if admitErr := s.admitCommitted(ctx, source.Scope, entry.Batch, entry.Committed, time.Now().UTC()); admitErr != nil {
|
||||
if errors.Is(admitErr, ErrOrganizationStorageQuotaExceeded) {
|
||||
continue
|
||||
}
|
||||
return admitErr
|
||||
}
|
||||
s.notifyProjector()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Recover performs full offline recovery. The server uses RecoverRaw and a
|
||||
// background projector; check and migration commands retain this blocking
|
||||
// form so they can prove every durable segment is queryable before returning.
|
||||
func (s *Store) Recover(ctx context.Context) error {
|
||||
if err := s.RecoverRaw(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
report, err := s.ProjectPending(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if report.ProjectedSegments == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) sourceByID(ctx context.Context, id string) (Source, error) {
|
||||
var source Source
|
||||
var active int
|
||||
err := s.control.QueryRowContext(ctx, `SELECT id, organization_id, project_id, environment_id, service_id, active FROM sources WHERE id=?`, id).Scan(&source.ID, &source.Scope.OrganizationID, &source.Scope.ProjectID, &source.Scope.EnvironmentID, &source.Scope.ServiceID, &active)
|
||||
if err != nil {
|
||||
return Source{}, fmt.Errorf("load source: %w", err)
|
||||
}
|
||||
source.Active = active == 1
|
||||
return source, nil
|
||||
}
|
||||
@@ -0,0 +1,516 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
func TestNativeEnvelopeReplayUsesBatchIdentityAndAllowsOverlappingTime(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
token, err := store.CreateSource(ctx, "source-framed", model.Scope{OrganizationID: "organization", ProjectID: "project", EnvironmentID: "production", ServiceID: "service"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 18, 20, 0, 0, 0, time.UTC)
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-framed", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}}
|
||||
body, _ := json.Marshal(batch)
|
||||
envelope, _ := batch.Envelope(body)
|
||||
ack, err := store.IngestNative(ctx, token, batch, envelope, body, now)
|
||||
if err != nil || ack.Duplicate {
|
||||
t.Fatalf("ack=%+v err=%v", ack, err)
|
||||
}
|
||||
preflight, exact, err := store.CheckNativeReplay(ctx, token, envelope)
|
||||
if err != nil || !exact || !preflight.Duplicate || preflight.Digest != ack.Digest {
|
||||
t.Fatalf("preflight=%+v exact=%t err=%v", preflight, exact, err)
|
||||
}
|
||||
// A known exact retry remains acknowledgeable after its original ingest
|
||||
// clock window; the immutable bytes and persisted envelope are authoritative.
|
||||
confirmed, err := store.ConfirmNativeReplay(ctx, token, envelope)
|
||||
if err != nil || !confirmed.Duplicate || confirmed.BatchDigest != envelope.BatchDigest {
|
||||
t.Fatalf("confirmed=%+v err=%v", confirmed, err)
|
||||
}
|
||||
conflict := envelope
|
||||
conflict.RecordCount++
|
||||
if _, _, err = store.CheckNativeReplay(ctx, token, conflict); err == nil {
|
||||
t.Fatal("same sequence with conflicting envelope was accepted")
|
||||
}
|
||||
|
||||
// A second batch may overlap the first batch's timestamps. Time is a
|
||||
// partition hint, not a deduplication key.
|
||||
batch.Sequence = 2
|
||||
batch.ObservedAt = now.Add(time.Second)
|
||||
body, _ = json.Marshal(batch)
|
||||
envelope, _ = batch.Envelope(body)
|
||||
if _, err = store.IngestNative(ctx, token, batch, envelope, body, now.Add(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var batchDigest, wireDigest string
|
||||
var recordCount int
|
||||
if err = store.control.QueryRow(`SELECT last_batch_digest,last_wire_digest,last_record_count FROM streams WHERE source_id=? AND stream_id=?`, batch.SourceID, batch.StreamID).Scan(&batchDigest, &wireDigest, &recordCount); err != nil || batchDigest != envelope.BatchDigest || wireDigest != envelope.WireDigest || recordCount != 1 {
|
||||
t.Fatalf("batch=%q wire=%q count=%d err=%v", batchDigest, wireDigest, recordCount, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestAutoSerializesConcurrentSequenceAssignment(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
token, err := store.CreateSource(ctx, "source-auto", model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
const requests = 16
|
||||
results := make(chan Ack, requests)
|
||||
errors := make(chan error, requests)
|
||||
var group sync.WaitGroup
|
||||
for index := 0; index < requests; index++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
ack, ingestErr := store.IngestAuto(ctx, token, "otlp-logs", model.SignalLogs, []model.Observation{{Timestamp: now, Name: "http.request"}}, now)
|
||||
if ingestErr != nil {
|
||||
errors <- ingestErr
|
||||
return
|
||||
}
|
||||
results <- ack
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
close(results)
|
||||
close(errors)
|
||||
for ingestErr := range errors {
|
||||
t.Errorf("ingest: %v", ingestErr)
|
||||
}
|
||||
var sequences []int
|
||||
for ack := range results {
|
||||
if ack.Duplicate || ack.Digest == "" || ack.StreamID != "otlp-logs" {
|
||||
t.Errorf("ack=%+v", ack)
|
||||
}
|
||||
sequences = append(sequences, int(ack.Sequence))
|
||||
}
|
||||
sort.Ints(sequences)
|
||||
if len(sequences) != requests {
|
||||
t.Fatalf("sequences=%v", sequences)
|
||||
}
|
||||
for index, sequence := range sequences {
|
||||
if sequence != index+1 {
|
||||
t.Fatalf("sequences=%v", sequences)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
root := filepath.Join(t.TempDir(), "data")
|
||||
if err := os.Mkdir(root, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err := Open(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func projectAll(t testing.TB, store *Store) {
|
||||
t.Helper()
|
||||
for {
|
||||
report, err := store.ProjectPending(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.ProjectedSegments == 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageRejectsSymlinkedSQLiteFilesAndProjectionDirectories(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
root := filepath.Join(base, "data")
|
||||
if err := os.Mkdir(root, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := filepath.Join(base, "target.sqlite")
|
||||
if err := os.WriteFile(target, []byte("target"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(target, filepath.Join(root, "control.sqlite")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Open(root); err == nil {
|
||||
t.Fatal("symlink control database was accepted")
|
||||
}
|
||||
if err := os.Remove(filepath.Join(root, "control.sqlite")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Link(target, filepath.Join(root, "control.sqlite")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Open(root); err == nil {
|
||||
t.Fatal("hard-linked control database was accepted")
|
||||
}
|
||||
if err := os.Remove(filepath.Join(root, "control.sqlite")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(root, "organizations"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(base, filepath.Join(root, "organizations", "organization-a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := openProjection(context.Background(), filepath.Join(root, "organizations", "organization-a", "projection.sqlite")); err == nil {
|
||||
t.Fatal("symlink organization directory was accepted")
|
||||
}
|
||||
if err := os.Remove(filepath.Join(root, "organizations", "organization-a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
organization := filepath.Join(root, "organizations", "organization-a")
|
||||
if err := os.Mkdir(organization, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projection := filepath.Join(organization, "projection.sqlite")
|
||||
if err := os.Symlink(target, projection+"-wal"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := openProjection(context.Background(), projection); err == nil {
|
||||
t.Fatal("symlink projection sidecar was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedIngestionDeduplicationAndReplay(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
scope := model.Scope{OrganizationID: "org-a", ProjectID: "site", EnvironmentID: "prod", ServiceID: "web"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
batch := model.Batch{Version: 1, SourceID: "source-a", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "http.request", Attributes: map[string]string{"route": "/"}}}}
|
||||
ack, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ack.Duplicate || ack.Digest == "" {
|
||||
t.Fatalf("unexpected acknowledgement: %#v", ack)
|
||||
}
|
||||
duplicate, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !duplicate.Duplicate || duplicate.Digest != ack.Digest {
|
||||
t.Fatalf("unexpected duplicate acknowledgement: %#v", duplicate)
|
||||
}
|
||||
batch.Records[0].Name = "changed"
|
||||
if _, err := store.Ingest(ctx, token, batch, now); err == nil {
|
||||
t.Fatal("expected conflicting duplicate rejection")
|
||||
}
|
||||
entries, err := store.segments.List()
|
||||
if err != nil || len(entries) != 1 || entries[0].Committed.Digest != ack.Digest {
|
||||
t.Fatalf("conflicting replay left raw evidence behind: entries=%+v err=%v", entries, err)
|
||||
}
|
||||
batch.Sequence = 3
|
||||
if _, err := store.Ingest(ctx, token, batch, now); err == nil {
|
||||
t.Fatal("expected sequence gap rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectorReusesProjectionHandleAndRejectsPathReplacement(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
ingest := func(sequence uint64) error {
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: sequence, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
|
||||
_, ingestErr := store.Ingest(ctx, token, batch, now)
|
||||
return ingestErr
|
||||
}
|
||||
if err = ingest(1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
store.projectionMu.Lock()
|
||||
first := store.projections[scope.OrganizationID]
|
||||
store.projectionMu.Unlock()
|
||||
if first.db == nil {
|
||||
t.Fatal("projection handle was not retained")
|
||||
}
|
||||
if err = ingest(2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
store.projectionMu.Lock()
|
||||
second := store.projections[scope.OrganizationID]
|
||||
store.projectionMu.Unlock()
|
||||
if second.db != first.db || second.device != first.device || second.inode != first.inode {
|
||||
t.Fatal("projection handle was reopened for an unchanged organization")
|
||||
}
|
||||
|
||||
projection := filepath.Join(store.root, "organizations", scope.OrganizationID, "projection.sqlite")
|
||||
replaced := projection + ".replaced"
|
||||
if err = os.Rename(projection, replaced); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = ingest(3); err != nil {
|
||||
t.Fatalf("durable ingestion depended on projection path: %v", err)
|
||||
}
|
||||
if _, err = store.ProjectPending(ctx); err == nil {
|
||||
t.Fatal("replaced projection path was accepted by projector")
|
||||
}
|
||||
if err = os.Rename(replaced, projection); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.Recover(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var projected int
|
||||
if err = second.db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&projected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if projected != 3 {
|
||||
t.Fatalf("projected=%d", projected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCredentialScopeCannotBeOverridden(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "org-a", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
batch := model.Batch{Version: 1, SourceID: "source-b", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}}
|
||||
if _, err := store.Ingest(ctx, token, batch, now); err == nil {
|
||||
t.Fatal("expected source mismatch rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryIndexesRawSegmentMissingFromControlDatabase(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
token, err := store.CreateSource(ctx, "source-a", model.Scope{OrganizationID: "org-a", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
batch := model.Batch{Version: 1, SourceID: "source-a", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}}
|
||||
if _, err := store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.control.Exec(`DELETE FROM streams; DELETE FROM segments`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Recover(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var segments, projected int
|
||||
if err := store.control.QueryRow(`SELECT COUNT(*), COUNT(projected_at) FROM segments`).Scan(&segments, &projected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if segments != 1 || projected != 1 {
|
||||
t.Fatalf("segments=%d projected=%d", segments, projected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryRejectsCataloguedRawMetadataMismatchWithoutDecoding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
scope := model.Scope{OrganizationID: "org-a", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
batch := model.Batch{Version: 1, SourceID: "source-a", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}}
|
||||
if _, err = store.Ingest(ctx, token, batch, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var original string
|
||||
if err = store.control.QueryRow(`SELECT path FROM segments`).Scan(&original); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
destination := filepath.Join(store.root, "raw", "org-a", "source-b", "access", filepath.Base(original))
|
||||
if err = os.MkdirAll(filepath.Dir(destination), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.Rename(original, destination); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.Recover(ctx); err == nil {
|
||||
t.Fatal("catalogued segment identity mismatch was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryDoesNotDecodeAlreadyCataloguedRawSegments(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
scope := model.Scope{OrganizationID: "org-a", ProjectID: "p", EnvironmentID: "prod", ServiceID: "s"}
|
||||
token, err := store.CreateSource(ctx, "source-a", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
batch := model.Batch{Version: 1, SourceID: "source-a", StreamID: "access", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "request"}}}
|
||||
ack, err := store.Ingest(ctx, token, batch, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projectAll(t, store)
|
||||
var path string
|
||||
if err = store.control.QueryRow(`SELECT path FROM segments WHERE digest=?`, ack.Digest).Scan(&path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body[0] ^= 0xff
|
||||
if err = os.WriteFile(path, body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.Recover(ctx); err != nil {
|
||||
t.Fatalf("startup decoded catalogued evidence: %v", err)
|
||||
}
|
||||
if _, err = store.segments.Read(path, ack.Digest); err == nil {
|
||||
t.Fatal("explicit forensic read accepted corrupt evidence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryProcessesUnprojectedSegmentsInBoundedPages(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
if _, err := store.CreateSource(ctx, "source-a", scope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
const count = recoveryPageSize + 3
|
||||
for sequence := uint64(1); sequence <= count; sequence++ {
|
||||
batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: sequence, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
|
||||
committed, err := store.segments.Commit(scope, batch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.recordCommitted(ctx, scope, batch, committed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := store.Recover(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var segments, projected int
|
||||
if err := store.control.QueryRow(`SELECT COUNT(*), COUNT(projected_at) FROM segments`).Scan(&segments, &projected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if segments != count || projected != count {
|
||||
t.Fatalf("segments=%d projected=%d", segments, projected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommittedAdmissionRejectsInvalidTimeWithoutControlState(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
if _, err := store.CreateSource(ctx, "source-a", scope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
batch := model.Batch{
|
||||
Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1,
|
||||
ObservedAt: now, Signal: model.SignalLogs,
|
||||
Records: []model.Observation{{Timestamp: now, Name: "application.request", Attributes: map[string]string{"workshop.unknown": "value"}}},
|
||||
}
|
||||
committed, err := store.segments.Commit(scope, batch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.admitCommitted(ctx, scope, batch, committed, time.Time{}); err == nil {
|
||||
t.Fatal("invalid committed time was accepted")
|
||||
}
|
||||
var segments, streams, proposals int
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM segments`).Scan(&segments); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM streams`).Scan(&streams); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM descriptor_proposals`).Scan(&proposals); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if segments != 0 || streams != 0 || proposals != 0 {
|
||||
t.Fatalf("partial durable control state: segments=%d streams=%d proposals=%d", segments, streams, proposals)
|
||||
}
|
||||
if err = store.Recover(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var projected int
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*), COUNT(projected_at) FROM segments`).Scan(&segments, &projected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM streams`).Scan(&streams); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.control.QueryRow(`SELECT COUNT(*) FROM descriptor_proposals`).Scan(&proposals); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if segments != 1 || projected != 1 || streams != 1 || proposals != 1 {
|
||||
t.Fatalf("recovery did not complete durable projection state: segments=%d projected=%d streams=%d proposals=%d", segments, projected, streams, proposals)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrollmentIsScopedExpiringAndSingleUse(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := testStore(t)
|
||||
defer store.Close()
|
||||
now := time.Date(2026, 8, 17, 2, 0, 0, 0, time.UTC)
|
||||
scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
|
||||
token, enrollment, err := store.CreateEnrollment(ctx, "source-a", scope, "operator-a", 15*time.Minute, now)
|
||||
if err != nil || token == "" || enrollment.Scope != scope {
|
||||
t.Fatalf("token_present=%t enrollment=%+v err=%v", token != "", enrollment, err)
|
||||
}
|
||||
got, credential, err := store.RedeemEnrollment(ctx, token, now.Add(time.Minute))
|
||||
if err != nil || got.SourceID != "source-a" || credential == "" {
|
||||
t.Fatalf("enrollment=%+v credential_present=%t err=%v", got, credential != "", err)
|
||||
}
|
||||
source, err := store.Authenticate(ctx, credential)
|
||||
if err != nil || source.Scope != scope {
|
||||
t.Fatalf("source=%+v err=%v", source, err)
|
||||
}
|
||||
if _, _, err = store.RedeemEnrollment(ctx, token, now.Add(2*time.Minute)); err == nil {
|
||||
t.Fatal("single-use enrollment redeemed twice")
|
||||
}
|
||||
expired, _, err := store.CreateEnrollment(ctx, "source-b", scope, "operator-a", 5*time.Minute, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = store.RedeemEnrollment(ctx, expired, now.Add(5*time.Minute)); err == nil {
|
||||
t.Fatal("expired enrollment accepted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user