docs: publish Preview 19 dogfood evidence
Export the reviewed allowlisted snapshot from private source commit 05928cebd01b586cf9e9d4b8c8537a7605a6068c. This records the exact candidate, bounded capacity result, stateful migration scratch requirement, authenticated batch identity proof, and immediate live acceptance evidence. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
This commit is contained in:
@@ -0,0 +1,855 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/segment"
|
||||
)
|
||||
|
||||
const maximumRetentionDays = 3650
|
||||
|
||||
var ErrOrganizationStorageQuotaExceeded = errors.New("organization storage quota exceeded")
|
||||
|
||||
// RetentionPolicy is the complete storage lifecycle for one organization.
|
||||
// Raw metric samples may expire before their five-minute aggregate projection.
|
||||
type RetentionPolicy struct {
|
||||
RawLogsDays int `json:"raw_logs_days"`
|
||||
RawTracesDays int `json:"raw_traces_days"`
|
||||
RawMetricsDays int `json:"raw_metrics_days"`
|
||||
ColdRawDays int `json:"cold_raw_days"`
|
||||
DeleteColdRaw bool `json:"delete_cold_raw"`
|
||||
MetricRollupsDays int `json:"metric_rollups_days"`
|
||||
EvidenceDays int `json:"evidence_days"`
|
||||
}
|
||||
|
||||
func (policy RetentionPolicy) Validate() error {
|
||||
for _, days := range []int{policy.RawLogsDays, policy.RawTracesDays, policy.RawMetricsDays, policy.ColdRawDays, policy.MetricRollupsDays, policy.EvidenceDays} {
|
||||
if days < 1 || days > maximumRetentionDays {
|
||||
return errors.New("retention values must be between 1 and 3650 days")
|
||||
}
|
||||
}
|
||||
if policy.MetricRollupsDays < policy.RawMetricsDays {
|
||||
return errors.New("metric rollup retention cannot be shorter than raw metric retention")
|
||||
}
|
||||
if policy.ColdRawDays < policy.RawLogsDays || policy.ColdRawDays < policy.RawTracesDays || policy.ColdRawDays < policy.RawMetricsDays || policy.ColdRawDays < policy.EvidenceDays {
|
||||
return errors.New("cold raw retention cannot be shorter than a hot raw or evidence retention window")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type OrganizationRetention struct {
|
||||
OrganizationID string `json:"organization_id"`
|
||||
Policy RetentionPolicy `json:"policy"`
|
||||
QuotaBytes int64 `json:"quota_bytes,omitempty"`
|
||||
ExtensionApproved bool `json:"extension_approved"`
|
||||
ExtensionApprovedBy string `json:"extension_approved_by,omitempty"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type SetRetentionInput struct {
|
||||
OrganizationID string
|
||||
Policy RetentionPolicy
|
||||
Defaults RetentionPolicy
|
||||
ActorUserID string
|
||||
ApproveExtensionFor string
|
||||
QuotaBytes int64
|
||||
}
|
||||
|
||||
type RetentionReport struct {
|
||||
Version int `json:"version"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
Organizations int `json:"organizations"`
|
||||
RawSegmentsRemoved int `json:"raw_segments_removed"`
|
||||
RawBytesRemoved int64 `json:"raw_bytes_removed"`
|
||||
RawSegmentsArchived int `json:"raw_segments_archived"`
|
||||
RawBytesArchived int64 `json:"raw_bytes_archived"`
|
||||
ProjectedObservationsRemoved int64 `json:"projected_observations_removed"`
|
||||
MetricRollupsRemoved int64 `json:"metric_rollups_removed"`
|
||||
LogRollupsRemoved int64 `json:"log_rollups_removed"`
|
||||
ResolvedIncidentsRemoved int64 `json:"resolved_incidents_removed"`
|
||||
PolicyEventsRemoved int64 `json:"policy_events_removed"`
|
||||
}
|
||||
|
||||
func migrateControlRetention(db *sql.DB) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin retention migration: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var segmentTable int
|
||||
if err = tx.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='segments'`).Scan(&segmentTable); err != nil {
|
||||
return fmt.Errorf("inspect segment schema: %w", err)
|
||||
}
|
||||
if segmentTable == 0 {
|
||||
if _, err = tx.Exec(`CREATE TABLE segments (
|
||||
digest TEXT PRIMARY KEY,
|
||||
organization_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
stream_id TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
compressed_bytes INTEGER NOT NULL,
|
||||
uncompressed_bytes INTEGER NOT NULL,
|
||||
committed_at TEXT NOT NULL,
|
||||
projected_at TEXT,
|
||||
signal TEXT NOT NULL DEFAULT '',
|
||||
first_observed_at TEXT NOT NULL DEFAULT '',
|
||||
last_observed_at TEXT NOT NULL DEFAULT '',
|
||||
tier TEXT NOT NULL DEFAULT 'hot' CHECK(tier IN ('hot','cold')),
|
||||
archiving_at TEXT,
|
||||
archive_path TEXT,
|
||||
cold_at TEXT,
|
||||
retiring_at TEXT,
|
||||
UNIQUE(source_id,stream_id,sequence)
|
||||
)`); err != nil {
|
||||
return fmt.Errorf("create retained segment schema: %w", err)
|
||||
}
|
||||
} else {
|
||||
columns, columnErr := sqliteColumns(tx, "segments")
|
||||
if columnErr != nil {
|
||||
return columnErr
|
||||
}
|
||||
for _, column := range []struct{ name, definition string }{
|
||||
{"signal", `TEXT NOT NULL DEFAULT ''`},
|
||||
{"first_observed_at", `TEXT NOT NULL DEFAULT ''`},
|
||||
{"last_observed_at", `TEXT NOT NULL DEFAULT ''`},
|
||||
{"tier", `TEXT NOT NULL DEFAULT 'hot' CHECK(tier IN ('hot','cold'))`},
|
||||
{"archiving_at", `TEXT`},
|
||||
{"archive_path", `TEXT`},
|
||||
{"cold_at", `TEXT`},
|
||||
{"retiring_at", `TEXT`},
|
||||
} {
|
||||
if columns[column.name] {
|
||||
continue
|
||||
}
|
||||
if _, err = tx.Exec(`ALTER TABLE segments ADD COLUMN ` + column.name + ` ` + column.definition); err != nil {
|
||||
return fmt.Errorf("add segment retention column: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, statement := range []string{
|
||||
`CREATE INDEX IF NOT EXISTS segments_retention ON segments(organization_id,tier,signal,last_observed_at,retiring_at)`,
|
||||
`CREATE INDEX IF NOT EXISTS segments_archiving ON segments(archiving_at,organization_id)`,
|
||||
`CREATE TABLE IF NOT EXISTS organization_retention_policies (
|
||||
organization_id TEXT PRIMARY KEY,
|
||||
raw_logs_days INTEGER NOT NULL CHECK(raw_logs_days BETWEEN 1 AND 3650),
|
||||
raw_traces_days INTEGER NOT NULL CHECK(raw_traces_days BETWEEN 1 AND 3650),
|
||||
raw_metrics_days INTEGER NOT NULL CHECK(raw_metrics_days BETWEEN 1 AND 3650),
|
||||
cold_raw_days INTEGER NOT NULL CHECK(cold_raw_days BETWEEN 1 AND 3650),
|
||||
delete_cold_raw INTEGER NOT NULL DEFAULT 0 CHECK(delete_cold_raw IN (0,1)),
|
||||
metric_rollups_days INTEGER NOT NULL CHECK(metric_rollups_days BETWEEN 1 AND 3650),
|
||||
evidence_days INTEGER NOT NULL CHECK(evidence_days BETWEEN 1 AND 3650),
|
||||
quota_bytes INTEGER CHECK(quota_bytes IS NULL OR quota_bytes > 0),
|
||||
extension_approved_by TEXT,
|
||||
updated_by TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS retention_policy_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
organization_id TEXT NOT NULL,
|
||||
actor_user_id TEXT NOT NULL,
|
||||
action TEXT NOT NULL CHECK(action IN ('created','updated')),
|
||||
summary TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS retention_policy_events_age ON retention_policy_events(organization_id,created_at)`,
|
||||
`UPDATE schema_version SET version=7 WHERE version=6`,
|
||||
} {
|
||||
if _, err = tx.Exec(statement); err != nil {
|
||||
return fmt.Errorf("migrate retention schema: %w", err)
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit retention migration: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateControlForensicRetention(db *sql.DB) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin forensic retention migration: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
columns, err := sqliteColumns(tx, "organization_retention_policies")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !columns["delete_cold_raw"] {
|
||||
if _, err = tx.Exec(`ALTER TABLE organization_retention_policies ADD COLUMN delete_cold_raw INTEGER NOT NULL DEFAULT 0 CHECK(delete_cold_raw IN (0,1))`); err != nil {
|
||||
return fmt.Errorf("add forensic retention policy: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(`UPDATE schema_version SET version=8 WHERE version=7`); err != nil {
|
||||
return fmt.Errorf("advance forensic retention schema: %w", err)
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit forensic retention migration: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type columnQuery interface {
|
||||
Query(string, ...any) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
func sqliteColumns(db columnQuery, table string) (map[string]bool, error) {
|
||||
rows, err := db.Query(`PRAGMA table_info(` + table + `)`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect SQLite columns: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
columns := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var index, notNull, primaryKey int
|
||||
var name, valueType string
|
||||
var defaultValue any
|
||||
if err = rows.Scan(&index, &name, &valueType, ¬Null, &defaultValue, &primaryKey); err != nil {
|
||||
return nil, fmt.Errorf("read SQLite columns: %w", err)
|
||||
}
|
||||
columns[name] = true
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read SQLite columns: %w", err)
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func retentionExtended(policy, defaults RetentionPolicy) bool {
|
||||
return policy.RawLogsDays > defaults.RawLogsDays ||
|
||||
policy.RawTracesDays > defaults.RawTracesDays ||
|
||||
policy.RawMetricsDays > defaults.RawMetricsDays ||
|
||||
(policy.DeleteColdRaw && defaults.DeleteColdRaw && policy.ColdRawDays > defaults.ColdRawDays) ||
|
||||
(!policy.DeleteColdRaw && defaults.DeleteColdRaw) ||
|
||||
policy.MetricRollupsDays > defaults.MetricRollupsDays ||
|
||||
policy.EvidenceDays > defaults.EvidenceDays
|
||||
}
|
||||
|
||||
// SetOrganizationRetention records a policy selected by an organization
|
||||
// owner. Extending a server default additionally requires an exact approval
|
||||
// string and a quota larger than current organization storage.
|
||||
func (s *Store) SetOrganizationRetention(ctx context.Context, input SetRetentionInput, now time.Time) (OrganizationRetention, error) {
|
||||
if err := model.ValidateSourceID(input.OrganizationID); err != nil || model.ValidateSourceID(input.ActorUserID) != nil || input.Policy.Validate() != nil || input.Defaults.Validate() != nil || now.IsZero() {
|
||||
return OrganizationRetention{}, errors.New("retention policy input is invalid")
|
||||
}
|
||||
extended := retentionExtended(input.Policy, input.Defaults)
|
||||
if extended {
|
||||
if input.ApproveExtensionFor != input.OrganizationID || input.QuotaBytes <= 0 {
|
||||
return OrganizationRetention{}, errors.New("retention extension requires exact organization approval and a positive quota")
|
||||
}
|
||||
used, err := s.organizationStorageBytes(input.OrganizationID)
|
||||
if err != nil {
|
||||
return OrganizationRetention{}, err
|
||||
}
|
||||
if used >= input.QuotaBytes {
|
||||
return OrganizationRetention{}, errors.New("organization storage already exceeds the approved quota")
|
||||
}
|
||||
} else if input.QuotaBytes != 0 || input.ApproveExtensionFor != "" {
|
||||
return OrganizationRetention{}, errors.New("retention extension approval is not applicable")
|
||||
}
|
||||
tx, err := s.control.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return OrganizationRetention{}, errors.New("begin retention policy update")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var existed int
|
||||
if err = tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM organization_retention_policies WHERE organization_id=?)`, input.OrganizationID).Scan(&existed); err != nil {
|
||||
return OrganizationRetention{}, errors.New("inspect retention policy")
|
||||
}
|
||||
var quota any
|
||||
var approvedBy any
|
||||
if extended {
|
||||
quota = input.QuotaBytes
|
||||
approvedBy = input.ActorUserID
|
||||
}
|
||||
timestamp := now.UTC().Format(time.RFC3339Nano)
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO organization_retention_policies(organization_id,raw_logs_days,raw_traces_days,raw_metrics_days,cold_raw_days,delete_cold_raw,metric_rollups_days,evidence_days,quota_bytes,extension_approved_by,updated_by,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(organization_id) DO UPDATE SET raw_logs_days=excluded.raw_logs_days,raw_traces_days=excluded.raw_traces_days,raw_metrics_days=excluded.raw_metrics_days,cold_raw_days=excluded.cold_raw_days,delete_cold_raw=excluded.delete_cold_raw,metric_rollups_days=excluded.metric_rollups_days,evidence_days=excluded.evidence_days,quota_bytes=excluded.quota_bytes,extension_approved_by=excluded.extension_approved_by,updated_by=excluded.updated_by,updated_at=excluded.updated_at`, input.OrganizationID, input.Policy.RawLogsDays, input.Policy.RawTracesDays, input.Policy.RawMetricsDays, input.Policy.ColdRawDays, input.Policy.DeleteColdRaw, input.Policy.MetricRollupsDays, input.Policy.EvidenceDays, quota, approvedBy, input.ActorUserID, timestamp)
|
||||
if err != nil {
|
||||
return OrganizationRetention{}, errors.New("store retention policy")
|
||||
}
|
||||
action := "created"
|
||||
if existed == 1 {
|
||||
action = "updated"
|
||||
}
|
||||
summary := fmt.Sprintf("Organization retention policy changed; delete_cold_raw=%t", input.Policy.DeleteColdRaw)
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO retention_policy_events(organization_id,actor_user_id,action,summary,created_at) VALUES(?,?,?,?,?)`, input.OrganizationID, input.ActorUserID, action, summary, timestamp); err != nil {
|
||||
return OrganizationRetention{}, errors.New("record retention policy event")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return OrganizationRetention{}, errors.New("commit retention policy update")
|
||||
}
|
||||
return OrganizationRetention{OrganizationID: input.OrganizationID, Policy: input.Policy, QuotaBytes: input.QuotaBytes, ExtensionApproved: extended, ExtensionApprovedBy: func() string {
|
||||
if extended {
|
||||
return input.ActorUserID
|
||||
}
|
||||
return ""
|
||||
}(), UpdatedBy: input.ActorUserID, UpdatedAt: now.UTC()}, nil
|
||||
}
|
||||
|
||||
func (s *Store) organizationStorageBytes(organizationID string) (int64, error) {
|
||||
if err := model.ValidateSourceID(organizationID); err != nil {
|
||||
return 0, errors.New("invalid organization identifier")
|
||||
}
|
||||
var raw int64
|
||||
if err := s.control.QueryRow(`SELECT COALESCE(SUM(compressed_bytes),0) FROM segments WHERE organization_id=?`, organizationID).Scan(&raw); err != nil {
|
||||
return 0, errors.New("measure organization raw storage")
|
||||
}
|
||||
projected, err := s.EstimateOrganizationBytes(organizationID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if raw > int64(^uint64(0)>>1)-projected {
|
||||
return 0, errors.New("organization storage size overflow")
|
||||
}
|
||||
return raw + projected, nil
|
||||
}
|
||||
|
||||
func (s *Store) checkCommittedQuota(ctx context.Context, organizationID string, committed segment.Committed) error {
|
||||
var recorded int
|
||||
if err := s.control.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM segments WHERE digest=?)`, committed.Digest).Scan(&recorded); err != nil {
|
||||
return errors.New("inspect committed segment quota state")
|
||||
}
|
||||
if recorded == 1 {
|
||||
return nil
|
||||
}
|
||||
var quota sql.NullInt64
|
||||
err := s.control.QueryRowContext(ctx, `SELECT quota_bytes FROM organization_retention_policies WHERE organization_id=?`, organizationID).Scan("a)
|
||||
if errors.Is(err, sql.ErrNoRows) || !quota.Valid {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.New("read organization storage quota")
|
||||
}
|
||||
used, err := s.organizationStorageBytes(organizationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
addition := committed.Compressed + committed.Uncompressed
|
||||
if addition < 0 || used > quota.Int64-addition {
|
||||
return ErrOrganizationStorageQuotaExceeded
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// admitCommitted serializes quota admission across every source belonging to
|
||||
// one organization. The raw object is already durable at this point; a quota
|
||||
// rejection removes that exact checksummed object before returning. The
|
||||
// segment catalog row and stream watermark are committed together before an
|
||||
// acknowledgement can be returned. Projection is intentionally independent.
|
||||
func (s *Store) admitCommitted(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, committedAt time.Time) error {
|
||||
return s.admitCommittedEnvelope(ctx, scope, batch, committed, nil, committedAt)
|
||||
}
|
||||
|
||||
func (s *Store) admitCommittedEnvelope(ctx context.Context, scope model.Scope, batch model.Batch, committed segment.Committed, envelope *model.BatchEnvelope, committedAt time.Time) error {
|
||||
lock := s.namedLock("quota:" + scope.OrganizationID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
if err := s.checkCommittedQuota(ctx, scope.OrganizationID, committed); err != nil {
|
||||
if deleteErr := s.segments.Delete(committed.Path, committed.Digest); deleteErr != nil {
|
||||
return fmt.Errorf("%w; remove rejected segment: %v", err, deleteErr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return s.recordCommittedAtEnvelope(ctx, scope, batch, committed, envelope, committedAt)
|
||||
}
|
||||
|
||||
func (s *Store) effectiveRetention(ctx context.Context, organizationID string, defaults RetentionPolicy) (RetentionPolicy, error) {
|
||||
if err := defaults.Validate(); err != nil {
|
||||
return RetentionPolicy{}, err
|
||||
}
|
||||
policy := defaults
|
||||
err := s.control.QueryRowContext(ctx, `SELECT raw_logs_days,raw_traces_days,raw_metrics_days,cold_raw_days,delete_cold_raw,metric_rollups_days,evidence_days FROM organization_retention_policies WHERE organization_id=?`, organizationID).Scan(&policy.RawLogsDays, &policy.RawTracesDays, &policy.RawMetricsDays, &policy.ColdRawDays, &policy.DeleteColdRaw, &policy.MetricRollupsDays, &policy.EvidenceDays)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return defaults, nil
|
||||
}
|
||||
if err != nil || policy.Validate() != nil {
|
||||
return RetentionPolicy{}, errors.New("organization retention policy is invalid")
|
||||
}
|
||||
return policy, nil
|
||||
}
|
||||
|
||||
// ApplyRetention materializes five-minute metric rollups before removing
|
||||
// expired raw projections, then retires only raw segments whose newest record
|
||||
// is outside the applicable window. Segment retirement is crash-recoverable.
|
||||
func (s *Store) ApplyRetention(ctx context.Context, defaults RetentionPolicy, now time.Time) (RetentionReport, error) {
|
||||
if defaults.Validate() != nil || now.IsZero() {
|
||||
return RetentionReport{}, errors.New("retention run input is invalid")
|
||||
}
|
||||
report := RetentionReport{Version: 1, StartedAt: now.UTC()}
|
||||
organizations, err := s.retentionOrganizations(ctx)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
for _, organizationID := range organizations {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
policy, policyErr := s.effectiveRetention(ctx, organizationID, defaults)
|
||||
if policyErr != nil {
|
||||
return report, policyErr
|
||||
}
|
||||
lock := s.namedLock("organization:" + organizationID)
|
||||
lock.Lock()
|
||||
organizationReport, applyErr := s.applyOrganizationRetention(ctx, organizationID, policy, now.UTC())
|
||||
lock.Unlock()
|
||||
if applyErr != nil {
|
||||
return report, applyErr
|
||||
}
|
||||
report.Organizations++
|
||||
report.RawSegmentsRemoved += organizationReport.RawSegmentsRemoved
|
||||
report.RawBytesRemoved += organizationReport.RawBytesRemoved
|
||||
report.RawSegmentsArchived += organizationReport.RawSegmentsArchived
|
||||
report.RawBytesArchived += organizationReport.RawBytesArchived
|
||||
report.ProjectedObservationsRemoved += organizationReport.ProjectedObservationsRemoved
|
||||
report.MetricRollupsRemoved += organizationReport.MetricRollupsRemoved
|
||||
report.LogRollupsRemoved += organizationReport.LogRollupsRemoved
|
||||
report.ResolvedIncidentsRemoved += organizationReport.ResolvedIncidentsRemoved
|
||||
report.PolicyEventsRemoved += organizationReport.PolicyEventsRemoved
|
||||
}
|
||||
report.CompletedAt = time.Now().UTC()
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (s *Store) retentionOrganizations(ctx context.Context) ([]string, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT organization_id FROM sources UNION SELECT organization_id FROM segments UNION SELECT organization_id FROM organization_retention_policies ORDER BY organization_id`)
|
||||
if err != nil {
|
||||
return nil, errors.New("list retention organizations")
|
||||
}
|
||||
defer rows.Close()
|
||||
var organizations []string
|
||||
for rows.Next() {
|
||||
var organizationID string
|
||||
if err = rows.Scan(&organizationID); err != nil || model.ValidateSourceID(organizationID) != nil {
|
||||
return nil, errors.New("retention organization is invalid")
|
||||
}
|
||||
organizations = append(organizations, organizationID)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, errors.New("list retention organizations")
|
||||
}
|
||||
return organizations, nil
|
||||
}
|
||||
|
||||
func retentionCutoff(now time.Time, days int) string {
|
||||
return now.Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
func (s *Store) applyOrganizationRetention(ctx context.Context, organizationID string, policy RetentionPolicy, now time.Time) (RetentionReport, error) {
|
||||
report := RetentionReport{Version: 1}
|
||||
path := s.organizationProjectionPath(organizationID)
|
||||
if info, err := os.Lstat(path); err == nil {
|
||||
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return report, errors.New("organization projection is unavailable")
|
||||
}
|
||||
db, openErr := openProjection(ctx, path)
|
||||
if openErr != nil {
|
||||
return report, openErr
|
||||
}
|
||||
tx, beginErr := db.BeginTx(ctx, nil)
|
||||
if beginErr != nil {
|
||||
_ = db.Close()
|
||||
return report, errors.New("begin organization retention")
|
||||
}
|
||||
for _, expiration := range []struct {
|
||||
signal model.Signal
|
||||
days int
|
||||
}{
|
||||
{model.SignalLogs, policy.RawLogsDays},
|
||||
{model.SignalTraces, policy.RawTracesDays},
|
||||
{model.SignalMetrics, policy.RawMetricsDays},
|
||||
{model.SignalDeployments, policy.EvidenceDays},
|
||||
} {
|
||||
result, deleteErr := tx.ExecContext(ctx, `DELETE FROM observations WHERE organization_id=? AND signal=? AND timestamp<?`, organizationID, expiration.signal, retentionCutoff(now, expiration.days))
|
||||
if deleteErr != nil {
|
||||
_ = tx.Rollback()
|
||||
_ = db.Close()
|
||||
return report, errors.New("remove expired observation projections")
|
||||
}
|
||||
removed, _ := result.RowsAffected()
|
||||
report.ProjectedObservationsRemoved += removed
|
||||
}
|
||||
result, deleteErr := tx.ExecContext(ctx, `DELETE FROM metric_rollups_5m WHERE organization_id=? AND bucket_start<?`, organizationID, retentionCutoff(now, policy.MetricRollupsDays))
|
||||
if deleteErr != nil {
|
||||
_ = tx.Rollback()
|
||||
_ = db.Close()
|
||||
return report, errors.New("remove expired metric rollups")
|
||||
}
|
||||
report.MetricRollupsRemoved, _ = result.RowsAffected()
|
||||
logCutoff := now.Add(-time.Duration(policy.RawLogsDays) * 24 * time.Hour).UTC().Truncate(logRollupWindow).Unix()
|
||||
result, deleteErr = tx.ExecContext(ctx, `DELETE FROM log_status_route_rollups_5m WHERE organization_id=? AND bucket_start<?`, organizationID, logCutoff)
|
||||
if deleteErr != nil {
|
||||
_ = tx.Rollback()
|
||||
_ = db.Close()
|
||||
return report, errors.New("remove expired log rollups")
|
||||
}
|
||||
report.LogRollupsRemoved, _ = result.RowsAffected()
|
||||
if err = tx.Commit(); err != nil {
|
||||
_ = db.Close()
|
||||
return report, errors.New("commit organization retention")
|
||||
}
|
||||
if report.ProjectedObservationsRemoved > 0 || report.MetricRollupsRemoved > 0 || report.LogRollupsRemoved > 0 {
|
||||
if _, err = db.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil {
|
||||
_ = db.Close()
|
||||
return report, errors.New("checkpoint retained organization projection")
|
||||
}
|
||||
if _, err = db.ExecContext(ctx, `VACUUM`); err != nil {
|
||||
_ = db.Close()
|
||||
return report, errors.New("compact retained organization projection")
|
||||
}
|
||||
}
|
||||
if err = db.Close(); err != nil {
|
||||
return report, errors.New("close retained organization projection")
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return report, errors.New("inspect organization projection for retention")
|
||||
}
|
||||
|
||||
evidenceCutoff := retentionCutoff(now, policy.EvidenceDays)
|
||||
result, err := s.control.ExecContext(ctx, `DELETE FROM incidents WHERE organization_id=? AND state='resolved' AND updated_at<?`, organizationID, evidenceCutoff)
|
||||
if err != nil {
|
||||
return report, errors.New("remove expired resolved incidents")
|
||||
}
|
||||
report.ResolvedIncidentsRemoved, _ = result.RowsAffected()
|
||||
result, err = s.control.ExecContext(ctx, `DELETE FROM retention_policy_events WHERE organization_id=? AND created_at<?`, organizationID, evidenceCutoff)
|
||||
if err != nil {
|
||||
return report, errors.New("remove expired retention policy events")
|
||||
}
|
||||
report.PolicyEventsRemoved, _ = result.RowsAffected()
|
||||
for _, expiration := range []struct {
|
||||
signal model.Signal
|
||||
days int
|
||||
}{
|
||||
{model.SignalLogs, policy.RawLogsDays},
|
||||
{model.SignalTraces, policy.RawTracesDays},
|
||||
{model.SignalMetrics, policy.RawMetricsDays},
|
||||
{model.SignalDeployments, policy.EvidenceDays},
|
||||
} {
|
||||
archived, bytes, archiveErr := s.archiveHotSegments(ctx, organizationID, expiration.signal, retentionCutoff(now, expiration.days), now)
|
||||
if archiveErr != nil {
|
||||
return report, archiveErr
|
||||
}
|
||||
report.RawSegmentsArchived += archived
|
||||
report.RawBytesArchived += bytes
|
||||
}
|
||||
if policy.DeleteColdRaw {
|
||||
if _, err = s.control.ExecContext(ctx, `UPDATE segments SET retiring_at=? WHERE organization_id=? AND tier='cold' AND projected_at IS NOT NULL AND retiring_at IS NULL AND last_observed_at<?`, now.Format(time.RFC3339Nano), organizationID, retentionCutoff(now, policy.ColdRawDays)); err != nil {
|
||||
return report, errors.New("mark expired cold segments")
|
||||
}
|
||||
}
|
||||
removed, bytes, err := s.finalizeRetiring(ctx, organizationID)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.RawSegmentsRemoved, report.RawBytesRemoved = removed, bytes
|
||||
return report, nil
|
||||
}
|
||||
|
||||
type archivingSegment struct {
|
||||
digest, path, archivePath, sourceID, streamID string
|
||||
signal model.Signal
|
||||
bytes int64
|
||||
}
|
||||
|
||||
func (s *Store) archiveHotSegments(ctx context.Context, organizationID string, signal model.Signal, cutoff string, now time.Time) (int, int64, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT digest,path,source_id,stream_id,compressed_bytes FROM segments WHERE organization_id=? AND signal=? AND tier='hot' AND projected_at IS NOT NULL AND retiring_at IS NULL AND archiving_at IS NULL AND last_observed_at<? ORDER BY last_observed_at,digest`, organizationID, signal, cutoff)
|
||||
if err != nil {
|
||||
return 0, 0, errors.New("list hot segments for archival")
|
||||
}
|
||||
var pending []archivingSegment
|
||||
for rows.Next() {
|
||||
var candidate archivingSegment
|
||||
candidate.signal = signal
|
||||
if err = rows.Scan(&candidate.digest, &candidate.path, &candidate.sourceID, &candidate.streamID, &candidate.bytes); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, 0, errors.New("read hot segment for archival")
|
||||
}
|
||||
candidate.archivePath, err = s.coldArchivePath(organizationID, candidate)
|
||||
if err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, 0, err
|
||||
}
|
||||
pending = append(pending, candidate)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return 0, 0, errors.New("close hot segments for archival")
|
||||
}
|
||||
stamp := now.UTC().Format(time.RFC3339Nano)
|
||||
for _, candidate := range pending {
|
||||
result, updateErr := s.control.ExecContext(ctx, `UPDATE segments SET archiving_at=?,archive_path=? WHERE digest=? AND organization_id=? AND tier='hot' AND archiving_at IS NULL AND retiring_at IS NULL`, stamp, candidate.archivePath, candidate.digest, organizationID)
|
||||
if updateErr != nil {
|
||||
return 0, 0, errors.New("mark hot segment for archival")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return 0, 0, errors.New("hot segment archival state changed")
|
||||
}
|
||||
}
|
||||
return s.finalizeArchiving(ctx, organizationID)
|
||||
}
|
||||
|
||||
func (s *Store) coldArchivePath(organizationID string, candidate archivingSegment) (string, error) {
|
||||
if model.ValidateSourceID(organizationID) != nil || model.ValidateSourceID(candidate.sourceID) != nil || model.ValidateStreamID(candidate.streamID) != nil {
|
||||
return "", errors.New("cold archive identity is invalid")
|
||||
}
|
||||
switch candidate.signal {
|
||||
case model.SignalLogs, model.SignalMetrics, model.SignalTraces, model.SignalDeployments:
|
||||
default:
|
||||
return "", errors.New("cold archive signal is invalid")
|
||||
}
|
||||
name := filepath.Base(filepath.Clean(candidate.path))
|
||||
if name == "." || name == string(os.PathSeparator) || !strings.HasSuffix(name, "-"+candidate.digest+".zst") {
|
||||
return "", errors.New("cold archive segment name is invalid")
|
||||
}
|
||||
return filepath.Join(s.root, "cold", organizationID, string(candidate.signal), candidate.sourceID, candidate.streamID, name), nil
|
||||
}
|
||||
|
||||
func (s *Store) finalizeArchiving(ctx context.Context, organizationID string) (int, int64, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT digest,path,archive_path,source_id,stream_id,signal,compressed_bytes FROM segments WHERE organization_id=? AND archiving_at IS NOT NULL ORDER BY archiving_at,digest`, organizationID)
|
||||
if err != nil {
|
||||
return 0, 0, errors.New("list interrupted cold archives")
|
||||
}
|
||||
var pending []archivingSegment
|
||||
for rows.Next() {
|
||||
var candidate archivingSegment
|
||||
if err = rows.Scan(&candidate.digest, &candidate.path, &candidate.archivePath, &candidate.sourceID, &candidate.streamID, &candidate.signal, &candidate.bytes); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, 0, errors.New("read interrupted cold archive")
|
||||
}
|
||||
pending = append(pending, candidate)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return 0, 0, errors.New("close interrupted cold archives")
|
||||
}
|
||||
archived := 0
|
||||
var archivedBytes int64
|
||||
for _, candidate := range pending {
|
||||
expected, pathErr := s.coldArchivePath(organizationID, candidate)
|
||||
if pathErr != nil || expected != candidate.archivePath {
|
||||
return archived, archivedBytes, errors.New("cold archive target is invalid")
|
||||
}
|
||||
if err = s.segments.MoveToCold(candidate.path, candidate.archivePath, candidate.digest); err != nil {
|
||||
return archived, archivedBytes, err
|
||||
}
|
||||
result, updateErr := s.control.ExecContext(ctx, `UPDATE segments SET path=archive_path,tier='cold',cold_at=archiving_at,archiving_at=NULL,archive_path=NULL WHERE digest=? AND organization_id=? AND tier='hot' AND archiving_at IS NOT NULL AND archive_path=?`, candidate.digest, organizationID, candidate.archivePath)
|
||||
if updateErr != nil {
|
||||
return archived, archivedBytes, errors.New("complete cold segment archival")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return archived, archivedBytes, errors.New("cold segment archival state changed")
|
||||
}
|
||||
archived++
|
||||
if candidate.bytes > math.MaxInt64-archivedBytes {
|
||||
return archived, archivedBytes, errors.New("cold archive byte count overflow")
|
||||
}
|
||||
archivedBytes += candidate.bytes
|
||||
removeEmptyPrivateDirectory(filepath.Dir(candidate.path))
|
||||
removeEmptyPrivateDirectory(filepath.Dir(filepath.Dir(candidate.path)))
|
||||
}
|
||||
return archived, archivedBytes, nil
|
||||
}
|
||||
|
||||
func (s *Store) finalizeRetiring(ctx context.Context, organizationID string) (int, int64, error) {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT digest,path,compressed_bytes FROM segments WHERE organization_id=? AND retiring_at IS NOT NULL ORDER BY retiring_at,digest`, organizationID)
|
||||
if err != nil {
|
||||
return 0, 0, errors.New("list retiring raw segments")
|
||||
}
|
||||
type retiringSegment struct {
|
||||
digest, path string
|
||||
bytes int64
|
||||
}
|
||||
var pending []retiringSegment
|
||||
for rows.Next() {
|
||||
var segment retiringSegment
|
||||
if err = rows.Scan(&segment.digest, &segment.path, &segment.bytes); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, 0, errors.New("read retiring raw segment")
|
||||
}
|
||||
pending = append(pending, segment)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return 0, 0, errors.New("close retiring raw segments")
|
||||
}
|
||||
path := s.organizationProjectionPath(organizationID)
|
||||
var projection *sql.DB
|
||||
if _, err = os.Lstat(path); err == nil {
|
||||
projection, err = openProjection(ctx, path)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defer projection.Close()
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return 0, 0, errors.New("inspect retiring segment projection")
|
||||
}
|
||||
removed := 0
|
||||
var removedBytes int64
|
||||
for _, segment := range pending {
|
||||
if projection != nil {
|
||||
tx, beginErr := projection.BeginTx(ctx, nil)
|
||||
if beginErr != nil {
|
||||
return removed, removedBytes, errors.New("begin retiring segment projection")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM observations WHERE organization_id=? AND segment_digest=?`, organizationID, segment.digest); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("remove retiring segment projection")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM metric_rollup_segments WHERE segment_digest=?`, segment.digest); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("remove retiring segment rollup ledger")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM log_rollup_segments WHERE segment_digest=?`, segment.digest); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("remove retiring segment log rollup ledger")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return removed, removedBytes, errors.New("commit retiring segment projection")
|
||||
}
|
||||
}
|
||||
if err = s.segments.Delete(segment.path, segment.digest); err != nil {
|
||||
return removed, removedBytes, err
|
||||
}
|
||||
tx, beginErr := s.control.BeginTx(ctx, nil)
|
||||
if beginErr != nil {
|
||||
return removed, removedBytes, errors.New("begin retiring segment acknowledgement")
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `DELETE FROM descriptor_proposal_segments WHERE segment_digest=?`, segment.digest); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("remove retired segment proposal evidence")
|
||||
}
|
||||
result, deleteErr := tx.ExecContext(ctx, `DELETE FROM segments WHERE digest=? AND organization_id=? AND retiring_at IS NOT NULL`, segment.digest, organizationID)
|
||||
if deleteErr != nil {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("acknowledge retired segment")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
_ = tx.Rollback()
|
||||
return removed, removedBytes, errors.New("retired segment state changed")
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return removed, removedBytes, errors.New("commit retired segment acknowledgement")
|
||||
}
|
||||
removed++
|
||||
removedBytes += segment.bytes
|
||||
removeEmptyPrivateDirectory(filepath.Dir(segment.path))
|
||||
removeEmptyPrivateDirectory(filepath.Dir(filepath.Dir(segment.path)))
|
||||
}
|
||||
return removed, removedBytes, nil
|
||||
}
|
||||
|
||||
func (s *Store) finishInterruptedRetention(ctx context.Context) error {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT DISTINCT organization_id FROM segments WHERE retiring_at IS NOT NULL ORDER BY organization_id`)
|
||||
if err != nil {
|
||||
return errors.New("list interrupted retention organizations")
|
||||
}
|
||||
var organizations []string
|
||||
for rows.Next() {
|
||||
var organizationID string
|
||||
if err = rows.Scan(&organizationID); err != nil {
|
||||
_ = rows.Close()
|
||||
return errors.New("read interrupted retention organization")
|
||||
}
|
||||
organizations = append(organizations, organizationID)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return errors.New("close interrupted retention organizations")
|
||||
}
|
||||
for _, organizationID := range organizations {
|
||||
lock := s.namedLock("organization:" + organizationID)
|
||||
lock.Lock()
|
||||
_, _, finalizeErr := s.finalizeRetiring(ctx, organizationID)
|
||||
lock.Unlock()
|
||||
if finalizeErr != nil {
|
||||
return finalizeErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) finishInterruptedArchival(ctx context.Context) error {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT DISTINCT organization_id FROM segments WHERE archiving_at IS NOT NULL ORDER BY organization_id`)
|
||||
if err != nil {
|
||||
return errors.New("list interrupted archive organizations")
|
||||
}
|
||||
var organizations []string
|
||||
for rows.Next() {
|
||||
var organizationID string
|
||||
if err = rows.Scan(&organizationID); err != nil || model.ValidateSourceID(organizationID) != nil {
|
||||
_ = rows.Close()
|
||||
return errors.New("read interrupted archive organization")
|
||||
}
|
||||
organizations = append(organizations, organizationID)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return errors.New("close interrupted archive organizations")
|
||||
}
|
||||
for _, organizationID := range organizations {
|
||||
lock := s.namedLock("organization:" + organizationID)
|
||||
lock.Lock()
|
||||
_, _, finalizeErr := s.finalizeArchiving(ctx, organizationID)
|
||||
lock.Unlock()
|
||||
if finalizeErr != nil {
|
||||
return finalizeErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) organizationProjectionPath(organizationID string) string {
|
||||
return filepath.Join(s.root, "organizations", organizationID, "projection.sqlite")
|
||||
}
|
||||
|
||||
func (s *Store) backfillSegmentRetentionMetadata(ctx context.Context) error {
|
||||
rows, err := s.control.QueryContext(ctx, `SELECT digest,path FROM segments WHERE signal='' OR first_observed_at='' OR last_observed_at='' ORDER BY committed_at,digest`)
|
||||
if err != nil {
|
||||
return errors.New("list legacy segment metadata")
|
||||
}
|
||||
type legacy struct{ digest, path string }
|
||||
var pending []legacy
|
||||
for rows.Next() {
|
||||
var item legacy
|
||||
if err = rows.Scan(&item.digest, &item.path); err != nil {
|
||||
_ = rows.Close()
|
||||
return errors.New("read legacy segment metadata")
|
||||
}
|
||||
pending = append(pending, item)
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return errors.New("close legacy segment metadata")
|
||||
}
|
||||
for _, item := range pending {
|
||||
batch, readErr := s.segments.Read(item.path, item.digest)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("backfill segment metadata: %w", readErr)
|
||||
}
|
||||
if len(batch.Records) == 0 {
|
||||
return errors.New("backfill segment metadata: segment has no records")
|
||||
}
|
||||
first, last := observationRange(batch)
|
||||
result, updateErr := s.control.ExecContext(ctx, `UPDATE segments SET signal=?,first_observed_at=?,last_observed_at=? WHERE digest=? AND (signal='' OR first_observed_at='' OR last_observed_at='')`, batch.Signal, first.Format(time.RFC3339Nano), last.Format(time.RFC3339Nano), item.digest)
|
||||
if updateErr != nil {
|
||||
return errors.New("backfill segment metadata")
|
||||
}
|
||||
if changed, _ := result.RowsAffected(); changed != 1 {
|
||||
return errors.New("segment metadata changed during backfill")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeEmptyPrivateDirectory(path string) {
|
||||
info, err := os.Lstat(path)
|
||||
if err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user