docs: publish Preview 19 dogfood evidence
Export the reviewed allowlisted snapshot from private source commit 05928cebd01b586cf9e9d4b8c8537a7605a6068c. This records the exact candidate, bounded capacity result, stateful migration scratch requirement, authenticated batch identity proof, and immediate live acceptance evidence. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"math"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
type FieldLookup func(field string) (string, bool)
|
||||
|
||||
// MatchesFilters applies the query's typed filters to a bounded record value
|
||||
// lookup. Storage projections and the local agent evaluator share this path so
|
||||
// edge and central comparisons cannot drift silently.
|
||||
func MatchesFilters(ast AST, registry Registry, lookup FieldLookup) (bool, error) {
|
||||
for _, filter := range ast.Filters {
|
||||
field := CanonicalField(filter.Field)
|
||||
descriptor, _ := ResolveDescriptor(ast.Signal, field, registry)
|
||||
value, present := lookup(field)
|
||||
if !present {
|
||||
return false, nil
|
||||
}
|
||||
matched, err := CompareValue(value, filter.Value, filter.Op, descriptor.Type)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !matched {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func MatchObservation(observation model.Observation, ast AST, registry Registry) (bool, error) {
|
||||
return MatchesFilters(ast, registry, func(field string) (string, bool) {
|
||||
switch CanonicalField(field) {
|
||||
case "timestamp":
|
||||
return observation.Timestamp.UTC().Format(time.RFC3339Nano), !observation.Timestamp.IsZero()
|
||||
case "name":
|
||||
return observation.Name, observation.Name != ""
|
||||
case "severity":
|
||||
return observation.Severity, observation.Severity != ""
|
||||
case "body":
|
||||
return observation.Body, observation.Body != ""
|
||||
case "value":
|
||||
if observation.Value == nil {
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatFloat(*observation.Value, 'g', -1, 64), true
|
||||
case "trace_id":
|
||||
return observation.TraceID, observation.TraceID != ""
|
||||
case "span_id":
|
||||
return observation.SpanID, observation.SpanID != ""
|
||||
case "correlation_id":
|
||||
return observation.CorrelationID, observation.CorrelationID != ""
|
||||
default:
|
||||
value, ok := observation.Attributes[CanonicalField(field)]
|
||||
return value, ok
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func CompareValue(left, right, operator string, valueType schema.Type) (bool, error) {
|
||||
if operator == "=~" {
|
||||
if valueType != schema.TypeString {
|
||||
return false, ErrTypeMismatch
|
||||
}
|
||||
expression, err := regexp.Compile(right)
|
||||
if err != nil {
|
||||
return false, ErrTypeMismatch
|
||||
}
|
||||
return expression.MatchString(left), nil
|
||||
}
|
||||
var comparison int
|
||||
switch valueType {
|
||||
case schema.TypeInteger, schema.TypeFloat, schema.TypeDuration:
|
||||
rightNumber, rightErr := strconv.ParseFloat(right, 64)
|
||||
if rightErr != nil || math.IsNaN(rightNumber) || math.IsInf(rightNumber, 0) {
|
||||
return false, ErrTypeMismatch
|
||||
}
|
||||
leftNumber, leftErr := strconv.ParseFloat(left, 64)
|
||||
if leftErr != nil || math.IsNaN(leftNumber) || math.IsInf(leftNumber, 0) {
|
||||
return false, nil
|
||||
}
|
||||
comparison = compareFloat(leftNumber, rightNumber)
|
||||
case schema.TypeTime:
|
||||
rightTime, rightErr := time.Parse(time.RFC3339Nano, right)
|
||||
if rightErr != nil {
|
||||
return false, ErrTypeMismatch
|
||||
}
|
||||
leftTime, leftErr := time.Parse(time.RFC3339Nano, left)
|
||||
if leftErr != nil {
|
||||
return false, nil
|
||||
}
|
||||
comparison = leftTime.Compare(rightTime)
|
||||
case schema.TypeBoolean:
|
||||
rightBool, rightErr := strconv.ParseBool(right)
|
||||
if rightErr != nil {
|
||||
return false, ErrTypeMismatch
|
||||
}
|
||||
leftBool, leftErr := strconv.ParseBool(left)
|
||||
if leftErr != nil {
|
||||
return false, nil
|
||||
}
|
||||
comparison = compareBool(leftBool, rightBool)
|
||||
default:
|
||||
comparison = strings.Compare(left, right)
|
||||
}
|
||||
switch operator {
|
||||
case "==":
|
||||
return comparison == 0, nil
|
||||
case "!=":
|
||||
return comparison != 0, nil
|
||||
case ">":
|
||||
return comparison > 0, nil
|
||||
case ">=":
|
||||
return comparison >= 0, nil
|
||||
case "<":
|
||||
return comparison < 0, nil
|
||||
case "<=":
|
||||
return comparison <= 0, nil
|
||||
default:
|
||||
return false, ErrTypeMismatch
|
||||
}
|
||||
}
|
||||
|
||||
func compareFloat(left, right float64) int {
|
||||
if left < right {
|
||||
return -1
|
||||
}
|
||||
if left > right {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func compareBool(left, right bool) int {
|
||||
if left == right {
|
||||
return 0
|
||||
}
|
||||
if !left {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
func TestMatchObservationUsesCanonicalTypedFilters(t *testing.T) {
|
||||
now := time.Date(2026, 8, 18, 23, 55, 0, 0, time.UTC)
|
||||
observation := model.Observation{Timestamp: now, Name: "http.request", Severity: "error", Attributes: map[string]string{"http.status_code": "503", "http.route": "/failed"}}
|
||||
ast, err := Parse(`logs | where status >= 500 | where route == "/failed" | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
matched, err := MatchObservation(observation, ast, nil)
|
||||
if err != nil || !matched {
|
||||
t.Fatalf("matched=%t err=%v", matched, err)
|
||||
}
|
||||
observation.Attributes["http.status_code"] = "200"
|
||||
matched, err = MatchObservation(observation, ast, nil)
|
||||
if err != nil || matched {
|
||||
t.Fatalf("matched=%t err=%v", matched, err)
|
||||
}
|
||||
delete(observation.Attributes, "http.status_code")
|
||||
matched, err = MatchObservation(observation, ast, nil)
|
||||
if err != nil || matched {
|
||||
t.Fatalf("missing field matched=%t err=%v", matched, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchObservationRejectsInvalidTypedFilterValue(t *testing.T) {
|
||||
ast, err := Parse(`logs | where status >= nope | limit 10`, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observation := model.Observation{Timestamp: time.Now().UTC(), Name: "http.request", Attributes: map[string]string{"http.status_code": "503"}}
|
||||
if _, err = MatchObservation(observation, ast, nil); err != ErrTypeMismatch {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
var ErrSensitivePermissionRequired = errors.New("query requires sensitive-field permission")
|
||||
|
||||
type Scope struct {
|
||||
OrganizationID string `json:"organization_id"`
|
||||
ProjectID string `json:"project_id,omitempty"`
|
||||
EnvironmentID string `json:"environment_id,omitempty"`
|
||||
ServiceID string `json:"service_id,omitempty"`
|
||||
Sensitive bool `json:"sensitive"`
|
||||
}
|
||||
|
||||
type Budget struct {
|
||||
MaxDuration time.Duration `json:"-"`
|
||||
MaxRows int `json:"max_rows"`
|
||||
MaxScannedBytes int64 `json:"max_scanned_bytes"`
|
||||
MaxMemoryBytes int64 `json:"max_memory_bytes"`
|
||||
}
|
||||
|
||||
type FieldPlan struct {
|
||||
Field string `json:"field"`
|
||||
Descriptor schema.Descriptor `json:"descriptor"`
|
||||
Indexed bool `json:"indexed"`
|
||||
Unknown bool `json:"unknown"`
|
||||
}
|
||||
|
||||
type Explain struct {
|
||||
AST AST `json:"ast"`
|
||||
ProjectedSources []string `json:"projected_sources"`
|
||||
Fields []FieldPlan `json:"fields"`
|
||||
EstimatedScanBytes int64 `json:"estimated_scan_bytes"`
|
||||
CacheEligible bool `json:"cache_eligible"`
|
||||
RequiredPermissions []string `json:"required_permissions"`
|
||||
Budget Budget `json:"budget"`
|
||||
}
|
||||
|
||||
type Registry interface {
|
||||
Lookup(model.Signal, string) (schema.Descriptor, bool)
|
||||
}
|
||||
|
||||
type MapRegistry map[string]schema.Descriptor
|
||||
|
||||
func (registry MapRegistry) Lookup(signal model.Signal, field string) (schema.Descriptor, bool) {
|
||||
descriptor, ok := registry[string(signal)+":"+CanonicalField(field)]
|
||||
return descriptor, ok
|
||||
}
|
||||
|
||||
func Plan(ast AST, scope Scope, registry Registry, estimatedScanBytes int64, budget Budget) (Explain, error) {
|
||||
if err := Validate(ast, budget.MaxRows); err != nil {
|
||||
return Explain{}, err
|
||||
}
|
||||
if !safeScope(scope) {
|
||||
return Explain{}, errors.New("query scope is invalid")
|
||||
}
|
||||
if budget.MaxDuration < time.Millisecond || budget.MaxDuration > time.Minute || budget.MaxRows < 1 || budget.MaxScannedBytes < 1 || budget.MaxMemoryBytes < 1 {
|
||||
return Explain{}, errors.New("query budget is invalid")
|
||||
}
|
||||
if estimatedScanBytes < 0 || estimatedScanBytes > budget.MaxScannedBytes {
|
||||
return Explain{}, errors.New("estimated query scan exceeds budget")
|
||||
}
|
||||
fields := ReferencedFields(ast)
|
||||
plans := make([]FieldPlan, 0, len(fields))
|
||||
requiresSensitive := false
|
||||
cacheEligible := true
|
||||
for _, field := range fields {
|
||||
canonical := CanonicalField(field)
|
||||
descriptor, unknown := ResolveDescriptor(ast.Signal, canonical, registry)
|
||||
if err := descriptor.Validate(); err != nil {
|
||||
return Explain{}, fmt.Errorf("field %s descriptor: %w", field, err)
|
||||
}
|
||||
if descriptor.Sensitivity == schema.SensitivitySensitive {
|
||||
requiresSensitive = true
|
||||
cacheEligible = false
|
||||
}
|
||||
plans = append(plans, FieldPlan{Field: field, Descriptor: descriptor, Indexed: descriptor.Index != schema.IndexNone, Unknown: unknown})
|
||||
}
|
||||
if requiresSensitive && !scope.Sensitive {
|
||||
return Explain{}, ErrSensitivePermissionRequired
|
||||
}
|
||||
for _, filter := range ast.Filters {
|
||||
if filter.Op == "=~" {
|
||||
cacheEligible = false
|
||||
}
|
||||
}
|
||||
permissions := []string{"telemetry:query"}
|
||||
if requiresSensitive {
|
||||
permissions = append(permissions, "telemetry:sensitive")
|
||||
}
|
||||
source := "organization:" + scope.OrganizationID + "/signal:" + string(ast.Signal)
|
||||
if scope.ProjectID != "" {
|
||||
source += "/project:" + scope.ProjectID
|
||||
}
|
||||
if scope.EnvironmentID != "" {
|
||||
source += "/environment:" + scope.EnvironmentID
|
||||
}
|
||||
if scope.ServiceID != "" {
|
||||
source += "/service:" + scope.ServiceID
|
||||
}
|
||||
return Explain{AST: ast, ProjectedSources: []string{source}, Fields: plans, EstimatedScanBytes: estimatedScanBytes, CacheEligible: cacheEligible, RequiredPermissions: permissions, Budget: budget}, nil
|
||||
}
|
||||
|
||||
func ReferencedFields(ast AST) []string {
|
||||
seen := map[string]bool{}
|
||||
var fields []string
|
||||
add := func(field string) {
|
||||
if field != "" && !seen[field] {
|
||||
seen[field] = true
|
||||
fields = append(fields, field)
|
||||
}
|
||||
}
|
||||
for _, filter := range ast.Filters {
|
||||
add(filter.Field)
|
||||
}
|
||||
if ast.Sort != nil {
|
||||
isAggregateAlias := false
|
||||
if ast.Summary != nil {
|
||||
for _, aggregate := range ast.Summary.Aggregates {
|
||||
if aggregate.Alias == ast.Sort.Field {
|
||||
isAggregateAlias = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !isAggregateAlias {
|
||||
add(ast.Sort.Field)
|
||||
}
|
||||
}
|
||||
if ast.Summary != nil {
|
||||
for _, aggregate := range ast.Summary.Aggregates {
|
||||
add(aggregate.Field)
|
||||
}
|
||||
for _, field := range ast.Summary.GroupBy {
|
||||
add(field)
|
||||
}
|
||||
}
|
||||
sort.Strings(fields)
|
||||
return fields
|
||||
}
|
||||
|
||||
func CanonicalField(field string) string {
|
||||
switch field {
|
||||
case "service":
|
||||
return "service.id"
|
||||
case "project":
|
||||
return "project.id"
|
||||
case "environment":
|
||||
return "environment.id"
|
||||
case "route":
|
||||
return "http.route"
|
||||
case "status":
|
||||
return "http.status_code"
|
||||
case "duration":
|
||||
return "duration_ns"
|
||||
default:
|
||||
return field
|
||||
}
|
||||
}
|
||||
|
||||
func ResolveDescriptor(signal model.Signal, field string, registry Registry) (schema.Descriptor, bool) {
|
||||
canonical := CanonicalField(field)
|
||||
descriptor, ok := BuiltinDescriptor(signal, canonical)
|
||||
if !ok && registry != nil {
|
||||
descriptor, ok = registry.Lookup(signal, canonical)
|
||||
}
|
||||
if !ok {
|
||||
return schema.Unknown(signal, canonical), true
|
||||
}
|
||||
return descriptor, false
|
||||
}
|
||||
|
||||
func BuiltinDescriptor(signal model.Signal, field string) (schema.Descriptor, bool) {
|
||||
descriptors := map[string]schema.Descriptor{
|
||||
"service.id": descriptor(signal, "service.id", schema.TypeString, "Application service identifier.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""),
|
||||
"project.id": descriptor(signal, "project.id", schema.TypeString, "Application project identifier.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""),
|
||||
"environment.id": descriptor(signal, "environment.id", schema.TypeString, "Deployment environment identifier.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""),
|
||||
"timestamp": descriptor(signal, "timestamp", schema.TypeTime, "Observation timestamp.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, "s"),
|
||||
"name": descriptor(signal, "name", schema.TypeString, "Observation name.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""),
|
||||
"severity": descriptor(signal, "severity", schema.TypeString, "Log severity.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""),
|
||||
"value": descriptor(signal, "value", schema.TypeFloat, "Numeric observation value.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, ""),
|
||||
"http.route": descriptor(signal, "http.route", schema.TypeString, "Application-normalized HTTP route.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""),
|
||||
"http.status_code": descriptor(signal, "http.status_code", schema.TypeInteger, "HTTP response status code.", schema.SensitivityPublic, schema.CardinalityLow, schema.IndexExact, ""),
|
||||
"duration_ns": descriptor(signal, "duration_ns", schema.TypeDuration, "Observed duration in nanoseconds.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, "ns"),
|
||||
"state": descriptor(signal, "state", schema.TypeString, "Bounded metric state dimension.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""),
|
||||
"period": descriptor(signal, "period", schema.TypeString, "Bounded measurement period dimension.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""),
|
||||
"interface": descriptor(signal, "interface", schema.TypeString, "Configured network interface dimension.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""),
|
||||
"direction": descriptor(signal, "direction", schema.TypeString, "Bounded input or output direction dimension.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""),
|
||||
"filesystem": descriptor(signal, "filesystem", schema.TypeString, "Configured filesystem dimension.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""),
|
||||
"process": descriptor(signal, "process", schema.TypeString, "Configured process dimension.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""),
|
||||
"cgroup": descriptor(signal, "cgroup", schema.TypeString, "Configured cgroup dimension.", schema.SensitivityInternal, schema.CardinalityMedium, schema.IndexExact, ""),
|
||||
"unit": descriptor(signal, "unit", schema.TypeString, "Metric unit supplied by a bounded collector.", schema.SensitivityInternal, schema.CardinalityLow, schema.IndexExact, ""),
|
||||
"trace_id": descriptor(signal, "trace_id", schema.TypeString, "Trace correlation identifier.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, ""),
|
||||
"span_id": descriptor(signal, "span_id", schema.TypeString, "Span correlation identifier.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, ""),
|
||||
"correlation_id": descriptor(signal, "correlation_id", schema.TypeString, "Cross-signal correlation identifier.", schema.SensitivityInternal, schema.CardinalityHigh, schema.IndexRange, ""),
|
||||
"body": descriptor(signal, "body", schema.TypeString, "Optional retained log text.", schema.SensitivitySensitive, schema.CardinalityHigh, schema.IndexNone, ""),
|
||||
}
|
||||
descriptor, ok := descriptors[field]
|
||||
if ok && signal != model.SignalMetrics {
|
||||
switch field {
|
||||
case "state", "period", "interface", "direction", "filesystem", "process", "cgroup", "unit":
|
||||
return schema.Descriptor{}, false
|
||||
}
|
||||
}
|
||||
if ok && signal == model.SignalMetrics {
|
||||
switch field {
|
||||
case "http.route", "http.status_code", "state", "period", "interface", "direction", "filesystem", "process", "cgroup", "unit":
|
||||
descriptor.Retention = schema.RetentionMetric
|
||||
}
|
||||
}
|
||||
return descriptor, ok
|
||||
}
|
||||
|
||||
func descriptor(signal model.Signal, field string, valueType schema.Type, meaning string, sensitivity schema.Sensitivity, cardinality schema.Cardinality, index schema.IndexPolicy, unit string) schema.Descriptor {
|
||||
return schema.Descriptor{Version: schema.DescriptorVersion, Signal: signal, Field: field, Type: valueType, Unit: unit, Meaning: meaning, Sensitivity: sensitivity, Cardinality: cardinality, Index: index, Retention: schema.RetentionRaw, ProjectionVersion: 1}
|
||||
}
|
||||
|
||||
func safeScope(scope Scope) bool {
|
||||
values := []string{scope.OrganizationID, scope.ProjectID, scope.EnvironmentID, scope.ServiceID}
|
||||
if values[0] == "" {
|
||||
return false
|
||||
}
|
||||
for _, value := range values {
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
if len(value) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("._-", r)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
func TestPlanInjectsScopeAndReportsIndexes(t *testing.T) {
|
||||
ast, err := Parse(`logs | where service == "eql" | where status >= 500 | window 24h | sort timestamp desc | limit 50`, 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
explain, err := Plan(ast, Scope{OrganizationID: "personal-cole", ProjectID: "eql", EnvironmentID: "production"}, nil, 10<<20, Budget{MaxDuration: 5 * time.Second, MaxRows: 1000, MaxScannedBytes: 100 << 20, MaxMemoryBytes: 64 << 20})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(explain.ProjectedSources) != 1 || explain.ProjectedSources[0] != "organization:personal-cole/signal:logs/project:eql/environment:production" || len(explain.Fields) != 3 {
|
||||
t.Fatalf("explain=%+v", explain)
|
||||
}
|
||||
for _, field := range explain.Fields {
|
||||
if !field.Indexed || field.Unknown {
|
||||
t.Fatalf("field=%+v", field)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanRequiresSensitivePermissionForUnknownAndBody(t *testing.T) {
|
||||
for _, text := range []string{`logs | where vendor.unknown == "x" | limit 10`, `logs | where body =~ "error" | limit 10`} {
|
||||
ast, err := Parse(text, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
budget := Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1 << 20, MaxMemoryBytes: 1 << 20}
|
||||
if _, err := Plan(ast, Scope{OrganizationID: "org"}, nil, 100, budget); err == nil {
|
||||
t.Fatalf("expected sensitive rejection for %q", text)
|
||||
}
|
||||
explain, err := Plan(ast, Scope{OrganizationID: "org", Sensitive: true}, nil, 100, budget)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(explain.RequiredPermissions) != 2 || explain.CacheEligible {
|
||||
t.Fatalf("explain=%+v", explain)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanRejectsCrossTenantScopeAndScanBudget(t *testing.T) {
|
||||
ast, _ := Parse(`metrics | limit 10`, 100)
|
||||
budget := Budget{MaxDuration: time.Second, MaxRows: 100, MaxScannedBytes: 1000, MaxMemoryBytes: 1000}
|
||||
if _, err := Plan(ast, Scope{OrganizationID: "../other"}, nil, 10, budget); err == nil {
|
||||
t.Fatal("expected scope rejection")
|
||||
}
|
||||
if _, err := Plan(ast, Scope{OrganizationID: "org"}, nil, 1001, budget); err == nil {
|
||||
t.Fatal("expected scan-budget rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltinMetricDimensionsAreSignalScopedAndRetentionAware(t *testing.T) {
|
||||
state, ok := BuiltinDescriptor(model.SignalMetrics, "state")
|
||||
if !ok || state.Retention != schema.RetentionMetric {
|
||||
t.Fatalf("metric state descriptor=%+v ok=%t", state, ok)
|
||||
}
|
||||
if _, ok = BuiltinDescriptor(model.SignalLogs, "state"); ok {
|
||||
t.Fatal("metric-only state dimension was exposed to logs")
|
||||
}
|
||||
|
||||
metricRoute, ok := BuiltinDescriptor(model.SignalMetrics, "http.route")
|
||||
if !ok || metricRoute.Retention != schema.RetentionMetric {
|
||||
t.Fatalf("metric route descriptor=%+v ok=%t", metricRoute, ok)
|
||||
}
|
||||
logRoute, ok := BuiltinDescriptor(model.SignalLogs, "http.route")
|
||||
if !ok || logRoute.Retention != schema.RetentionRaw {
|
||||
t.Fatalf("log route descriptor=%+v ok=%t", logRoute, ok)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/observatory/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
ASTVersion = 1
|
||||
maxQueryWindow = 3650 * 24 * time.Hour
|
||||
)
|
||||
|
||||
type AST struct {
|
||||
Version int `json:"version"`
|
||||
Signal model.Signal `json:"signal"`
|
||||
Filters []Filter `json:"filters,omitempty"`
|
||||
Sort *Sort `json:"sort,omitempty"`
|
||||
Summary *Summary `json:"summary,omitempty"`
|
||||
Limit int `json:"limit"`
|
||||
Window time.Duration `json:"-"`
|
||||
WindowText string `json:"window,omitempty"`
|
||||
Bucket time.Duration `json:"-"`
|
||||
BucketText string `json:"bucket,omitempty"`
|
||||
}
|
||||
|
||||
type Filter struct {
|
||||
Field string `json:"field"`
|
||||
Op string `json:"op"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type Sort struct {
|
||||
Field string `json:"field"`
|
||||
Descending bool `json:"descending"`
|
||||
}
|
||||
|
||||
type Summary struct {
|
||||
Aggregates []Aggregate `json:"aggregates"`
|
||||
GroupBy []string `json:"group_by,omitempty"`
|
||||
}
|
||||
|
||||
type Aggregate struct {
|
||||
Function string `json:"function"`
|
||||
Field string `json:"field,omitempty"`
|
||||
Alias string `json:"alias"`
|
||||
}
|
||||
|
||||
func Parse(text string, maxLimit int) (AST, error) {
|
||||
if len(text) == 0 || len(text) > 16_384 {
|
||||
return AST{}, errors.New("query length outside accepted bounds")
|
||||
}
|
||||
parts := strings.Split(text, "|")
|
||||
if len(parts) > 16 {
|
||||
return AST{}, errors.New("too many query stages")
|
||||
}
|
||||
ast := AST{Version: ASTVersion, Signal: model.Signal(strings.TrimSpace(parts[0])), Limit: min(100, maxLimit)}
|
||||
if ast.Signal != model.SignalLogs && ast.Signal != model.SignalMetrics && ast.Signal != model.SignalTraces && ast.Signal != model.SignalDeployments {
|
||||
return AST{}, errors.New("query must begin with logs, metrics, traces, or deployments")
|
||||
}
|
||||
for _, raw := range parts[1:] {
|
||||
stage := strings.TrimSpace(raw)
|
||||
switch {
|
||||
case strings.HasPrefix(stage, "where "):
|
||||
filter, err := parseFilter(strings.TrimSpace(strings.TrimPrefix(stage, "where ")))
|
||||
if err != nil {
|
||||
return AST{}, err
|
||||
}
|
||||
ast.Filters = append(ast.Filters, filter)
|
||||
case strings.HasPrefix(stage, "sort "):
|
||||
fields := strings.Fields(strings.TrimSpace(strings.TrimPrefix(stage, "sort ")))
|
||||
if len(fields) < 1 || len(fields) > 2 || !validField(fields[0]) {
|
||||
return AST{}, errors.New("invalid sort stage")
|
||||
}
|
||||
desc := len(fields) == 2 && fields[1] == "desc"
|
||||
if len(fields) == 2 && fields[1] != "asc" && fields[1] != "desc" {
|
||||
return AST{}, errors.New("sort direction must be asc or desc")
|
||||
}
|
||||
ast.Sort = &Sort{Field: fields[0], Descending: desc}
|
||||
case strings.HasPrefix(stage, "summarize "):
|
||||
if ast.Summary != nil {
|
||||
return AST{}, errors.New("query may contain only one summarize stage")
|
||||
}
|
||||
summary, bucket, bucketText, err := parseSummary(strings.TrimSpace(strings.TrimPrefix(stage, "summarize ")))
|
||||
if err != nil {
|
||||
return AST{}, err
|
||||
}
|
||||
ast.Summary = &summary
|
||||
if bucket > 0 {
|
||||
ast.Bucket, ast.BucketText = bucket, bucketText
|
||||
}
|
||||
case strings.HasPrefix(stage, "limit "):
|
||||
n, err := strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(stage, "limit ")))
|
||||
if err != nil || n < 1 || n > maxLimit {
|
||||
return AST{}, fmt.Errorf("limit must be between 1 and %d", maxLimit)
|
||||
}
|
||||
ast.Limit = n
|
||||
case strings.HasPrefix(stage, "window "):
|
||||
windowText := strings.TrimSpace(strings.TrimPrefix(stage, "window "))
|
||||
d, err := time.ParseDuration(windowText)
|
||||
if err != nil || d < time.Second || d > maxQueryWindow {
|
||||
return AST{}, errors.New("window must be between 1s and 87600h")
|
||||
}
|
||||
ast.Window, ast.WindowText = d, windowText
|
||||
default:
|
||||
return AST{}, fmt.Errorf("unsupported query stage %q", stage)
|
||||
}
|
||||
}
|
||||
if err := Validate(ast, maxLimit); err != nil {
|
||||
return AST{}, err
|
||||
}
|
||||
return ast, nil
|
||||
}
|
||||
|
||||
func Validate(ast AST, maxLimit int) error {
|
||||
if ast.Version != ASTVersion || (ast.Signal != model.SignalLogs && ast.Signal != model.SignalMetrics && ast.Signal != model.SignalTraces && ast.Signal != model.SignalDeployments) {
|
||||
return errors.New("query AST identity is invalid")
|
||||
}
|
||||
if ast.Limit < 1 || ast.Limit > maxLimit {
|
||||
return fmt.Errorf("limit must be between 1 and %d", maxLimit)
|
||||
}
|
||||
if ast.Window < 0 || ast.Window > maxQueryWindow || ast.Window > 0 && ast.Window < time.Second {
|
||||
return errors.New("query window is invalid")
|
||||
}
|
||||
if ast.Bucket < 0 || ast.Bucket > maxQueryWindow || ast.Bucket > 0 && (ast.Bucket < time.Second || ast.Summary == nil) {
|
||||
return errors.New("query summary bucket is invalid")
|
||||
}
|
||||
if len(ast.Filters) > 16 {
|
||||
return errors.New("too many query filters")
|
||||
}
|
||||
for _, filter := range ast.Filters {
|
||||
if !validField(filter.Field) || len(filter.Value) > 4096 {
|
||||
return errors.New("query filter is invalid")
|
||||
}
|
||||
switch filter.Op {
|
||||
case "!=", ">=", "<=", "==", ">", "<":
|
||||
case "=~":
|
||||
if len(filter.Value) > 512 {
|
||||
return errors.New("regular expression exceeds 512 bytes")
|
||||
}
|
||||
if _, err := regexp.Compile(filter.Value); err != nil {
|
||||
return errors.New("invalid regular expression")
|
||||
}
|
||||
default:
|
||||
return errors.New("query filter operator is invalid")
|
||||
}
|
||||
}
|
||||
if ast.Sort != nil && !validField(ast.Sort.Field) {
|
||||
return errors.New("query sort is invalid")
|
||||
}
|
||||
if ast.Summary != nil {
|
||||
if len(ast.Summary.Aggregates) < 1 || len(ast.Summary.Aggregates) > 16 || len(ast.Summary.GroupBy) > 16 {
|
||||
return errors.New("query summary is invalid")
|
||||
}
|
||||
aliases := map[string]bool{}
|
||||
for _, aggregate := range ast.Summary.Aggregates {
|
||||
switch aggregate.Function {
|
||||
case "count":
|
||||
if aggregate.Field != "" {
|
||||
return errors.New("count accepts no field")
|
||||
}
|
||||
case "min", "max", "sum", "avg", "p50", "p95", "p99":
|
||||
if !validField(aggregate.Field) {
|
||||
return errors.New("aggregate field is invalid")
|
||||
}
|
||||
default:
|
||||
return errors.New("aggregate function is unsupported")
|
||||
}
|
||||
if !validField(aggregate.Alias) || aliases[aggregate.Alias] {
|
||||
return errors.New("aggregate alias is invalid or duplicated")
|
||||
}
|
||||
aliases[aggregate.Alias] = true
|
||||
}
|
||||
for _, group := range ast.Summary.GroupBy {
|
||||
if !validField(group) {
|
||||
return errors.New("grouping field is invalid")
|
||||
}
|
||||
if aliases[group] {
|
||||
return errors.New("grouping field conflicts with aggregate alias")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseFilter(expr string) (Filter, error) {
|
||||
for _, op := range []string{"!=", ">=", "<=", "==", "=~", ">", "<"} {
|
||||
if i := strings.Index(expr, op); i > 0 {
|
||||
field := strings.TrimSpace(expr[:i])
|
||||
value := strings.TrimSpace(expr[i+len(op):])
|
||||
if !validField(field) || value == "" || len(value) > 4096 {
|
||||
return Filter{}, errors.New("invalid where stage")
|
||||
}
|
||||
if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' {
|
||||
unquoted, err := strconv.Unquote(value)
|
||||
if err != nil {
|
||||
return Filter{}, errors.New("invalid quoted filter value")
|
||||
}
|
||||
value = unquoted
|
||||
}
|
||||
if op == "=~" {
|
||||
if len(value) > 512 {
|
||||
return Filter{}, errors.New("regular expression exceeds 512 bytes")
|
||||
}
|
||||
if _, err := regexp.Compile(value); err != nil {
|
||||
return Filter{}, errors.New("invalid regular expression")
|
||||
}
|
||||
}
|
||||
return Filter{Field: field, Op: op, Value: value}, nil
|
||||
}
|
||||
}
|
||||
return Filter{}, errors.New("where stage requires a comparison")
|
||||
}
|
||||
|
||||
func parseSummary(stage string) (Summary, time.Duration, string, error) {
|
||||
aggregateText, groupText, found := strings.Cut(stage, " by ")
|
||||
aggregateParts := strings.Split(aggregateText, ",")
|
||||
if len(aggregateParts) < 1 || len(aggregateParts) > 16 {
|
||||
return Summary{}, 0, "", errors.New("summarize requires between 1 and 16 aggregates")
|
||||
}
|
||||
summary := Summary{}
|
||||
aliases := map[string]bool{}
|
||||
for _, raw := range aggregateParts {
|
||||
expression := strings.TrimSpace(raw)
|
||||
open := strings.IndexByte(expression, '(')
|
||||
if open < 1 || !strings.HasSuffix(expression, ")") {
|
||||
return Summary{}, 0, "", errors.New("invalid aggregate")
|
||||
}
|
||||
function := expression[:open]
|
||||
field := strings.TrimSpace(expression[open+1 : len(expression)-1])
|
||||
switch function {
|
||||
case "count":
|
||||
if field != "" {
|
||||
return Summary{}, 0, "", errors.New("count accepts no field")
|
||||
}
|
||||
case "min", "max", "sum", "avg", "p50", "p95", "p99":
|
||||
if !validField(field) {
|
||||
return Summary{}, 0, "", errors.New("aggregate field is invalid")
|
||||
}
|
||||
default:
|
||||
return Summary{}, 0, "", errors.New("aggregate function is unsupported")
|
||||
}
|
||||
alias := function
|
||||
if field != "" {
|
||||
alias += "_" + strings.ReplaceAll(field, ".", "_")
|
||||
}
|
||||
if aliases[alias] {
|
||||
return Summary{}, 0, "", errors.New("aggregate alias is duplicated")
|
||||
}
|
||||
aliases[alias] = true
|
||||
summary.Aggregates = append(summary.Aggregates, Aggregate{Function: function, Field: field, Alias: alias})
|
||||
}
|
||||
var window time.Duration
|
||||
var windowText string
|
||||
if found {
|
||||
groups := strings.Split(groupText, ",")
|
||||
if len(groups) > 16 {
|
||||
return Summary{}, 0, "", errors.New("too many grouping fields")
|
||||
}
|
||||
for _, raw := range groups {
|
||||
group := strings.TrimSpace(raw)
|
||||
if strings.HasPrefix(group, "window(") && strings.HasSuffix(group, ")") {
|
||||
if window > 0 {
|
||||
return Summary{}, 0, "", errors.New("summary window is duplicated")
|
||||
}
|
||||
windowText = strings.TrimSpace(group[len("window(") : len(group)-1])
|
||||
parsed, err := time.ParseDuration(windowText)
|
||||
if err != nil || parsed < time.Second || parsed > maxQueryWindow {
|
||||
return Summary{}, 0, "", errors.New("summary window must be between 1s and 87600h")
|
||||
}
|
||||
window = parsed
|
||||
continue
|
||||
}
|
||||
if !validField(group) {
|
||||
return Summary{}, 0, "", errors.New("grouping field is invalid")
|
||||
}
|
||||
summary.GroupBy = append(summary.GroupBy, group)
|
||||
}
|
||||
}
|
||||
return summary, window, windowText, nil
|
||||
}
|
||||
|
||||
func validField(field string) bool {
|
||||
if field == "" || len(field) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, r := range field {
|
||||
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '.') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBoundedQuery(t *testing.T) {
|
||||
ast, err := Parse(`logs | where service == "eql" | where status >= 500 | window 24h | sort timestamp desc | limit 50`, 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ast.Limit != 50 || len(ast.Filters) != 2 || ast.WindowText != "24h" || ast.Sort == nil || !ast.Sort.Descending {
|
||||
t.Fatalf("unexpected AST: %#v", ast)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextAndVisualBuilderShareValidatedAST(t *testing.T) {
|
||||
fromText, err := Parse(`metrics | where service == "eql" | window 1h | limit 25`, 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := json.Marshal(fromText)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var fromBuilder AST
|
||||
if err := json.Unmarshal(b, &fromBuilder); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fromBuilder.Window = fromText.Window
|
||||
fromBuilder.Bucket = fromText.Bucket
|
||||
if err := Validate(fromBuilder, 1000); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(fromText, fromBuilder) {
|
||||
t.Fatalf("text=%+v builder=%+v", fromText, fromBuilder)
|
||||
}
|
||||
fromBuilder.Filters[0].Op = "SQL"
|
||||
if err := Validate(fromBuilder, 1000); err == nil {
|
||||
t.Fatal("expected hostile builder AST rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsSQLAndUnboundedLimit(t *testing.T) {
|
||||
for _, input := range []string{"select * from logs", "logs | limit 1001", "logs | where route;drop == x"} {
|
||||
if _, err := Parse(input, 1000); err == nil {
|
||||
t.Fatalf("expected rejection for %q", input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseUnifiedSummaryAndSafeRegex(t *testing.T) {
|
||||
ast, err := Parse(`logs | where route =~ "^/items/[0-9]+$" | summarize count(), p95(duration) by route, window(5m) | sort count desc | limit 50`, 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ast.Summary == nil || len(ast.Summary.Aggregates) != 2 || len(ast.Summary.GroupBy) != 1 || ast.WindowText != "" || ast.BucketText != "5m" || ast.Summary.Aggregates[1].Alias != "p95_duration" {
|
||||
t.Fatalf("ast=%+v", ast)
|
||||
}
|
||||
if _, err := Parse(`logs | where route =~ "["`, 1000); err == nil {
|
||||
t.Fatal("expected invalid regular expression rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSupportsApprovedTenYearColdLookback(t *testing.T) {
|
||||
ast, err := Parse(`logs | window 87600h | limit 10`, 100)
|
||||
if err != nil || ast.Window != maxQueryWindow {
|
||||
t.Fatalf("ast=%+v err=%v", ast, err)
|
||||
}
|
||||
if _, err = Parse(`logs | window 87601h | limit 10`, 100); err == nil {
|
||||
t.Fatal("lookback beyond retention ceiling was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzParse(f *testing.F) {
|
||||
for _, seed := range []string{
|
||||
`logs | where service == "eql" | limit 50`,
|
||||
`metrics | summarize count(), p95(duration) by route, window(5m) | limit 50`,
|
||||
`traces | where trace_id =~ "^[0-9a-f]{32}$" | window 1h | limit 10`,
|
||||
`select * from logs`,
|
||||
string([]byte{0, 1, 2, 3}),
|
||||
} {
|
||||
f.Add(seed)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, text string) {
|
||||
if len(text) > 20_000 {
|
||||
return
|
||||
}
|
||||
ast, err := Parse(text, 1_000)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err = Validate(ast, 1_000); err != nil {
|
||||
t.Fatalf("parser returned an invalid AST: %v", err)
|
||||
}
|
||||
if ast.Limit < 1 || ast.Limit > 1_000 || len(ast.Filters) > 16 {
|
||||
t.Fatalf("accepted AST violates hard bounds: %+v", ast)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"gamertan.com/observatory/internal/schema"
|
||||
)
|
||||
|
||||
const ResultVersion = 1
|
||||
|
||||
var (
|
||||
ErrBudgetExceeded = errors.New("query execution budget exceeded")
|
||||
ErrTypeMismatch = errors.New("query value does not match its field type")
|
||||
)
|
||||
|
||||
type Column struct {
|
||||
Field string `json:"field"`
|
||||
Type schema.Type `json:"type"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
}
|
||||
|
||||
// Row values align positionally with Result.Columns. Nil is a missing value;
|
||||
// non-nil values use the canonical string form described by the column type.
|
||||
type Row struct {
|
||||
Values []*string `json:"values"`
|
||||
}
|
||||
|
||||
type Statistics struct {
|
||||
ScannedRows int `json:"scanned_rows"`
|
||||
MatchedRows int `json:"matched_rows"`
|
||||
ScannedBytes int64 `json:"scanned_bytes"`
|
||||
DurationNS int64 `json:"duration_ns"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Approximate bool `json:"approximate,omitempty"`
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Version int `json:"version"`
|
||||
Explain Explain `json:"explain"`
|
||||
Columns []Column `json:"columns"`
|
||||
Rows []Row `json:"rows"`
|
||||
Stats Statistics `json:"statistics"`
|
||||
}
|
||||
Reference in New Issue
Block a user