observatory/internal/storage/projector_test.go

// SPDX-License-Identifier: AGPL-3.0-only

package storage

import (
	"context"
	"os"
	"testing"
	"time"

	"gamertan.com/observatory/internal/model"
	"gamertan.com/observatory/internal/query"
)

func TestDurableAcknowledgementPrecedesProjection(t *testing.T) {
	ctx := t.Context()
	store := testStore(t)
	defer store.Close()
	now := time.Now().UTC()
	scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
	token, err := store.CreateSource(ctx, "source-a", scope)
	if err != nil {
		t.Fatal(err)
	}
	batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
	ack, err := store.Ingest(ctx, token, batch, now)
	if err != nil || ack.Duplicate {
		t.Fatalf("ack=%+v err=%v", ack, err)
	}
	duplicate, err := store.Ingest(ctx, token, batch, now)
	if err != nil || !duplicate.Duplicate || duplicate.Digest != ack.Digest {
		t.Fatalf("duplicate=%+v err=%v", duplicate, err)
	}
	status, err := store.ProjectionStatus(ctx, now.Add(time.Second))
	if err != nil || status.PendingSegments != 1 || status.PendingBytes < 1 || status.OldestPendingLag < time.Second {
		t.Fatalf("status=%+v err=%v", status, err)
	}
	var projected int
	if err = store.control.QueryRowContext(ctx, `SELECT COUNT(projected_at) FROM segments`).Scan(&projected); err != nil || projected != 0 {
		t.Fatalf("projected=%d err=%v", projected, err)
	}
	report, err := store.ProjectPending(ctx)
	if err != nil || report.ProjectedSegments != 1 || report.ProjectedRecords != 1 || report.ProjectedBytes < 1 {
		t.Fatalf("report=%+v err=%v", report, err)
	}
	status, err = store.ProjectionStatus(ctx, now.Add(2*time.Second))
	if err != nil || status.PendingSegments != 0 || status.PendingBytes != 0 || !status.OldestCommitted.IsZero() || status.OldestPendingLag != 0 {
		t.Fatalf("projected status=%+v err=%v", status, err)
	}
}

func TestProjectorGroupsBoundedSegmentsAndResumesAfterReopen(t *testing.T) {
	ctx := t.Context()
	store := testStore(t)
	root := store.root
	now := time.Now().UTC()
	scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
	token, err := store.CreateSource(ctx, "source-a", scope)
	if err != nil {
		t.Fatal(err)
	}
	const total = projectionGroupMaxSegments + 3
	for sequence := uint64(1); sequence <= total; sequence++ {
		batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: sequence, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
		if _, err = store.Ingest(ctx, token, batch, now); err != nil {
			t.Fatal(err)
		}
	}
	first, err := store.ProjectPending(ctx)
	if err != nil || first.ProjectedSegments != projectionGroupMaxSegments {
		t.Fatalf("first=%+v err=%v", first, err)
	}
	if err = store.Close(); err != nil {
		t.Fatal(err)
	}
	store, err = Open(root)
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	if err = store.RecoverRaw(ctx); err != nil {
		t.Fatal(err)
	}
	status, err := store.ProjectionStatus(ctx, now.Add(time.Second))
	if err != nil || status.PendingSegments != total-projectionGroupMaxSegments {
		t.Fatalf("status=%+v err=%v", status, err)
	}
	second, err := store.ProjectPending(ctx)
	if err != nil || second.ProjectedSegments != total-projectionGroupMaxSegments {
		t.Fatalf("second=%+v err=%v", second, err)
	}
	ast, err := query.Parse(`logs | limit 100`, 100)
	if err != nil {
		t.Fatal(err)
	}
	result, err := store.Query(ctx, ast, query.Scope{OrganizationID: scope.OrganizationID}, testQueryBudget(), now)
	if err != nil || len(result.Rows) != total {
		t.Fatalf("rows=%d err=%v", len(result.Rows), err)
	}
}

func TestProjectorFailureIsolatedFromAcknowledgementAndOtherOrganization(t *testing.T) {
	ctx := t.Context()
	store := testStore(t)
	defer store.Close()
	now := time.Now().UTC()
	for index, organizationID := range []string{"organization-a", "organization-b"} {
		sourceID := "source-" + string(rune('a'+index))
		scope := model.Scope{OrganizationID: organizationID, ProjectID: "project", EnvironmentID: "production", ServiceID: "service"}
		token, err := store.CreateSource(ctx, sourceID, scope)
		if err != nil {
			t.Fatal(err)
		}
		batch := model.Batch{Version: model.BatchVersion, SourceID: sourceID, StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
		if _, err = store.Ingest(ctx, token, batch, now); err != nil {
			t.Fatal(err)
		}
	}
	var corruptPath string
	if err := store.control.QueryRowContext(ctx, `SELECT path FROM segments WHERE organization_id='organization-a'`).Scan(&corruptPath); err != nil {
		t.Fatal(err)
	}
	corruptBody, err := os.ReadFile(corruptPath)
	if err != nil {
		t.Fatal(err)
	}
	corruptBody[0] ^= 0xff
	if err := os.WriteFile(corruptPath, corruptBody, 0o600); err != nil {
		t.Fatal(err)
	}
	report, err := store.ProjectPending(ctx)
	if err == nil || report.ProjectedSegments != 1 {
		t.Fatalf("report=%+v err=%v", report, err)
	}
	var projectedA, projectedB int
	if err = store.control.QueryRowContext(ctx, `SELECT COUNT(projected_at) FROM segments WHERE organization_id='organization-a'`).Scan(&projectedA); err != nil {
		t.Fatal(err)
	}
	if err = store.control.QueryRowContext(ctx, `SELECT COUNT(projected_at) FROM segments WHERE organization_id='organization-b'`).Scan(&projectedB); err != nil {
		t.Fatal(err)
	}
	if projectedA != 0 || projectedB != 1 {
		t.Fatalf("projected organization-a=%d organization-b=%d", projectedA, projectedB)
	}
	if err = store.RecoverRaw(ctx); err != nil {
		t.Fatalf("raw reconciliation decoded catalogued pending evidence: %v", err)
	}
}

func TestRunProjectorWakesOnAcceptedBatch(t *testing.T) {
	ctx, cancel := context.WithCancel(t.Context())
	store := testStore(t)
	defer store.Close()
	done := make(chan struct{})
	go func() {
		defer close(done)
		store.RunProjector(ctx, time.Minute, func(err error) { t.Errorf("projector: %v", err) })
	}()
	now := time.Now().UTC()
	scope := model.Scope{OrganizationID: "organization-a", ProjectID: "project-a", EnvironmentID: "production", ServiceID: "service-a"}
	token, err := store.CreateSource(ctx, "source-a", scope)
	if err != nil {
		t.Fatal(err)
	}
	batch := model.Batch{Version: model.BatchVersion, SourceID: "source-a", StreamID: "logs", Sequence: 1, ObservedAt: now, Signal: model.SignalLogs, Records: []model.Observation{{Timestamp: now, Name: "application.request"}}}
	if _, err = store.Ingest(ctx, token, batch, now); err != nil {
		t.Fatal(err)
	}
	deadline := time.Now().Add(2 * time.Second)
	for {
		status, statusErr := store.ProjectionStatus(t.Context(), time.Now().UTC())
		if statusErr != nil {
			t.Fatal(statusErr)
		}
		if status.PendingSegments == 0 {
			break
		}
		if time.Now().After(deadline) {
			t.Fatalf("projection remained pending: %+v", status)
		}
		time.Sleep(10 * time.Millisecond)
	}
	cancel()
	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatal("projector did not stop after cancellation")
	}
}