Push the rest

2026-05-11 10:58:46 +02:00
parent adb5c1a439
commit 0031caf16c
94 changed files with 11777 additions and 3474 deletions
--- a/backend/analytics/internal/persistence/audit.go
+++ b/backend/analytics/internal/persistence/audit.go
@@ -0,0 +1,152 @@
+package persistence
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/trace"
+)
+
+var (
+	persistTracer = otel.Tracer("otel-bi.persistence")
+	persistMeter  = otel.Meter("otel-bi.persistence")
+
+	persistWritesTotal, _ = persistMeter.Int64Counter(
+		"persistence.writes_total",
+		metric.WithDescription("Total persistence write operations"),
+	)
+)
+
+// newUUID generates a random UUID v4.
+func newUUID() string {
+	var b [16]byte
+	rand.Read(b[:]) //nolint:errcheck
+	b[6] = (b[6] & 0x0f) | 0x40
+	b[8] = (b[8] & 0x3f) | 0x80
+	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
+		b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
+}
+
+// spanContext extracts trace_id and span_id from the current span as nullable strings.
+func spanContext(span trace.Span) (traceID, spanID *string) {
+	sctx := span.SpanContext()
+	if !sctx.IsValid() {
+		return nil, nil
+	}
+	tid := sctx.TraceID().String()
+	sid := sctx.SpanID().String()
+	return &tid, &sid
+}
+
+// mustJSON marshals v to JSON bytes, returning nil on error.
+func mustJSON(v any) []byte {
+	b, _ := json.Marshal(v)
+	return b
+}
+
+// ---------------------------------------------------------------------------
+// Job execution tracking
+// ---------------------------------------------------------------------------
+
+func RecordJobStart(ctx context.Context, pool *pgxpool.Pool, jobName, domain string, traceID, spanID *string) string {
+	ctx, span := persistTracer.Start(ctx, "persistence.record_job_start")
+	defer span.End()
+
+	id := newUUID()
+	_, err := pool.Exec(ctx,
+		`INSERT INTO job_executions (id, started_at, job_name, domain, status, trace_id, span_id)
+		 VALUES ($1, NOW(), $2, $3, 'running', $4, $5)`,
+		id, jobName, domain, traceID, spanID,
+	)
+	if err != nil {
+		slog.Warn("failed to record job start", "job", jobName, "err", err)
+	}
+	return id
+}
+
+func RecordJobComplete(ctx context.Context, pool *pgxpool.Pool, jobID string, startedAt time.Time, records int) {
+	ctx, span := persistTracer.Start(ctx, "persistence.record_job_complete")
+	defer span.End()
+
+	durationMs := int64(time.Since(startedAt).Milliseconds())
+	_, err := pool.Exec(ctx,
+		`UPDATE job_executions
+		 SET status = 'success', completed_at = NOW(), duration_ms = $2, records_processed = $3
+		 WHERE id = $1`,
+		jobID, durationMs, records,
+	)
+	if err != nil {
+		slog.Warn("failed to record job complete", "id", jobID, "err", err)
+	}
+}
+
+func RecordJobFailure(ctx context.Context, pool *pgxpool.Pool, jobID string, startedAt time.Time, errMsg string) {
+	ctx, span := persistTracer.Start(ctx, "persistence.record_job_failure")
+	defer span.End()
+
+	durationMs := int64(time.Since(startedAt).Milliseconds())
+	if len(errMsg) > 2000 {
+		errMsg = errMsg[:2000]
+	}
+	_, err := pool.Exec(ctx,
+		`UPDATE job_executions
+		 SET status = 'failure', completed_at = NOW(), duration_ms = $2, error_message = $3
+		 WHERE id = $1`,
+		jobID, durationMs, errMsg,
+	)
+	if err != nil {
+		slog.Warn("failed to record job failure", "id", jobID, "err", err)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Audit log
+// ---------------------------------------------------------------------------
+
+type AuditEntry struct {
+	Action     string
+	ActorType  string
+	ActorID    string
+	Domain     string
+	Service    string
+	EntityType string
+	Status     string
+	Payload    any
+}
+
+func AppendAudit(ctx context.Context, pool *pgxpool.Pool, e AuditEntry) {
+	ctx, span := persistTracer.Start(ctx, "persistence.append_audit",
+		trace.WithAttributes(
+			attribute.String("audit.action", e.Action),
+			attribute.String("audit.domain", e.Domain),
+		),
+	)
+	defer span.End()
+
+	traceID, spanID := spanContext(span)
+	status := e.Status
+	if status == "" {
+		status = "success"
+	}
+	payloadJSON := mustJSON(e.Payload)
+
+	_, err := pool.Exec(ctx,
+		`INSERT INTO audit_log
+		 (id, occurred_at, action, status, actor_type, actor_id, domain, service, entity_type, trace_id, span_id, payload)
+		 VALUES ($1, NOW(), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb)`,
+		newUUID(), e.Action, status, e.ActorType, e.ActorID,
+		e.Domain, e.Service, e.EntityType,
+		traceID, spanID, payloadJSON,
+	)
+	if err != nil {
+		slog.Warn("failed to append audit", "action", e.Action, "err", err)
+	}
+}
--- a/backend/analytics/internal/persistence/aw.go
+++ b/backend/analytics/internal/persistence/aw.go
@@ -0,0 +1,140 @@
+package persistence
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/trace"
+
+	"otel-bi-analytics/internal/analytics"
+)
+
+func PersistForecast(ctx context.Context, pool *pgxpool.Pool, data []analytics.ForecastPoint, horizonDays int, source string) {
+	ctx, span := persistTracer.Start(ctx, "persistence.aw.persist_forecast",
+		trace.WithAttributes(
+			attribute.Int("horizon_days", horizonDays),
+			attribute.Int("point_count", len(data)),
+		),
+	)
+	defer span.End()
+
+	traceID, spanID := spanContext(span)
+	_, err := pool.Exec(ctx,
+		`INSERT INTO aw_sales_forecasts
+		 (id, created_at, horizon_days, point_count, trigger_source, trace_id, span_id, payload)
+		 VALUES ($1, NOW(), $2, $3, $4, $5, $6, $7::jsonb)`,
+		newUUID(), horizonDays, len(data), source, traceID, spanID, mustJSON(data),
+	)
+	if err != nil {
+		slog.Warn("failed to persist AW forecast", "err", err)
+		span.RecordError(err)
+		return
+	}
+	persistWritesTotal.Add(ctx, 1, metric.WithAttributes(attribute.String("entity", "aw_sales_forecast")))
+
+	AppendAudit(ctx, pool, AuditEntry{
+		Action: "forecast.generated", ActorType: actorType(source), ActorID: source,
+		Domain: "aw", Service: "otel-bi-analytics", EntityType: "sales_forecast",
+		Payload: map[string]any{"horizon_days": horizonDays, "point_count": len(data)},
+	})
+}
+
+func PersistRepScores(ctx context.Context, pool *pgxpool.Pool, data []analytics.RepScore, topN int, source string) {
+	ctx, span := persistTracer.Start(ctx, "persistence.aw.persist_rep_scores",
+		trace.WithAttributes(attribute.Int("rep_count", len(data))),
+	)
+	defer span.End()
+
+	traceID, spanID := spanContext(span)
+	_, err := pool.Exec(ctx,
+		`INSERT INTO aw_rep_scores
+		 (id, computed_at, rep_count, trigger_source, trace_id, span_id, payload)
+		 VALUES ($1, NOW(), $2, $3, $4, $5, $6::jsonb)`,
+		newUUID(), len(data), source, traceID, spanID, mustJSON(data),
+	)
+	if err != nil {
+		slog.Warn("failed to persist AW rep scores", "err", err)
+		span.RecordError(err)
+		return
+	}
+	persistWritesTotal.Add(ctx, 1, metric.WithAttributes(attribute.String("entity", "aw_rep_scores")))
+
+	AppendAudit(ctx, pool, AuditEntry{
+		Action: "scores.generated", ActorType: actorType(source), ActorID: source,
+		Domain: "aw", Service: "otel-bi-analytics", EntityType: "rep_scores",
+		Payload: map[string]any{"rep_count": len(data), "top_n": topN},
+	})
+}
+
+func PersistProductDemand(ctx context.Context, pool *pgxpool.Pool, data []analytics.ProductDemand, topN int, source string) {
+	ctx, span := persistTracer.Start(ctx, "persistence.aw.persist_product_demand",
+		trace.WithAttributes(attribute.Int("product_count", len(data))),
+	)
+	defer span.End()
+
+	traceID, spanID := spanContext(span)
+	_, err := pool.Exec(ctx,
+		`INSERT INTO aw_product_demand
+		 (id, computed_at, product_count, top_n, trigger_source, trace_id, span_id, payload)
+		 VALUES ($1, NOW(), $2, $3, $4, $5, $6, $7::jsonb)`,
+		newUUID(), len(data), topN, source, traceID, spanID, mustJSON(data),
+	)
+	if err != nil {
+		slog.Warn("failed to persist AW product demand", "err", err)
+		span.RecordError(err)
+		return
+	}
+	persistWritesTotal.Add(ctx, 1, metric.WithAttributes(attribute.String("entity", "aw_product_demand")))
+
+	AppendAudit(ctx, pool, AuditEntry{
+		Action: "scores.generated", ActorType: actorType(source), ActorID: source,
+		Domain: "aw", Service: "otel-bi-analytics", EntityType: "product_demand",
+		Payload: map[string]any{"product_count": len(data), "top_n": topN},
+	})
+}
+
+func PersistAnomalyRun(ctx context.Context, pool *pgxpool.Pool, data []analytics.AnomalyPoint, source string) {
+	ctx, span := persistTracer.Start(ctx, "persistence.aw.persist_anomaly_run")
+	defer span.End()
+
+	anomalyCount := 0
+	for _, p := range data {
+		if p.IsAnomaly {
+			anomalyCount++
+		}
+	}
+	span.SetAttributes(
+		attribute.Int("series_points", len(data)),
+		attribute.Int("anomaly_count", anomalyCount),
+	)
+
+	traceID, spanID := spanContext(span)
+	_, err := pool.Exec(ctx,
+		`INSERT INTO aw_anomaly_runs
+		 (id, detected_at, anomaly_count, series_days, window_days, threshold_sigma, trigger_source, trace_id, span_id, payload)
+		 VALUES ($1, NOW(), $2, 365, 30, 2.0, $3, $4, $5, $6::jsonb)`,
+		newUUID(), anomalyCount, source, traceID, spanID, mustJSON(data),
+	)
+	if err != nil {
+		slog.Warn("failed to persist AW anomaly run", "err", err)
+		span.RecordError(err)
+		return
+	}
+	persistWritesTotal.Add(ctx, 1, metric.WithAttributes(attribute.String("entity", "aw_anomaly_run")))
+
+	AppendAudit(ctx, pool, AuditEntry{
+		Action: "anomaly_detection.ran", ActorType: actorType(source), ActorID: source,
+		Domain: "aw", Service: "otel-bi-analytics", EntityType: "anomaly_detection",
+		Payload: map[string]any{"series_days": 365, "window_days": 30, "anomaly_count": anomalyCount},
+	})
+}
+
+func actorType(source string) string {
+	if len(source) >= 9 && source[:9] == "scheduler" {
+		return "scheduler"
+	}
+	return "api"
+}
--- a/backend/analytics/internal/persistence/wwi.go
+++ b/backend/analytics/internal/persistence/wwi.go
@@ -0,0 +1,151 @@
+package persistence
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/trace"
+
+	"otel-bi-analytics/internal/analytics"
+)
+
+var (
+	businessEventsTotal, _ = persistMeter.Int64Counter(
+		"wwi.business_events_generated_total",
+		metric.WithDescription("Business events generated from reorder data"),
+	)
+)
+
+func PersistReorderRecommendations(ctx context.Context, pool *pgxpool.Pool, data []analytics.ReorderRecommendation, source string) {
+	ctx, span := persistTracer.Start(ctx, "persistence.wwi.persist_reorder_recommendations",
+		trace.WithAttributes(attribute.Int("item_count", len(data))),
+	)
+	defer span.End()
+
+	traceID, spanID := spanContext(span)
+	_, err := pool.Exec(ctx,
+		`INSERT INTO wwi_reorder_recommendations
+		 (id, created_at, item_count, trigger_source, trace_id, span_id, payload)
+		 VALUES ($1, NOW(), $2, $3, $4, $5, $6::jsonb)`,
+		newUUID(), len(data), source, traceID, spanID, mustJSON(data),
+	)
+	if err != nil {
+		slog.Warn("failed to persist WWI reorder recommendations", "err", err)
+		span.RecordError(err)
+		return
+	}
+	persistWritesTotal.Add(ctx, 1, metric.WithAttributes(attribute.String("entity", "wwi_reorder_recommendations")))
+
+	AppendAudit(ctx, pool, AuditEntry{
+		Action: "recommendations.generated", ActorType: actorType(source), ActorID: source,
+		Domain: "wwi", Service: "otel-bi-analytics", EntityType: "reorder_recommendations",
+		Payload: map[string]any{"item_count": len(data)},
+	})
+}
+
+func PersistSupplierScores(ctx context.Context, pool *pgxpool.Pool, data []analytics.SupplierScore, topN int, source string) {
+	ctx, span := persistTracer.Start(ctx, "persistence.wwi.persist_supplier_scores",
+		trace.WithAttributes(attribute.Int("supplier_count", len(data))),
+	)
+	defer span.End()
+
+	traceID, spanID := spanContext(span)
+	_, err := pool.Exec(ctx,
+		`INSERT INTO wwi_supplier_scores
+		 (id, computed_at, supplier_count, top_n, trigger_source, trace_id, span_id, payload)
+		 VALUES ($1, NOW(), $2, $3, $4, $5, $6, $7::jsonb)`,
+		newUUID(), len(data), topN, source, traceID, spanID, mustJSON(data),
+	)
+	if err != nil {
+		slog.Warn("failed to persist WWI supplier scores", "err", err)
+		span.RecordError(err)
+		return
+	}
+	persistWritesTotal.Add(ctx, 1, metric.WithAttributes(attribute.String("entity", "wwi_supplier_scores")))
+
+	AppendAudit(ctx, pool, AuditEntry{
+		Action: "scores.generated", ActorType: actorType(source), ActorID: source,
+		Domain: "wwi", Service: "otel-bi-analytics", EntityType: "supplier_scores",
+		Payload: map[string]any{"supplier_count": len(data), "top_n": topN},
+	})
+}
+
+// GenerateStockEvents writes LOW_STOCK business events for HIGH-urgency items,
+// deduplicating within a 24-hour window.
+func GenerateStockEvents(ctx context.Context, pool *pgxpool.Pool, items []analytics.ReorderRecommendation) error {
+	ctx, span := persistTracer.Start(ctx, "persistence.wwi.generate_stock_events")
+	defer span.End()
+
+	cutoff := time.Now().UTC().Add(-24 * time.Hour)
+
+	tx, err := pool.Begin(ctx)
+	if err != nil {
+		return fmt.Errorf("begin transaction: %w", err)
+	}
+	defer tx.Rollback(ctx) //nolint:errcheck
+
+	inserted := 0
+	for _, item := range items {
+		if item.Urgency != "HIGH" {
+			continue
+		}
+		entityKey := fmt.Sprintf("%d", item.StockItemKey)
+
+		var existingID string
+		err := tx.QueryRow(ctx,
+			`SELECT id FROM wwi_business_events
+			 WHERE event_type = 'LOW_STOCK' AND entity_key = $1 AND occurred_at >= $2
+			 LIMIT 1`,
+			entityKey, cutoff,
+		).Scan(&existingID)
+		if err == nil {
+			continue // already exists within 24h
+		}
+		if !errors.Is(err, pgx.ErrNoRows) {
+			slog.Warn("error checking existing business event", "err", err)
+			continue
+		}
+
+		daysStr := "immediately"
+		if item.DaysUntilStockout != nil {
+			daysStr = fmt.Sprintf("%.1f days", *item.DaysUntilStockout)
+		}
+		message := fmt.Sprintf(
+			"Stock for '%s' will be exhausted in %s. Current stock: %.0f units, daily demand: %.1f units.",
+			item.StockItemName, daysStr, item.CurrentStock, item.AvgDailyDemand,
+		)
+
+		traceID, spanID := spanContext(span)
+		details := mustJSON(map[string]any{
+			"current_stock":           item.CurrentStock,
+			"avg_daily_demand":        item.AvgDailyDemand,
+			"recommended_reorder_qty": item.RecommendedReorderQty,
+		})
+
+		_, err = tx.Exec(ctx,
+			`INSERT INTO wwi_business_events
+			 (id, occurred_at, event_type, severity, entity_key, entity_name, message, trace_id, span_id, details)
+			 VALUES ($1, NOW(), 'LOW_STOCK', 'HIGH', $2, $3, $4, $5, $6, $7::jsonb)`,
+			newUUID(), entityKey, item.StockItemName, message, traceID, spanID, details,
+		)
+		if err != nil {
+			slog.Warn("failed to insert business event", "item", item.StockItemKey, "err", err)
+			continue
+		}
+		inserted++
+		businessEventsTotal.Add(ctx, 1, metric.WithAttributes(attribute.String("event_type", "LOW_STOCK")))
+	}
+
+	if err := tx.Commit(ctx); err != nil {
+		return fmt.Errorf("commit stock events: %w", err)
+	}
+	span.SetAttributes(attribute.Int("events_inserted", inserted))
+	return nil
+}