From 5cbc1d50fd50695f4d2783b8c9ed513cb3b34534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domagoj=20Andri=C4=87?= Date: Mon, 11 May 2026 16:04:10 +0200 Subject: [PATCH] Add more changes --- .env.example | 96 ++++++++ README.md | 537 +++++++++++++++++++++--------------------- backend/.env.example | 68 ------ frontend/.env.example | 19 -- 4 files changed, 365 insertions(+), 355 deletions(-) create mode 100644 .env.example delete mode 100644 backend/.env.example delete mode 100644 frontend/.env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0d891df --- /dev/null +++ b/.env.example @@ -0,0 +1,96 @@ +# ============================================================================= +# OTel BI Platform — master environment reference +# +# For Docker Compose: copy to .env in the project root. +# For local dev: each service reads its own subset — see comments below. +# +# POSTGRES_DSN for the Go analytics service is constructed in docker-compose.yml +# from the individual POSTGRES_* vars below — do not duplicate the password here. +# ============================================================================= + + +# ----------------------------------------------------------------------------- +# PostgreSQL — shared by both backend services +# Change POSTGRES_PASSWORD here and update it in POSTGRES_DSN below. +# ----------------------------------------------------------------------------- +POSTGRES_PASSWORD=changeme +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_DATABASE=otel_bi +POSTGRES_USERNAME=otel_bi +POSTGRES_SSLMODE=prefer + + +# ----------------------------------------------------------------------------- +# Go analytics service (otel-bi-analytics) → backend/analytics/ +# ----------------------------------------------------------------------------- + +# MSSQL data warehouses — required +# sqlserver://user:pass@host:port?database=name&TrustServerCertificate=true&ApplicationIntent=ReadOnly +AW_MSSQL_DSN=sqlserver://sa:YourStrongPassword123!@host.docker.internal:1433?database=AdventureWorksDW2022&TrustServerCertificate=true&ApplicationIntent=ReadOnly +WWI_MSSQL_DSN=sqlserver://sa:YourStrongPassword123!@host.docker.internal:1433?database=WideWorldImportersDW&TrustServerCertificate=true&ApplicationIntent=ReadOnly + +# POSTGRES_DSN is assembled by docker-compose.yml from the vars above: +# postgresql://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DATABASE}?sslmode=${POSTGRES_SSLMODE} +# For local dev without Compose, export POSTGRES_DSN manually in your shell. + +# HTTP port (default 8080) +# PORT=8080 + +# Analytics tuning — all optional +# DEFAULT_TOP_N=10 +# FORECAST_HORIZON_DAYS=30 +# DEFAULT_HISTORY_DAYS=365 + + +# ----------------------------------------------------------------------------- +# Python API (otel-bi-api) → backend/ +# ----------------------------------------------------------------------------- + +# URL of the Go analytics service +ANALYTICS_SERVICE_URL=http://analytics:8080 + +# PostgreSQL — reads the individual POSTGRES_* vars defined above + +# Comma-separated allowed CORS origins +CORS_ORIGINS=http://localhost:8080 + +# JWT validation for incoming browser requests +# Set false only for local dev — always true in production +REQUIRE_FRONTEND_AUTH=false +# FRONTEND_JWT_ISSUER_URL=https://sso.example.com/realms/otel-bi +# FRONTEND_JWT_AUDIENCE=otel-bi-api +# FRONTEND_JWT_JWKS_URL=https://sso.example.com/realms/otel-bi/protocol/openid-connect/certs +# FRONTEND_REQUIRED_SCOPES=openid profile + +# OIDC client config — served to the SPA at runtime via GET /api/config +# NOT baked into the frontend image +# FRONTEND_OIDC_CLIENT_ID=otel-bi-frontend +# FRONTEND_OIDC_SCOPE=openid profile email + +# Path for generated full-report XLSX + PDF files +# Mount a PVC here in Kubernetes +REPORT_OUTPUT_DIR=/reports + +APP_ENV=prod +LOG_LEVEL=INFO + + +# ----------------------------------------------------------------------------- +# Frontend (otel-bi-frontend) → frontend/ +# Baked into the image at build time via Docker build-args. +# In Docker Compose these are passed as build args, not runtime env. +# For local dev copy frontend/.env.example to frontend/.env.local instead. +# ----------------------------------------------------------------------------- +VITE_API_BASE_URL=http://localhost:8000 +VITE_OTEL_COLLECTOR_ENDPOINT=http://alloy:4318 +VITE_OTEL_SERVICE_NAME=otel-bi-frontend +VITE_OTEL_SERVICE_NAMESPACE=final-thesis + + +# ----------------------------------------------------------------------------- +# OpenTelemetry — shared collector endpoint +# Same value goes to Go analytics, Python API, and frontend build arg above +# ----------------------------------------------------------------------------- +OTEL_COLLECTOR_ENDPOINT=http://alloy:4318 +OTEL_SERVICE_NAMESPACE=final-thesis diff --git a/README.md b/README.md index 04f2694..b258c93 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# OTel BI App +# OTel BI Platform -OpenTelemetry-instrumented business intelligence platform with two MSSQL data warehouse sources (AdventureWorks, WorldWideImporters), a PostgreSQL write store, scheduled analytics jobs, an audit log, per-view CSV/PDF data exports, full XLSX+PDF report generation to a shared network path, and an OIDC-secured React frontend. +OpenTelemetry-instrumented business intelligence platform backed by two MSSQL data warehouses (AdventureWorks DW, WideWorldImporters DW), a PostgreSQL write store, a Go analytics engine with a built-in cron scheduler, and an OIDC-secured React frontend. --- @@ -11,26 +11,38 @@ Browser (React + TypeScript) │ OIDC Authorization Code + PKCE │ Bearer JWT on every API call ▼ -api-gateway :8000 ← public surface; validates JWT; mints internal tokens - ├── /api/aw/** → aw-service :8001 (AdventureWorks DW, MSSQL, read-only) - ├── /api/wwi/** → wwi-service :8002 (WorldWideImporters DW, MSSQL, read-only) - ├── /api/reports/generate → aggregates both services, writes XLSX + PDF to shared mount - ├── /api/*/export/** → streams per-view XLSX or PDF directly to browser - ├── /api/audit → reads AuditLog from PostgreSQL - ├── /api/jobs/** → proxies job history from services - └── /api/config → runtime OIDC config (unauthenticated) +otel-bi-api :8000 Python FastAPI — JWT validation, report generation + ├── GET /api/config OIDC config for the SPA (unauthenticated) + ├── GET /api/health + ├── GET /api/telemetry/status + ├── GET /api/audit AuditLog rows from PostgreSQL + ├── GET /api/exports ExportRecord rows from PostgreSQL + ├── GET /api/jobs/aw JobExecution rows from PostgreSQL + ├── GET /api/jobs/wwi + ├── POST /api/reports/generate fan-out to Go → XLSX + PDF to shared mount + ├── /api/aw/** proxied to otel-bi-analytics + PDF export + └── /api/wwi/** proxied to otel-bi-analytics + PDF export -aw-service AdventureWorks analytics + APScheduler background jobs -wwi-service WorldWideImporters analytics + APScheduler background jobs +otel-bi-analytics :8080 Go — both MSSQL sources, robfig/cron scheduler + ├── GET /aw/sales/** AdventureWorks analytics + ├── GET /aw/reps/** + ├── GET /aw/products/** + ├── GET /aw/anomalies + ├── GET /aw/export/** XLSX generation (excelize) + ├── GET /wwi/sales/** WideWorldImporters analytics + ├── GET /wwi/stock/** + ├── GET /wwi/suppliers/** + ├── GET /wwi/export/** XLSX generation + └── POST /scheduler/** manual job triggers -PostgreSQL Shared write store - ├── audit_log Append-only event trail - ├── job_executions Scheduled job run history - ├── export_records Per-view export metadata - ├── aw_* AW persistence tables - └── wwi_* WWI persistence tables +PostgreSQL Shared write store + ├── audit_log Append-only event trail + ├── job_executions Scheduled job run history + ├── export_records Per-view export metadata + ├── aw_* AW persistence tables + └── wwi_* WWI persistence tables -Grafana Alloy (OTLP/HTTP) Traces + metrics receiver → Tempo / Prometheus +Grafana Alloy (OTLP/HTTP) Traces + metrics → Tempo / Prometheus ``` --- @@ -39,15 +51,15 @@ Grafana Alloy (OTLP/HTTP) Traces + metrics receiver → Tempo / Prometheus | Feature | Description | |---|---| -| **AW Analytics** | Sales KPIs, 4-year history, linear-regression forecast, rep scores, product demand | -| **WWI Analytics** | Sales KPIs, reorder recommendations, supplier reliability scores, business events, what-if scenarios | -| **Scheduled Jobs** | APScheduler cron jobs in each service; per-job OTel root span; `job_executions` DB record | -| **Audit Log** | Append-only `audit_log` table; every analytics call, job, export, and report is logged | -| **Data Export** | Per-view download as XLSX or PDF; `export_records` table; OTel span per download | -| **Full Reports** | `POST /api/reports/generate` aggregates all views; writes `.xlsx` + `.pdf` to SMB/CSI mount | -| **Runtime OIDC** | Frontend fetches OIDC config from `GET /api/config` at boot — no build-time secrets baked into the image | -| **Data Quality** | Daily DQ checks on both warehouses; results surfaced in job history | -| **OTel** | Frontend W3C trace propagation; FastAPI, SQLAlchemy, HTTPX auto-instrumentation; manual spans on analytics | +| **AW Analytics** | Sales KPIs, 4-year history, linear-regression forecast, rep scores, product demand, anomaly detection | +| **WWI Analytics** | Sales KPIs, reorder recommendations, supplier reliability scores, business events | +| **Scheduled Jobs** | 8 robfig/cron jobs in the Go service; per-job OTel root span; `job_executions` record | +| **Manual Triggers** | `POST /api/aw/jobs/{job}/trigger` and `/api/wwi/jobs/{job}/trigger` | +| **Audit Log** | Append-only `audit_log`; every analytics call, job, export, and report is recorded | +| **Data Export** | Per-view XLSX (Go/excelize) or PDF (Python/reportlab); `export_records` metadata | +| **Full Reports** | `POST /api/reports/generate` aggregates all views; writes `.xlsx` + `.pdf` to a shared mount | +| **Runtime OIDC** | Frontend fetches OIDC config from `GET /api/config` at boot — nothing baked into the image | +| **OTel** | W3C trace propagation end-to-end; auto-instrumentation on HTTP, SQL, and HTTPX layers | --- @@ -56,117 +68,119 @@ Grafana Alloy (OTLP/HTTP) Traces + metrics receiver → Tempo / Prometheus ``` . ├── .gitea/workflows/ -│ └── docker-publish.yml # Gitea Actions: build & push all 4 images (matrix) +│ └── ci.yml Gitea Actions: test + build 3 images on Rocky Linux 10 +├── .env.example Docker Compose top-level vars ├── backend/ +│ ├── Dockerfile otel-bi-api (Python FastAPI) +│ ├── Dockerfile.analytics otel-bi-analytics (Go) │ ├── pyproject.toml │ ├── uv.lock -│ ├── .env.example -│ ├── shared/ +│ ├── .env.example Python API local dev +│ ├── app/ │ │ ├── core/ -│ │ │ ├── audit.py # SharedBase, AuditLog, JobExecution, ExportRecord, helpers -│ │ │ ├── config.py # Pydantic settings (all services) -│ │ │ ├── export.py # to_xlsx_bytes(), to_pdf_bytes() -│ │ │ ├── otel.py # configure_otel(), instrument_fastapi/sqlalchemy/httpx -│ │ │ ├── reports.py # save_report() → XLSX + PDF on filesystem -│ │ │ └── security.py # JWT validation, internal token mint/verify -│ │ └── db/ -│ │ ├── mssql.py # create_aw_engine(), create_wwi_engine() -│ │ └── postgres.py # create_postgres_engine(), create_session_factory() -│ └── services/ -│ ├── api_gateway/ -│ │ └── main.py -│ ├── aw_service/ -│ │ ├── main.py -│ │ ├── analytics.py -│ │ ├── models.py -│ │ └── scheduler.py -│ └── wwi_service/ -│ ├── main.py -│ ├── analytics.py -│ ├── models.py -│ └── scheduler.py -├── frontend/ -│ ├── Dockerfile -│ ├── src/ -│ │ ├── api/ -│ │ │ └── config.ts # fetchAppConfig() / getAppConfig() -│ │ ├── auth/ -│ │ │ └── oidc.ts -│ │ └── main.tsx -│ └── ... -└── docker-compose.yml +│ │ │ ├── audit.py AuditLog, JobExecution, ExportRecord ORM + helpers +│ │ │ ├── config.py Pydantic settings +│ │ │ ├── db.py PostgreSQL engine + session factory +│ │ │ ├── export.py to_pdf_bytes() via reportlab +│ │ │ ├── executor.py ThreadPoolExecutor +│ │ │ ├── otel.py OTel setup +│ │ │ ├── reports.py save_report() → XLSX + PDF +│ │ │ └── security.py JWT validation +│ │ ├── domain/ +│ │ │ ├── aw/ AW ORM models + persistence helpers +│ │ │ └── wwi/ WWI ORM models + persistence helpers +│ │ ├── routers/ +│ │ │ ├── aw.py /api/aw/** — proxy to Go + PDF exports +│ │ │ ├── wwi.py /api/wwi/** — proxy to Go + PDF exports +│ │ │ └── platform.py /api/config, /api/health, /api/audit, /api/reports +│ │ └── main.py +│ └── analytics/ Go analytics service +│ ├── go.mod +│ ├── .env.example Go analytics local dev +│ └── cmd/server/main.go +│ └── internal/ +│ ├── analytics/ AW + WWI query logic +│ ├── config/ env var loading +│ ├── db/ MSSQL (database/sql) + PostgreSQL (pgxpool) +│ ├── export/ XLSX generation (excelize) +│ ├── handler/ HTTP handlers + route registration +│ ├── persistence/ PostgreSQL writes + audit appends +│ └── scheduler/ robfig/cron job definitions + OTel metrics +└── frontend/ + ├── Dockerfile + ├── nginx.conf + ├── .env.example build-time vars only + └── src/ + ├── api/ API client, types, runtime config fetch + ├── auth/ OIDC (oidc-client-ts) + └── pages/ Dashboard pages ``` --- ## Prerequisites -| Tool | Version | -|---|---| -| Docker + Docker Compose | 24+ | -| Python | 3.11+ (local dev only) | -| uv | latest (local dev only) | -| Node.js | 20+ (local dev only) | -| SQL Server | AdventureWorks DW + WorldWideImporters DW accessible | -| PostgreSQL | 15+ | +| Tool | Min Version | Purpose | +|---|---|---| +| Docker + Docker Compose | 24+ | Run the full stack | +| Go | 1.23+ | Analytics service local dev | +| Python | 3.12+ | API local dev | +| uv | latest | Python package manager | +| Node.js | 22+ | Frontend local dev | +| SQL Server | 2019+ | AdventureWorks DW + WideWorldImporters DW | +| PostgreSQL | 15+ | Write store | --- -## Installation - -### Docker Compose (recommended) +## Quick Start (Docker Compose) ```bash -git clone +git clone ssh://git@git.andric.com.hr:2222/domagoj/zavrsni-rad-otel-app.git cd zavrsni-rad-otel-app -# Copy and fill in environment variables -cp backend/.env.example backend/.env -$EDITOR backend/.env +cp .env.example .env +$EDITOR .env # fill in MSSQL DSNs and PostgreSQL password -# Start everything docker compose up -d - -# View logs -docker compose logs -f api-gateway ``` -The frontend is served at `http://localhost:5173` (Vite dev server in the compose file) or the container port if using the production image. +| Service | URL | +|---|---| +| Frontend | http://localhost:8080 | +| API | http://localhost:8000 | +| Grafana | http://localhost:3000 | -### Local Development +--- -**Backend:** +## Local Development + +### Go analytics service + +```bash +cd backend/analytics +cp .env.example .env +# edit AW_MSSQL_DSN, WWI_MSSQL_DSN, POSTGRES_DSN +set -a && source .env && set +a +go run ./cmd/server +``` + +### Python API ```bash cd backend - -# Install uv if needed -pip install uv - -# Create virtual environment and install all dependencies -uv sync - -# Copy and edit env cp .env.example .env -$EDITOR .env - -# Run services in separate terminals -uvicorn services.api_gateway.main:app --host 0.0.0.0 --port 8000 --reload -uvicorn services.aw_service.main:app --host 0.0.0.0 --port 8001 --reload -uvicorn services.wwi_service.main:app --host 0.0.0.0 --port 8002 --reload +# edit POSTGRES_* and ANALYTICS_SERVICE_URL +uv sync +uv run uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload ``` -**Frontend:** +### Frontend ```bash cd frontend -npm install - -# Only non-OIDC vars needed at build time cp .env.example .env.local -# Set VITE_API_BASE_URL=http://localhost:8000 -# OIDC config is fetched at runtime from GET /api/config - +# edit VITE_API_BASE_URL if the API is not on port 8000 +npm install npm run dev ``` @@ -174,216 +188,203 @@ npm run dev ## Configuration Reference -All backend services share `shared/core/config.py` (Pydantic `BaseSettings`). Values are read from environment variables or `.env`. +### Go analytics service (`otel-bi-analytics`) -### Database +| Variable | Required | Default | Description | +|---|---|---|---| +| `AW_MSSQL_DSN` | Yes | — | AdventureWorks DSN (go-mssqldb format) | +| `WWI_MSSQL_DSN` | Yes | — | WideWorldImporters DSN | +| `POSTGRES_DSN` | Yes | — | PostgreSQL DSN (pgx format) | +| `PORT` | No | `8080` | HTTP listen port | +| `OTEL_COLLECTOR_ENDPOINT` | No | `http://localhost:4318` | OTLP/HTTP endpoint | +| `OTEL_SERVICE_NAME` | No | `otel-bi-analytics` | | +| `OTEL_SERVICE_NAMESPACE` | No | `final-thesis` | | +| `DEFAULT_TOP_N` | No | `10` | Default ranking list length | +| `FORECAST_HORIZON_DAYS` | No | `30` | Default forecast horizon | +| `DEFAULT_HISTORY_DAYS` | No | `365` | Default sales history look-back | -| Variable | Default | Description | -|---|---|---| -| `AW_MSSQL_SERVER` | — | AdventureWorks SQL Server host | -| `AW_MSSQL_DATABASE` | `AdventureWorksDW2022` | Database name | -| `AW_MSSQL_USERNAME` | — | SQL login (read-only recommended) | -| `AW_MSSQL_PASSWORD` | — | SQL password | -| `WWI_MSSQL_SERVER` | — | WorldWideImporters SQL Server host | -| `WWI_MSSQL_DATABASE` | `WideWorldImportersDW` | Database name | -| `WWI_MSSQL_USERNAME` | — | SQL login | -| `WWI_MSSQL_PASSWORD` | — | SQL password | -| `MSSQL_TRUST_SERVER_CERTIFICATE` | `false` | Set `true` for self-signed certs (dev only) | -| `POSTGRES_URL` | — | PostgreSQL connection URL | -| `POSTGRES_SSLMODE` | `prefer` | `require` in production | +### Python API (`otel-bi-api`) -### Security - -| Variable | Default | Description | -|---|---|---| -| `REQUIRE_FRONTEND_AUTH` | `true` | Enforce JWT validation on public endpoints | -| `FRONTEND_JWT_ISSUER_URL` | — | OIDC issuer (e.g. `https://sso.example.com/realms/myapp`) | -| `FRONTEND_JWT_JWKS_URL` | — | JWKS endpoint URL | -| `FRONTEND_JWT_AUDIENCE` | — | Expected `aud` claim | -| `FRONTEND_REQUIRED_SCOPES` | `""` | Space-separated required scopes | -| `INTERNAL_SERVICE_SHARED_SECRET` | — | Shared secret for internal JWT (≥32 bytes) | -| `INTERNAL_SERVICE_ALLOWED_ISSUERS` | `api-gateway` | Accepted internal token issuers | - -### OIDC (served to frontend at runtime) - -| Variable | Default | Description | -|---|---|---| -| `OIDC_ENABLED` | `false` | Enable OIDC in the frontend | -| `OIDC_AUTHORITY` | `""` | OIDC provider base URL | -| `FRONTEND_OIDC_CLIENT_ID` | `""` | Frontend client ID | -| `FRONTEND_OIDC_SCOPE` | `openid profile email` | Requested scopes | - -These are returned by `GET /api/config` (unauthenticated) and consumed by the frontend at boot. They are never baked into the Docker image. - -### Analytics - -| Variable | Default | Description | -|---|---|---| -| `DEFAULT_HISTORY_DAYS` | `365` | Sales history look-back | -| `FORECAST_HORIZON_DAYS` | `30` | Forecast horizon | -| `RANKING_DEFAULT_TOP_N` | `10` | Default ranking list length | -| `STORAGE_DEFAULT_LIMIT` | `100` | Default record list page size | - -### Reports / Exports - -| Variable | Default | Description | -|---|---|---| -| `REPORT_OUTPUT_DIR` | `/tmp/otel-bi-reports` | Directory where full reports are written; mount an SMB/CSI volume here in Kubernetes | - -### Observability - -| Variable | Default | Description | -|---|---|---| -| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4318` | OTLP/HTTP endpoint (Grafana Alloy) | -| `OTEL_SERVICE_NAME` | *(per service)* | Service name in traces | -| `LOG_LEVEL` | `INFO` | Python log level | +| Variable | Required | Default | Description | +|---|---|---|---| +| `ANALYTICS_SERVICE_URL` | No | `http://localhost:8080` | Go analytics base URL | +| `POSTGRES_HOST` | No | `localhost` | | +| `POSTGRES_PORT` | No | `5432` | | +| `POSTGRES_DATABASE` | No | `otel_bi` | | +| `POSTGRES_USERNAME` | No | `otel_bi` | | +| `POSTGRES_PASSWORD` | No | `otel_bi` | | +| `POSTGRES_SSLMODE` | No | `prefer` | Use `require` in production | +| `POSTGRES_CONNECTION_STRING` | No | — | Full DSN override | +| `CORS_ORIGINS` | No | `http://localhost:5173` | Comma-separated allowed origins | +| `REQUIRE_FRONTEND_AUTH` | No | `true` | Enforce JWT on all endpoints | +| `FRONTEND_JWT_ISSUER_URL` | If auth | — | OIDC issuer URL | +| `FRONTEND_JWT_AUDIENCE` | If auth | — | Expected `aud` claim | +| `FRONTEND_JWT_JWKS_URL` | No | — | JWKS URL (derived from issuer if omitted) | +| `FRONTEND_JWT_ALGORITHM` | No | `RS256` | | +| `FRONTEND_REQUIRED_SCOPES` | No | `""` | Space-separated required scopes | +| `FRONTEND_OIDC_CLIENT_ID` | No | `""` | Served to SPA via `GET /api/config` | +| `FRONTEND_OIDC_SCOPE` | No | `openid profile email` | Served to SPA via `GET /api/config` | +| `REPORT_OUTPUT_DIR` | No | `/tmp/otel-bi-reports` | Mount a PVC here in Kubernetes | +| `OTEL_SERVICE_NAME` | No | `otel-bi-api` | | +| `OTEL_SERVICE_NAMESPACE` | No | `final-thesis` | | +| `OTEL_COLLECTOR_ENDPOINT` | No | `http://localhost:4318` | | +| `APP_ENV` | No | `dev` | Set `prod` to disable `/docs` | +| `LOG_LEVEL` | No | `INFO` | | ### Frontend (build-time only) -| Variable | Description | -|---|---| -| `VITE_API_BASE_URL` | Gateway base URL seen by the browser | -| `VITE_OTEL_COLLECTOR_ENDPOINT` | OTLP/HTTP endpoint for frontend traces | -| `VITE_OTEL_SERVICE_NAME` | Frontend service name in traces | -| `VITE_OTEL_SERVICE_NAMESPACE` | Frontend service namespace | +| Variable | Default | Description | +|---|---|---| +| `VITE_API_BASE_URL` | `http://localhost:8000` | API base URL seen by the browser | +| `VITE_OTEL_COLLECTOR_ENDPOINT` | `http://localhost:4318` | OTLP/HTTP for frontend traces | +| `VITE_OTEL_SERVICE_NAME` | `otel-bi-frontend` | | +| `VITE_OTEL_SERVICE_NAMESPACE` | `final-thesis` | | + +OIDC config is **not** a build-time variable — it is fetched at runtime from `GET /api/config`. --- ## Scheduled Jobs -Jobs run automatically on startup. All jobs are recorded in `job_executions` and emit an OTel root span. +All jobs run inside the Go analytics service via robfig/cron. Each job emits an OTel root span and writes a `job_executions` row to PostgreSQL. -| Job ID | Service | Schedule (UTC) | What it does | -|---|---|---|---| -| `aw.daily.forecast` | aw-service | 02:00 daily | Recompute sales forecast | -| `aw.daily.scores` | aw-service | 02:30 daily | Recompute rep + product demand scores | -| `aw.daily.data_quality` | aw-service | 03:00 daily | Data quality checks on AW DW | -| `wwi.hourly.reorder` | wwi-service | :00 every hour | Refresh reorder recommendations | -| `wwi.hourly.events` | wwi-service | :30 every hour | Scan for stock level events | -| `wwi.daily.supplier_scores` | wwi-service | 03:30 daily | Recompute supplier reliability scores | -| `wwi.daily.data_quality` | wwi-service | 04:00 daily | Data quality checks on WWI DW | +| Job ID | Schedule (UTC) | Description | +|---|---|---| +| `aw.daily.forecast` | 02:00 daily | Sales forecast via linear regression | +| `aw.daily.scores` | 02:30 daily | Rep performance + product demand scores | +| `aw.daily.data_quality` | 03:00 daily | AW data quality checks | +| `aw.daily.anomaly_detection` | 03:30 daily | Revenue anomaly detection | +| `wwi.hourly.reorder` | :00 every hour | Reorder recommendations + stock events | +| `wwi.daily.supplier_scores` | 03:30 daily | Supplier reliability scores | +| `wwi.hourly.events` | :30 every hour | HIGH-urgency stock-level event scan | +| `wwi.daily.data_quality` | 04:00 daily | WWI data quality checks | + +**Manual trigger** (authenticated): + +```bash +# AW jobs: forecast | scores | data_quality | anomaly_detection +curl -X POST http://localhost:8000/api/aw/jobs/forecast/trigger \ + -H "Authorization: Bearer $TOKEN" + +# WWI jobs: reorder | supplier_scores | events | data_quality +curl -X POST http://localhost:8000/api/wwi/jobs/reorder/trigger \ + -H "Authorization: Bearer $TOKEN" +``` --- ## API Reference -### Public (require valid Bearer JWT unless `REQUIRE_FRONTEND_AUTH=false`) +All endpoints are on `otel-bi-api` (port 8000). Endpoints marked *proxy* forward the request to `otel-bi-analytics` unchanged. -| Method | Path | Description | -|---|---|---| -| `GET` | `/api/config` | Runtime OIDC config (**unauthenticated**) | -| `GET` | `/api/health` | Gateway health | -| `GET` | `/api/aw/sales/kpis` | AW sales KPIs | -| `GET` | `/api/aw/sales/history` | AW sales history (`?days_back=`) | -| `GET` | `/api/aw/sales/forecast` | AW sales forecast (`?horizon_days=`) | -| `GET` | `/api/aw/reps/scores` | AW rep scores (`?top_n=`) | -| `GET` | `/api/aw/products/demand` | AW product demand (`?top_n=`) | -| `GET` | `/api/wwi/sales/kpis` | WWI sales KPIs | -| `GET` | `/api/wwi/stock/recommendations` | WWI reorder recommendations | -| `GET` | `/api/wwi/suppliers/scores` | WWI supplier scores (`?top_n=`) | -| `GET` | `/api/wwi/events` | WWI business events (`?limit=`) | -| `GET` | `/api/wwi/scenarios` | List what-if scenarios | -| `POST` | `/api/wwi/scenarios` | Create what-if scenario | -| `GET` | `/api/aw/records/forecasts` | Stored AW forecasts | -| `GET` | `/api/aw/records/rep-scores` | Stored AW rep scores | -| `GET` | `/api/aw/records/product-demand` | Stored AW product demand | -| `GET` | `/api/wwi/records/reorder-recommendations` | Stored WWI reorder records | -| `GET` | `/api/wwi/records/supplier-scores` | Stored WWI supplier scores | -| `GET` | `/api/audit` | Audit log (`?domain=aw\|wwi&limit=`) | -| `GET` | `/api/jobs/aw` | AW job history | -| `GET` | `/api/jobs/wwi` | WWI job history | -| `POST` | `/api/reports/generate` | Generate full XLSX + PDF report to shared mount | - -### Per-view Exports (download as attachment) - -| Method | Path | Format | -|---|---|---| -| `GET` | `/api/aw/export/sales-history` | `?format=xlsx\|pdf` | -| `GET` | `/api/aw/export/sales-forecast` | `?format=xlsx\|pdf` | -| `GET` | `/api/aw/export/rep-scores` | `?format=xlsx\|pdf` | -| `GET` | `/api/aw/export/product-demand` | `?format=xlsx\|pdf` | -| `GET` | `/api/wwi/export/stock-recommendations` | `?format=xlsx\|pdf` | -| `GET` | `/api/wwi/export/supplier-scores` | `?format=xlsx\|pdf` | -| `GET` | `/api/wwi/export/business-events` | `?format=xlsx\|pdf` | +| Method | Path | Auth | Description | +|---|---|---|---| +| GET | `/api/config` | No | OIDC config for the SPA | +| GET | `/api/health` | No | Health check | +| GET | `/api/telemetry/status` | Yes | OTel instrumentation info | +| GET | `/api/audit` | Yes | Audit log (`?domain=aw\|wwi&limit=`) | +| GET | `/api/exports` | Yes | Export history (`?domain=aw\|wwi&limit=`) | +| GET | `/api/jobs/aw` | Yes | AW job history | +| GET | `/api/jobs/wwi` | Yes | WWI job history | +| POST | `/api/reports/generate` | Yes | Full XLSX + PDF report | +| GET | `/api/aw/sales/kpis` | Yes | AW KPIs (proxy) | +| GET | `/api/aw/sales/history` | Yes | AW sales history — `?days_back=` (proxy) | +| GET | `/api/aw/sales/forecast` | Yes | AW forecast — `?horizon_days=` (proxy) | +| GET | `/api/aw/reps/scores` | Yes | AW rep scores — `?top_n=` (proxy) | +| GET | `/api/aw/products/demand` | Yes | AW product demand — `?top_n=` (proxy) | +| GET | `/api/aw/export/sales-history` | Yes | `?format=xlsx\|pdf` | +| GET | `/api/aw/export/sales-forecast` | Yes | `?format=xlsx\|pdf` | +| GET | `/api/aw/export/rep-scores` | Yes | `?format=xlsx\|pdf` | +| GET | `/api/aw/export/product-demand` | Yes | `?format=xlsx\|pdf` | +| POST | `/api/aw/jobs/{job}/trigger` | Yes | Trigger AW job immediately | +| GET | `/api/wwi/sales/kpis` | Yes | WWI KPIs (proxy) | +| GET | `/api/wwi/stock/recommendations` | Yes | Reorder recommendations (proxy) | +| GET | `/api/wwi/suppliers/scores` | Yes | Supplier scores — `?top_n=` (proxy) | +| GET | `/api/wwi/export/stock-recommendations` | Yes | `?format=xlsx\|pdf` | +| GET | `/api/wwi/export/supplier-scores` | Yes | `?format=xlsx\|pdf` | +| GET | `/api/wwi/export/business-events` | Yes | `?format=xlsx\|pdf` | +| POST | `/api/wwi/jobs/{job}/trigger` | Yes | Trigger WWI job immediately | --- ## PostgreSQL Schema -| Table | Purpose | Mutability | -|---|---|---| -| `audit_log` | Append-only event trail for every analytics call, job, export, report | Immutable rows | -| `job_executions` | One row per scheduled job run; updated on completion/failure | Mutable | -| `export_records` | Metadata for every per-view download | Immutable rows | -| `aw_forecasts` | Persisted AW sales forecast points | Append | -| `aw_rep_scores` | Persisted AW rep score snapshots | Append | -| `aw_product_demand` | Persisted AW product demand snapshots | Append | -| `wwi_reorder_recommendations` | Persisted WWI reorder recommendations | Append | -| `wwi_supplier_scores` | Persisted WWI supplier score snapshots | Append | -| `wwi_whatif_scenarios` | What-if scenario results | Append | -| `wwi_business_events` | Stock-level business events | Append | +Tables are created automatically on API startup via SQLAlchemy `metadata.create_all()`. -All tables are created automatically on service startup via `metadata.create_all()` (idempotent). +| Table | Purpose | +|---|---| +| `audit_log` | Append-only event trail | +| `job_executions` | One row per scheduled job run | +| `export_records` | Per-view download metadata | +| `aw_forecasts` | Persisted AW forecast points | +| `aw_rep_scores` | AW rep score snapshots | +| `aw_product_demand` | AW product demand snapshots | +| `aw_anomaly_runs` | AW anomaly detection results | +| `wwi_reorder_recommendations` | WWI reorder snapshots | +| `wwi_supplier_scores` | WWI supplier score snapshots | +| `wwi_business_events` | Stock-level business events | --- -## CI/CD (Gitea Actions) +## CI/CD -`.gitea/workflows/docker-publish.yml` builds and pushes all four images in parallel using a matrix strategy. +`.gitea/workflows/ci.yml` — three independent pipelines on push to `master` or a version tag. Test jobs run inside `rockylinux/rockylinux:10` containers using shell-based checkout. -**Required repository variables (`vars.`):** +| Pipeline | Test job | Build job | Image | +|---|---|---|---| +| Python API | `test` (uv + pytest) | `build-api` | `domagoj/otel-bi-api` | +| Go analytics | `test-analytics` (go vet + go test) | `build-analytics` | `domagoj/otel-bi-analytics` | +| Frontend | `test` | `build-frontend` | `domagoj/otel-bi-frontend` | -| Name | Example | -|---|---| -| `REGISTRY` | `registry.example.com` | -| `IMAGE_PREFIX` | `myorg/otel-bi` | - -**Required repository secrets (`secrets.`):** - -| Name | Description | +**Required repository secrets:** + +| Secret | Description | |---|---| +| `REGISTRY_HOST` | Container registry hostname | | `REGISTRY_USERNAME` | Registry login | -| `REGISTRY_PASSWORD` | Registry password / token | - -Images are tagged with `sha-`, branch name, semver (on tags), and `latest` on pushes to `master`. Push is skipped on pull requests. +| `REGISTRY_TOKEN` | Registry PAT (packages read + write) | --- ## OTel Coverage **Frontend:** -- W3C `traceparent`/`tracestate` propagation on all fetch calls +- W3C `traceparent` / `tracestate` propagation on all fetch calls - `@opentelemetry/instrumentation-document-load` -- `@opentelemetry/instrumentation-user-interaction` -- `@opentelemetry/instrumentation-fetch` / XHR -- Manual spans around dashboard aggregation +- `@opentelemetry/instrumentation-fetch` +- Manual spans on dashboard data aggregation -**Backend (all services):** +**Go analytics service:** +- `otelhttp` HTTP server auto-instrumentation +- Manual spans on all analytics and export functions +- Per-job root span (`trace.WithNewRoot()`) for independent trace trees +- OTel metrics: `scheduler.job.duration_seconds`, `scheduler.job.success_total`, `scheduler.job.failure_total`, `scheduler.job.records_processed_total` + +**Python API:** - FastAPI auto-instrumentation (request span per endpoint) -- SQLAlchemy auto-instrumentation (MSSQL + PostgreSQL) -- HTTPX auto-instrumentation (service-to-service calls) -- Manual root spans per scheduled job -- Manual spans on analytics functions -- `x-trace-id` / `x-span-id` response headers on internal endpoints +- SQLAlchemy auto-instrumentation (PostgreSQL) +- HTTPX auto-instrumentation (calls to Go analytics) +- `x-trace-id` / `x-span-id` response headers +- W3C context propagation injected into all Go analytics calls --- ## Read-Only Guarantee (MSSQL) -- `ApplicationIntent=ReadOnly` on all MSSQL connection strings -- Query layer only accepts `SELECT` / `WITH` statements -- All write operations target PostgreSQL only -- Use a SQL Server login with `SELECT` grants only — no DDL or DML permissions needed +- `ApplicationIntent=ReadOnly` on all MSSQL DSNs +- SQL layer only executes `SELECT` / `WITH` statements +- All writes target PostgreSQL only +- Use a SQL Server login with `db_datareader` only — no DDL or DML permissions needed --- ## Verification Checklist -1. `GET /api/config` returns OIDC settings without authentication. -2. `GET /api/health` returns `{"status": "ok"}`. -3. After login, `GET /api/aw/sales/kpis` returns data. -4. `GET /api/audit` shows rows for the KPI call. -5. `GET /api/jobs/aw` shows scheduled job runs (populated after the next cron tick or after ~1 minute if jobs fired on startup). -6. `GET /api/aw/export/sales-forecast?format=xlsx` downloads an `.xlsx` file. -7. `POST /api/reports/generate` returns paths to `.xlsx` and `.pdf` files; confirm they appear in `REPORT_OUTPUT_DIR`. -8. In Grafana Tempo, verify a trace spans `api-gateway → aw-service → PostgreSQL/MSSQL`. +1. `GET /api/health` → `{"status": "ok", "service": "otel-bi-backend"}` +2. `GET /api/config` → OIDC config without a token +3. After login, `GET /api/aw/sales/kpis` → data rows +4. `GET /api/audit` → rows for the KPI call +5. `GET /api/jobs/aw` → job run records (after cron tick or manual trigger) +6. `GET /api/aw/export/sales-forecast?format=xlsx` → downloads `.xlsx` +7. `POST /api/reports/generate` → paths to `.xlsx` and `.pdf` in `REPORT_OUTPUT_DIR` +8. Grafana Tempo → trace spanning `otel-bi-api → otel-bi-analytics → MSSQL / PostgreSQL` diff --git a/backend/.env.example b/backend/.env.example deleted file mode 100644 index d075492..0000000 --- a/backend/.env.example +++ /dev/null @@ -1,68 +0,0 @@ -# --------------------------------------------------------------------------- -# OTel BI Backend — local development (without Docker) -# Copy to .env and fill in your values. -# Run services from the backend/ directory so pydantic-settings finds .env. -# --------------------------------------------------------------------------- - -APP_ENV=dev -LOG_LEVEL=INFO - -# ============================================================ -# Go analytics service (same image, ROLE=analytics) -# Set this to wherever the analytics container is reachable. -# ============================================================ -ANALYTICS_SERVICE_URL=http://localhost:8080 - -# MSSQL — required when ROLE=analytics -# go-mssqldb DSN: sqlserver://user:pass@host:port?database=name&... -AW_MSSQL_DSN=sqlserver://sa:YourStrongPassword123!@localhost:1433?database=AdventureWorksDW2022&TrustServerCertificate=true&ApplicationIntent=ReadOnly -WWI_MSSQL_DSN=sqlserver://sa:YourStrongPassword123!@localhost:1433?database=WideWorldImportersDW&TrustServerCertificate=true&ApplicationIntent=ReadOnly - -# ============================================================ -# PostgreSQL — write store for derived data -# ============================================================ -POSTGRES_HOST=localhost -POSTGRES_PORT=5432 -POSTGRES_DATABASE=otel_bi -POSTGRES_USERNAME=otel_bi -POSTGRES_PASSWORD=otel_bi_dev -# prefer for dev, require for production -POSTGRES_SSLMODE=prefer - -# Optional: override the generated connection URL directly -# POSTGRES_CONNECTION_STRING=postgresql+psycopg://otel_bi:otel_bi_dev@localhost:5432/otel_bi?sslmode=prefer - -# ============================================================ -# Frontend JWT validation -# Validates the Bearer token the browser sends on every request. -# ============================================================ -# Set false to disable auth entirely (dev only) -REQUIRE_FRONTEND_AUTH=false - -# When REQUIRE_FRONTEND_AUTH=true, fill in your OIDC provider: -# FRONTEND_JWT_ISSUER_URL=https://your-idp.example.com/realms/your-realm -# FRONTEND_JWT_AUDIENCE=your-api-audience -# FRONTEND_JWT_JWKS_URL=https://your-idp.example.com/realms/your-realm/protocol/openid-connect/certs -# FRONTEND_REQUIRED_SCOPES=openid profile - -# ============================================================ -# Frontend OIDC runtime config (served to the SPA via GET /api/config -# — NOT baked into the JS bundle) -# ============================================================ -# FRONTEND_OIDC_CLIENT_ID=otel-bi-frontend -# FRONTEND_OIDC_SCOPE=openid profile email - -CORS_ORIGINS=http://localhost:5173 - -# ============================================================ -# Reports — filesystem path for generated XLSX + PDF files -# Mount a K8s CSI / SMB PVC here in production. -# ============================================================ -REPORT_OUTPUT_DIR=/tmp/otel-bi-reports - -# ============================================================ -# OpenTelemetry -# ============================================================ -OTEL_SERVICE_NAME=otel-bi-backend -OTEL_SERVICE_NAMESPACE=final-thesis -OTEL_COLLECTOR_ENDPOINT=http://localhost:4318 diff --git a/frontend/.env.example b/frontend/.env.example deleted file mode 100644 index 425befe..0000000 --- a/frontend/.env.example +++ /dev/null @@ -1,19 +0,0 @@ -# --------------------------------------------------------------------------- -# OTel BI Frontend — build-time variables only -# Copy to .env.local for local dev. -# -# OIDC configuration is NOT set here. The frontend fetches it at runtime -# from GET /api/config on the gateway, which reads it from the gateway's -# own environment variables. Nothing OIDC-related is baked into the bundle. -# --------------------------------------------------------------------------- - -# URL the browser uses to reach the API gateway -VITE_API_BASE_URL=http://localhost:8000 - -# OpenTelemetry collector endpoint (Grafana Alloy OTLP/HTTP) -VITE_OTEL_COLLECTOR_ENDPOINT=http://localhost:4318 -# K8s + Alloy example: -# VITE_OTEL_COLLECTOR_ENDPOINT=http://alloy.monitoring.svc.cluster.local:4318 - -VITE_OTEL_SERVICE_NAME=otel-bi-frontend -VITE_OTEL_SERVICE_NAMESPACE=final-thesis