Push the rest
This commit is contained in:
0
backend/app/domain/aw/__init__.py
Normal file
0
backend/app/domain/aw/__init__.py
Normal file
258
backend/app/domain/aw/analytics.py
Normal file
258
backend/app/domain/aw/analytics.py
Normal file
@@ -0,0 +1,258 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from sqlalchemy.orm import sessionmaker, Session
|
||||
|
||||
from app.core.audit import append_audit
|
||||
from app.domain.aw.models import AWSalesForecast, AWRepScore, AWProductDemand, AWAnomalyRun
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
tracer = trace.get_tracer("otel-bi.domain.aw")
|
||||
meter = metrics.get_meter("otel-bi.domain.aw")
|
||||
|
||||
_persist_counter = meter.create_counter(
|
||||
"aw_persist_writes_total",
|
||||
description="Number of AW PostgreSQL write operations",
|
||||
)
|
||||
|
||||
|
||||
def _current_span_context() -> tuple[str | None, str | None]:
|
||||
ctx = trace.get_current_span().get_span_context()
|
||||
if not ctx.is_valid:
|
||||
return None, None
|
||||
return f"{ctx.trace_id:032x}", f"{ctx.span_id:016x}"
|
||||
|
||||
|
||||
def _actor_type(trigger_source: str) -> str:
|
||||
return "scheduler" if trigger_source.startswith("scheduler") else "api"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Persist functions — called after Go service returns data
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def persist_forecast(
|
||||
factory: sessionmaker[Session],
|
||||
data: list[dict],
|
||||
horizon_days: int,
|
||||
trigger_source: str,
|
||||
) -> None:
|
||||
trace_id, span_id = _current_span_context()
|
||||
try:
|
||||
with factory() as session:
|
||||
session.add(AWSalesForecast(
|
||||
horizon_days=horizon_days,
|
||||
point_count=len(data),
|
||||
trigger_source=trigger_source,
|
||||
trace_id=trace_id,
|
||||
span_id=span_id,
|
||||
payload=data,
|
||||
))
|
||||
session.commit()
|
||||
_persist_counter.add(1, {"entity": "sales_forecast"})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
LOGGER.warning("Failed to persist AW forecast: %s", exc)
|
||||
append_audit(
|
||||
factory,
|
||||
action="forecast.generated",
|
||||
actor_type=_actor_type(trigger_source),
|
||||
actor_id=trigger_source,
|
||||
domain="aw",
|
||||
service="otel-bi-backend",
|
||||
entity_type="sales_forecast",
|
||||
payload={"horizon_days": horizon_days, "point_count": len(data)},
|
||||
)
|
||||
|
||||
|
||||
def persist_rep_scores(
|
||||
factory: sessionmaker[Session],
|
||||
data: list[dict],
|
||||
top_n: int,
|
||||
trigger_source: str,
|
||||
) -> None:
|
||||
trace_id, span_id = _current_span_context()
|
||||
try:
|
||||
with factory() as session:
|
||||
session.add(AWRepScore(
|
||||
rep_count=len(data),
|
||||
trigger_source=trigger_source,
|
||||
trace_id=trace_id,
|
||||
span_id=span_id,
|
||||
payload=data,
|
||||
))
|
||||
session.commit()
|
||||
_persist_counter.add(1, {"entity": "rep_scores"})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
LOGGER.warning("Failed to persist AW rep scores: %s", exc)
|
||||
append_audit(
|
||||
factory,
|
||||
action="scores.generated",
|
||||
actor_type=_actor_type(trigger_source),
|
||||
actor_id=trigger_source,
|
||||
domain="aw",
|
||||
service="otel-bi-backend",
|
||||
entity_type="rep_scores",
|
||||
payload={"rep_count": len(data), "top_n": top_n},
|
||||
)
|
||||
|
||||
|
||||
def persist_product_demand(
|
||||
factory: sessionmaker[Session],
|
||||
data: list[dict],
|
||||
top_n: int,
|
||||
trigger_source: str,
|
||||
) -> None:
|
||||
trace_id, span_id = _current_span_context()
|
||||
try:
|
||||
with factory() as session:
|
||||
session.add(AWProductDemand(
|
||||
product_count=len(data),
|
||||
top_n=top_n,
|
||||
trigger_source=trigger_source,
|
||||
trace_id=trace_id,
|
||||
span_id=span_id,
|
||||
payload=data,
|
||||
))
|
||||
session.commit()
|
||||
_persist_counter.add(1, {"entity": "product_demand"})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
LOGGER.warning("Failed to persist AW product demand: %s", exc)
|
||||
append_audit(
|
||||
factory,
|
||||
action="scores.generated",
|
||||
actor_type=_actor_type(trigger_source),
|
||||
actor_id=trigger_source,
|
||||
domain="aw",
|
||||
service="otel-bi-backend",
|
||||
entity_type="product_demand",
|
||||
payload={"product_count": len(data), "top_n": top_n},
|
||||
)
|
||||
|
||||
|
||||
def persist_anomaly_run(
|
||||
factory: sessionmaker[Session],
|
||||
data: list[dict],
|
||||
trigger_source: str,
|
||||
) -> None:
|
||||
anomaly_count = sum(1 for p in data if p.get("is_anomaly"))
|
||||
trace_id, span_id = _current_span_context()
|
||||
try:
|
||||
with factory() as session:
|
||||
session.add(AWAnomalyRun(
|
||||
anomaly_count=anomaly_count,
|
||||
series_days=365,
|
||||
window_days=30,
|
||||
threshold_sigma=2.0,
|
||||
trigger_source=trigger_source,
|
||||
trace_id=trace_id,
|
||||
span_id=span_id,
|
||||
payload=data,
|
||||
))
|
||||
session.commit()
|
||||
_persist_counter.add(1, {"entity": "anomaly_run"})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
LOGGER.warning("Failed to persist AW anomaly run: %s", exc)
|
||||
append_audit(
|
||||
factory,
|
||||
action="anomaly_detection.ran",
|
||||
actor_type=_actor_type(trigger_source),
|
||||
actor_id=trigger_source,
|
||||
domain="aw",
|
||||
service="otel-bi-backend",
|
||||
entity_type="anomaly_detection",
|
||||
payload={"series_days": 365, "window_days": 30, "anomaly_count": anomaly_count},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Read functions — query PostgreSQL for stored results
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def list_forecasts(factory: sessionmaker[Session], limit: int = 50) -> list[dict]:
|
||||
with factory() as session:
|
||||
rows = (
|
||||
session.query(AWSalesForecast)
|
||||
.order_by(AWSalesForecast.created_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"created_at": r.created_at.isoformat(),
|
||||
"horizon_days": r.horizon_days,
|
||||
"point_count": r.point_count,
|
||||
"trigger_source": r.trigger_source,
|
||||
"trace_id": r.trace_id,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def list_rep_scores(factory: sessionmaker[Session], limit: int = 50) -> list[dict]:
|
||||
with factory() as session:
|
||||
rows = (
|
||||
session.query(AWRepScore)
|
||||
.order_by(AWRepScore.computed_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"computed_at": r.computed_at.isoformat(),
|
||||
"rep_count": r.rep_count,
|
||||
"trigger_source": r.trigger_source,
|
||||
"trace_id": r.trace_id,
|
||||
"payload": r.payload,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def list_product_demand(factory: sessionmaker[Session], limit: int = 50) -> list[dict]:
|
||||
with factory() as session:
|
||||
rows = (
|
||||
session.query(AWProductDemand)
|
||||
.order_by(AWProductDemand.computed_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"computed_at": r.computed_at.isoformat(),
|
||||
"product_count": r.product_count,
|
||||
"top_n": r.top_n,
|
||||
"trigger_source": r.trigger_source,
|
||||
"trace_id": r.trace_id,
|
||||
"payload": r.payload,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def list_anomaly_runs(factory: sessionmaker[Session], limit: int = 20) -> list[dict]:
|
||||
with factory() as session:
|
||||
rows = (
|
||||
session.query(AWAnomalyRun)
|
||||
.order_by(AWAnomalyRun.detected_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"detected_at": r.detected_at.isoformat(),
|
||||
"anomaly_count": r.anomaly_count,
|
||||
"series_days": r.series_days,
|
||||
"window_days": r.window_days,
|
||||
"threshold_sigma": r.threshold_sigma,
|
||||
"trigger_source": r.trigger_source,
|
||||
"trace_id": r.trace_id,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
77
backend/app/domain/aw/models.py
Normal file
77
backend/app/domain/aw/models.py
Normal file
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Integer, String
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class AWBase(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class AWSalesForecast(AWBase):
|
||||
"""Persisted AW sales forecast runs."""
|
||||
|
||||
__tablename__ = "aw_sales_forecasts"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, index=True)
|
||||
horizon_days: Mapped[int] = mapped_column(Integer)
|
||||
point_count: Mapped[int] = mapped_column(Integer)
|
||||
trigger_source: Mapped[str] = mapped_column(String(64), index=True)
|
||||
trace_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||
span_id: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
payload: Mapped[list[dict]] = mapped_column(JSON, default=list)
|
||||
|
||||
|
||||
class AWRepScore(AWBase):
|
||||
"""Persisted AW sales rep performance scoring runs."""
|
||||
|
||||
__tablename__ = "aw_rep_scores"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
computed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, index=True)
|
||||
rep_count: Mapped[int] = mapped_column(Integer)
|
||||
trigger_source: Mapped[str] = mapped_column(String(64), index=True)
|
||||
trace_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||
span_id: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
payload: Mapped[list[dict]] = mapped_column(JSON, default=list)
|
||||
|
||||
|
||||
class AWProductDemand(AWBase):
|
||||
"""Persisted AW product demand scoring runs."""
|
||||
|
||||
__tablename__ = "aw_product_demand"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
computed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, index=True)
|
||||
product_count: Mapped[int] = mapped_column(Integer)
|
||||
top_n: Mapped[int] = mapped_column(Integer)
|
||||
trigger_source: Mapped[str] = mapped_column(String(64), index=True)
|
||||
trace_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||
span_id: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
payload: Mapped[list[dict]] = mapped_column(JSON, default=list)
|
||||
|
||||
|
||||
class AWAnomalyRun(AWBase):
|
||||
"""Persisted AW revenue anomaly detection runs."""
|
||||
|
||||
__tablename__ = "aw_anomaly_runs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
detected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, index=True)
|
||||
anomaly_count: Mapped[int] = mapped_column(Integer)
|
||||
series_days: Mapped[int] = mapped_column(Integer)
|
||||
window_days: Mapped[int] = mapped_column(Integer)
|
||||
threshold_sigma: Mapped[float] = mapped_column(default=2.0)
|
||||
trigger_source: Mapped[str] = mapped_column(String(64), index=True)
|
||||
trace_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||
span_id: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
# Full annotated series (date, revenue, rolling_mean, lower_band, upper_band, is_anomaly, z_score)
|
||||
payload: Mapped[list[dict]] = mapped_column(JSON, default=list)
|
||||
131
backend/app/domain/aw/queries.py
Normal file
131
backend/app/domain/aw/queries.py
Normal file
@@ -0,0 +1,131 @@
|
||||
from __future__ import annotations
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AdventureWorksDW2022 — read-only MSSQL queries
|
||||
# Each list contains fallback variants tried in order.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Daily sales combining FactInternetSales + FactResellerSales
|
||||
AW_DAILY_SALES: list[str] = [
|
||||
"""
|
||||
SELECT
|
||||
CAST(d.FullDateAlternateKey AS date) AS sale_date,
|
||||
SUM(f.SalesAmount) AS revenue,
|
||||
SUM(f.TotalProductCost) AS cost,
|
||||
SUM(f.OrderQuantity) AS quantity,
|
||||
COUNT_BIG(*) AS orders
|
||||
FROM dbo.FactInternetSales AS f
|
||||
INNER JOIN dbo.DimDate AS d ON d.DateKey = f.OrderDateKey
|
||||
GROUP BY CAST(d.FullDateAlternateKey AS date)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
CAST(d.FullDateAlternateKey AS date) AS sale_date,
|
||||
SUM(r.SalesAmount) AS revenue,
|
||||
SUM(r.TotalProductCost) AS cost,
|
||||
SUM(r.OrderQuantity) AS quantity,
|
||||
COUNT_BIG(*) AS orders
|
||||
FROM dbo.FactResellerSales AS r
|
||||
INNER JOIN dbo.DimDate AS d ON d.DateKey = r.OrderDateKey
|
||||
GROUP BY CAST(d.FullDateAlternateKey AS date)
|
||||
|
||||
ORDER BY sale_date;
|
||||
""",
|
||||
# Fallback: internet sales only using OrderDate column directly
|
||||
"""
|
||||
SELECT
|
||||
CAST(OrderDate AS date) AS sale_date,
|
||||
SUM(SalesAmount) AS revenue,
|
||||
SUM(TotalProductCost) AS cost,
|
||||
SUM(OrderQuantity) AS quantity,
|
||||
COUNT_BIG(*) AS orders
|
||||
FROM dbo.FactInternetSales
|
||||
GROUP BY CAST(OrderDate AS date)
|
||||
ORDER BY sale_date;
|
||||
""",
|
||||
]
|
||||
|
||||
# Sales rep performance — reseller sales attributed to employees
|
||||
AW_REP_PERFORMANCE: list[str] = [
|
||||
"""
|
||||
SELECT
|
||||
e.EmployeeKey AS employee_key,
|
||||
e.FirstName + ' ' + e.LastName AS rep_name,
|
||||
COALESCE(e.Title, 'Sales Rep') AS rep_title,
|
||||
COALESCE(st.SalesTerritoryRegion, 'Unknown') AS territory,
|
||||
SUM(r.SalesAmount) AS revenue,
|
||||
SUM(r.TotalProductCost) AS cost,
|
||||
COUNT_BIG(*) AS orders,
|
||||
AVG(r.SalesAmount) AS avg_deal_size
|
||||
FROM dbo.FactResellerSales AS r
|
||||
INNER JOIN dbo.DimEmployee AS e
|
||||
ON e.EmployeeKey = r.EmployeeKey
|
||||
INNER JOIN dbo.DimSalesTerritory AS st
|
||||
ON st.SalesTerritoryKey = r.SalesTerritoryKey
|
||||
WHERE e.SalesPersonFlag = 1
|
||||
GROUP BY
|
||||
e.EmployeeKey,
|
||||
e.FirstName, e.LastName,
|
||||
e.Title,
|
||||
st.SalesTerritoryRegion
|
||||
ORDER BY revenue DESC;
|
||||
""",
|
||||
# Fallback without SalesPersonFlag filter
|
||||
"""
|
||||
SELECT
|
||||
e.EmployeeKey AS employee_key,
|
||||
e.FirstName + ' ' + e.LastName AS rep_name,
|
||||
COALESCE(e.Title, 'Employee') AS rep_title,
|
||||
'Unknown' AS territory,
|
||||
SUM(r.SalesAmount) AS revenue,
|
||||
SUM(r.TotalProductCost) AS cost,
|
||||
COUNT_BIG(*) AS orders,
|
||||
AVG(r.SalesAmount) AS avg_deal_size
|
||||
FROM dbo.FactResellerSales AS r
|
||||
INNER JOIN dbo.DimEmployee AS e ON e.EmployeeKey = r.EmployeeKey
|
||||
GROUP BY e.EmployeeKey, e.FirstName, e.LastName, e.Title
|
||||
ORDER BY revenue DESC;
|
||||
""",
|
||||
]
|
||||
|
||||
# Product demand — internet sales with full category hierarchy
|
||||
AW_PRODUCT_DEMAND: list[str] = [
|
||||
"""
|
||||
SELECT
|
||||
p.ProductAlternateKey AS product_id,
|
||||
p.EnglishProductName AS product_name,
|
||||
COALESCE(pc.EnglishProductCategoryName, 'Unknown') AS category,
|
||||
SUM(f.SalesAmount) AS revenue,
|
||||
SUM(f.TotalProductCost) AS cost,
|
||||
SUM(f.OrderQuantity) AS quantity,
|
||||
COUNT_BIG(*) AS orders
|
||||
FROM dbo.FactInternetSales AS f
|
||||
INNER JOIN dbo.DimProduct AS p
|
||||
ON p.ProductKey = f.ProductKey
|
||||
LEFT JOIN dbo.DimProductSubcategory AS sc
|
||||
ON sc.ProductSubcategoryKey = p.ProductSubcategoryKey
|
||||
LEFT JOIN dbo.DimProductCategory AS pc
|
||||
ON pc.ProductCategoryKey = sc.ProductCategoryKey
|
||||
GROUP BY
|
||||
p.ProductAlternateKey,
|
||||
p.EnglishProductName,
|
||||
pc.EnglishProductCategoryName
|
||||
ORDER BY revenue DESC;
|
||||
""",
|
||||
# Fallback: no category join
|
||||
"""
|
||||
SELECT
|
||||
CAST(f.ProductKey AS nvarchar(50)) AS product_id,
|
||||
COALESCE(p.EnglishProductName, CAST(f.ProductKey AS nvarchar(50))) AS product_name,
|
||||
'Unknown' AS category,
|
||||
SUM(f.SalesAmount) AS revenue,
|
||||
SUM(f.TotalProductCost) AS cost,
|
||||
SUM(f.OrderQuantity) AS quantity,
|
||||
COUNT_BIG(*) AS orders
|
||||
FROM dbo.FactInternetSales AS f
|
||||
LEFT JOIN dbo.DimProduct AS p ON p.ProductKey = f.ProductKey
|
||||
GROUP BY f.ProductKey, p.EnglishProductName
|
||||
ORDER BY revenue DESC;
|
||||
""",
|
||||
]
|
||||
Reference in New Issue
Block a user