Files
bellsystems-cp/backend/mqtt/router.py
T
bonaminandClaude Opus 5.5 963516ece0 feat(mqtt): store F-070 crash detail and group crashes fleet-wide
Firmware F-070 adds a fuller `crash` object (backtrace, backtrace_corrupted,
elf_sha256) and a new `pre_crash` snapshot (uptime, heap stats, optional
abort_msg) to boot_report and to telemetry.get_boot_history entries.

- Migration c9d0e1f2a3b4: device_boot_events gains crash / pre_crash JSONB,
  stored verbatim so later additive fields need no migration. The legacy
  crash_* columns are still filled; old rows and old firmware are unchanged.
  JSON is bound as CAST(CAST(:x AS TEXT) AS JSONB): a bare JSONB cast makes
  asyncpg JSON-encode the already-encoded string a second time.
- boot_report ingestion passes both objects through.
- telemetry.get_boot_history replies (control/ack) are merged into boot
  history whoever sent the command: an entry matches an existing row on
  boot_count + reset_reason + time within 30 min (boot_count alone is not
  unique - the counter gets reset), and only fills crash/pre_crash the row
  lacks; unmatched entries are boots we never saw live and are inserted at
  the device's timestamp; entries without ts are skipped.
- GET /api/mqtt/crash-groups: fault boots grouped by abort_msg with hex
  addresses stripped, else task + exception cause, else reset reason. When
  abort_msg is present, pc/exc_cause describe abort() itself and are
  ignored for grouping.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 16:28:40 +03:00

289 lines
12 KiB
Python

from fastapi import APIRouter, Depends, Query, WebSocket, WebSocketDisconnect
from typing import Optional, List
from auth.models import TokenPayload
from auth.dependencies import require_permission
from mqtt.models import (
MqttCommandRequest, CommandSendResponse, MqttStatusResponse,
DeviceMqttStatus, LogListResponse, HeartbeatListResponse,
CommandListResponse, HeartbeatEntry, AlertEventEntry,
AlertEventListResponse, BootEventListResponse, PingSampleListResponse,
DiagnosticsReportListResponse, LatestMetricsResponse,
LatestDiagnosticsEntry, LatestPingEntry, DeviceReportListResponse,
CrashGroupListResponse,
)
from mqtt.crash_groups import group_crashes
from mqtt.client import mqtt_manager
from mqtt import presence
import database as db
from datetime import datetime, timezone
router = APIRouter(prefix="/api/mqtt", tags=["mqtt"])
@router.get("/status", response_model=MqttStatusResponse)
async def get_all_device_status(
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
heartbeats = await db.get_latest_heartbeats()
alert_events = await db.get_latest_alert_events()
alert_by_serial = {a["device_serial"]: a for a in alert_events}
now = datetime.now(timezone.utc)
devices = []
for hb in heartbeats:
received_str = hb["received_at"]
try:
received = datetime.fromisoformat(received_str)
if received.tzinfo is None:
received = received.replace(tzinfo=timezone.utc)
seconds_ago = int((now - received).total_seconds())
except (ValueError, TypeError):
seconds_ago = 9999
alert_event = alert_by_serial.get(hb["device_serial"])
devices.append(DeviceMqttStatus(
device_serial=hb["device_serial"],
online=seconds_ago < 90 and not presence.is_marked_offline(hb["device_serial"]),
last_heartbeat=HeartbeatEntry(**hb),
seconds_since_heartbeat=seconds_ago,
last_alert_event=AlertEventEntry(**alert_event) if alert_event else None,
))
return MqttStatusResponse(
devices=devices,
broker_connected=mqtt_manager.connected,
)
@router.get("/latest-metrics", response_model=LatestMetricsResponse)
async def get_latest_metrics(
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
# Fleet-wide "last known" CPU temp + ping RTT, one query each — used by
# DeviceList to render optional columns without polling any device.
# Uptime/firmware/RSSI don't need this: they're already in /mqtt/status.
diag_reports = await db.get_latest_diagnostics_reports()
ping_samples = await db.get_latest_ping_samples()
return LatestMetricsResponse(
diagnostics=[LatestDiagnosticsEntry(**d) for d in diag_reports],
pings=[LatestPingEntry(**p) for p in ping_samples],
)
@router.post("/command/{device_serial}", response_model=CommandSendResponse)
async def send_command(
device_serial: str,
body: MqttCommandRequest,
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
command_id = await db.insert_command(
device_serial=device_serial,
command_name=body.cmd,
command_payload={"cmd": body.cmd, "contents": body.contents},
)
success = mqtt_manager.publish_command(
device_serial=device_serial,
cmd=body.cmd,
contents=body.contents,
)
if not success:
await db.update_command_response(
command_id, "error",
{"error": "MQTT broker not connected"},
)
return CommandSendResponse(
success=False, command_id=command_id,
message="MQTT broker not connected",
)
return CommandSendResponse(
success=True, command_id=command_id,
message=f"Command '{body.cmd}' sent to {device_serial}",
)
@router.get("/logs/{device_serial}", response_model=LogListResponse)
async def get_device_logs(
device_serial: str,
level: Optional[str] = Query(None, description="Filter: INFO, WARN, ERROR"),
min_level: bool = Query(False, description="If true, level is a floor — also includes higher-severity levels"),
search: Optional[str] = Query(None),
source: Optional[List[str]] = Query(None, description="Filter by source: log, info. Repeat param to include several."),
limit: int = Query(100, ge=1, le=1000),
offset: int = Query(0, ge=0),
since: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/after this time"),
until: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/before this time"),
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
logs, total = await db.get_logs(
device_serial, level=level, search=search, source=source,
min_level=min_level, limit=limit, offset=offset, since=since, until=until,
)
return LogListResponse(logs=logs, total=total)
@router.get("/heartbeats/{device_serial}", response_model=HeartbeatListResponse)
async def get_device_heartbeats(
device_serial: str,
limit: int = Query(100, ge=1, le=5000),
offset: int = Query(0, ge=0),
since: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/after this time"),
until: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/before this time"),
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
heartbeats, total = await db.get_heartbeats(
device_serial, limit=limit, offset=offset, since=since, until=until,
)
return HeartbeatListResponse(heartbeats=heartbeats, total=total)
@router.get("/commands/{device_serial}", response_model=CommandListResponse)
async def get_device_commands(
device_serial: str,
limit: int = Query(100, ge=1, le=1000),
offset: int = Query(0, ge=0),
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
commands, total = await db.get_commands(
device_serial, limit=limit, offset=offset,
)
return CommandListResponse(commands=commands, total=total)
@router.get("/alert-events/{device_serial}", response_model=AlertEventListResponse)
async def get_device_alert_events(
device_serial: str,
limit: int = Query(100, ge=1, le=1000),
offset: int = Query(0, ge=0),
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
events, total = await db.get_alert_events(
device_serial, limit=limit, offset=offset,
)
return AlertEventListResponse(events=events, total=total)
@router.get("/boot-events/{device_serial}", response_model=BootEventListResponse)
async def get_device_boot_events(
device_serial: str,
limit: int = Query(100, ge=1, le=2000),
offset: int = Query(0, ge=0),
since: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/after this time"),
until: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/before this time"),
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
events, total = await db.get_boot_events(
device_serial, limit=limit, offset=offset, since=since, until=until,
)
return BootEventListResponse(events=events, total=total)
@router.get("/crash-groups", response_model=CrashGroupListResponse)
async def get_crash_groups(
since: Optional[datetime] = Query(None, description="ISO timestamp — only crashes at/after this time"),
until: Optional[datetime] = Query(None, description="ISO timestamp — only crashes at/before this time"),
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
"""Fault boots across the whole fleet, grouped by crash signature."""
events = await db.get_crash_events(since=since, until=until)
return CrashGroupListResponse(groups=group_crashes(events), total_crashes=len(events))
@router.get("/ping-samples/{device_serial}", response_model=PingSampleListResponse)
async def get_device_ping_samples(
device_serial: str,
limit: int = Query(200, ge=1, le=5000),
offset: int = Query(0, ge=0),
since: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/after this time"),
until: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/before this time"),
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
samples, total = await db.get_ping_samples(
device_serial, limit=limit, offset=offset, since=since, until=until,
)
return PingSampleListResponse(samples=samples, total=total)
@router.get("/diagnostics-reports/{device_serial}", response_model=DiagnosticsReportListResponse)
async def get_device_diagnostics_reports(
device_serial: str,
limit: int = Query(200, ge=1, le=5000),
offset: int = Query(0, ge=0),
since: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/after this time"),
until: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/before this time"),
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
reports, total = await db.get_diagnostics_reports(
device_serial, limit=limit, offset=offset, since=since, until=until,
)
return DiagnosticsReportListResponse(reports=reports, total=total)
@router.get("/reports/{device_serial}", response_model=DeviceReportListResponse)
async def get_device_reports(
device_serial: str,
limit: int = Query(200, ge=1, le=2000),
offset: int = Query(0, ge=0),
since: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/after this time"),
until: Optional[datetime] = Query(None, description="ISO timestamp — only rows at/before this time"),
_user: TokenPayload = Depends(require_permission("mqtt", "view")),
):
"""Critical, unsolicited board-initiated events from control/reports
(currently only bell_overload). History/audit only — the tablets are the
real-time consumer of this data, not the console."""
reports, total = await db.get_reports(
device_serial, limit=limit, offset=offset, since=since, until=until,
)
return DeviceReportListResponse(reports=reports, total=total)
@router.websocket("/ws")
async def mqtt_websocket(websocket: WebSocket):
"""Live MQTT data stream. Auth via query param: ?token=JWT"""
token = websocket.query_params.get("token")
if not token:
await websocket.close(code=4001, reason="Missing token")
return
try:
from auth.utils import decode_access_token
from sqlalchemy import select
from database.postgres import AsyncSessionLocal
from staff.orm import Staff
payload = decode_access_token(token)
role = payload.get("role", "")
# sysadmin and admin always have MQTT access
if role not in ("sysadmin", "admin"):
user_sub = payload.get("sub", "")
async with AsyncSessionLocal() as session:
result = await session.execute(
select(Staff).where(Staff.id == user_sub).limit(1)
)
staff = result.scalar_one_or_none()
if staff is None:
await websocket.close(code=4003, reason="User not found")
return
perms = staff.permissions or {}
if not perms.get("mqtt", {}).get("access", False):
await websocket.close(code=4003, reason="MQTT access denied")
return
except Exception:
await websocket.close(code=4001, reason="Invalid token")
return
await websocket.accept()
mqtt_manager.add_ws_subscriber(websocket)
try:
while True:
await websocket.receive_text()
except WebSocketDisconnect:
pass
finally:
mqtt_manager.remove_ws_subscriber(websocket)