fix(mqtt): treat broker-replayed retained messages as stale state

The firmware publishes status/heartbeat, system/alerts, system/info and
status/playback with retain=true. On every backend (re)connect - every
restart and every uvicorn --reload - the broker replays the last message on
each of those topics for every device that ever connected. We handled those
replays as if they had just happened:

- heartbeats: a row with received_at=now() for every device, so devices
  that have been dead for months showed ONLINE for 90s after each restart
  and got pinged. ~770k such rows exist locally.
- boot_report: the last boot logged again as a new reboot (the phantom
  PANIC entries on the Health tab).
- alerts / other info events: logged again as new occurrences.

MQTT delivers retain=1 only for replays caused by a new subscription; live
publishes always arrive with retain=0. The flag is now passed through to the
handlers and the WS broadcast:

- heartbeat: replays are not stored. A live heartbeat is.
- {"state":"offline"} heartbeat (LWT / graceful disconnect) is no longer
  stored as a sign of life. It marks the device offline immediately in
  a small in-memory set (mqtt/presence.py) used by /mqtt/status and the ping
  loop; a later live heartbeat clears it. Replayed offline markers also mark
  offline, since a retained message is the device's last word.
- boot_report: live -> always a new boot. Replay -> stored only if it
  differs from the device's latest boot row (i.e. we missed it while down).
- alerts: replay still syncs the current-alert row; history gets a row on a
  live alert (even an identical repeat - faults recur) or on a replay that
  changes state. Replaces the 98dd16b rule that dropped identical live alerts.
- other info events: replays are not logged.
- Frontend (DeviceList, DeviceDetail, LogsTab) ignores retained WS messages
  for live updates, and flips a device offline on the offline marker instead
  of marking it online.

Verified locally after a backend restart: only the 7 actually-live devices
got new heartbeat rows (none from the replays), and no boot/alert/info rows
were created.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-30 16:01:41 +03:00
co-authored by Claude Opus 5.5
parent 18e57c6e5f
commit 46c5c0a846
8 changed files with 111 additions and 27 deletions
+44 -12
View File
@@ -1,6 +1,7 @@
import logging
import time
import database as db
from mqtt import presence
logger = logging.getLogger("mqtt.logger")
@@ -15,15 +16,22 @@ LEVEL_MAP = {
}
async def handle_message(serial: str, topic_type: str, payload: dict):
async def handle_message(serial: str, topic_type: str, payload: dict,
retained: bool = False):
"""`retained` is True when the broker replayed this from its retained store
on (re)subscribe: a stale copy of the device's last message, NOT a new
event. The firmware publishes status/heartbeat, system/alerts, system/info
and status/playback retained, so every backend restart replays them for
every device that ever connected. Handlers for those topics must never
record a retained replay as something that just happened."""
try:
# v2 topic set — see project-vesper's vesper_mqtt_topic_spec_v2.md.
if topic_type == "status/heartbeat":
await _handle_heartbeat(serial, payload)
await _handle_heartbeat(serial, payload, retained)
elif topic_type == "system/alerts":
await _handle_alerts(serial, payload)
await _handle_alerts(serial, payload, retained)
elif topic_type == "system/info":
await _handle_info(serial, payload)
await _handle_info(serial, payload, retained)
elif topic_type == "system/logs":
await _handle_log(serial, payload)
elif topic_type == "system/metrics":
@@ -40,7 +48,21 @@ async def handle_message(serial: str, topic_type: str, payload: dict):
logger.error(f"Error handling {topic_type} for {serial}: {e}")
async def _handle_heartbeat(serial: str, payload: dict):
async def _handle_heartbeat(serial: str, payload: dict, retained: bool = False):
# Online status is "a heartbeat row newer than 90s", so only a live sign of
# life may be stored:
# - retained replay: the device's last heartbeat from whenever. Storing it
# with received_at=now() marked every device ever seen (even long-dead
# ones) online for 90s after each backend restart.
# - state "offline": the LWT / graceful-disconnect marker published when
# the device goes AWAY. Not a heartbeat, but it IS current state even
# when replayed (a retained message is the device's last word).
if payload.get("state") == "offline":
presence.mark_offline(serial)
return
if retained:
return
presence.mark_alive(serial)
# Store silently — do not log as a visible event.
# The console surfaces an alert only when the device goes silent (no heartbeat for 90s).
# v2 heartbeat payload is FLAT — no {"status","type","payload"} wrapper,
@@ -76,7 +98,7 @@ async def _handle_log(serial: str, payload: dict):
)
async def _handle_alerts(serial: str, payload: dict):
async def _handle_alerts(serial: str, payload: dict, retained: bool = False):
subsystem = payload.get("subsystem", "")
state = payload.get("state", "")
if not subsystem or not state:
@@ -86,16 +108,18 @@ async def _handle_alerts(serial: str, payload: dict):
if state == "CLEARED":
await db.delete_alert(serial, subsystem)
else:
# Retained replays still sync the current-alert row, so a fresh DB
# learns about alerts raised while the backend was down.
changed = await db.upsert_alert(serial, subsystem, state, payload.get("msg"))
# Append-only history — survives past the alert being resolved, used to
# answer "when was the most recent issue" even once it's cleared.
# Skipped when nothing changed: alerts are retained, so the broker
# re-sends every active one on each backend (re)subscribe.
if changed:
# A live alert is always a new occurrence (the same fault can recur);
# a retained replay only counts if it isn't already the current state.
if changed or not retained:
await db.insert_alert_event(serial, subsystem, state, payload.get("msg"))
async def _handle_info(serial: str, payload: dict):
async def _handle_info(serial: str, payload: dict, retained: bool = False):
event_type = payload.get("type", "")
# boot_report carries structured fields (boot_count, crash detail, etc.) —
@@ -106,7 +130,12 @@ async def _handle_info(serial: str, payload: dict):
# Note: diagnostics_report used to also arrive here (F-056) but moved to
# its own system/metrics topic in the v2 spec — see _handle_metrics below.
if event_type == "boot_report":
await _handle_boot_report(serial, payload)
await _handle_boot_report(serial, payload, retained)
return
# Any other info event replayed from the retained store (e.g. an old
# playback_started) already happened in the past — don't log it again.
if retained:
return
data = payload.get("payload", {})
@@ -126,7 +155,7 @@ async def _handle_info(serial: str, payload: dict):
)
async def _handle_boot_report(serial: str, payload: dict):
async def _handle_boot_report(serial: str, payload: dict, retained: bool = False):
"""Parses the firmware's consolidated boot_report event (see
CommunicationRouter::reportBootOnce() / publishInfo()) into
device_boot_events. Like all status/info events, the report fields are
@@ -145,6 +174,9 @@ async def _handle_boot_report(serial: str, payload: dict):
crash_pc=crash.get("pc"),
crash_exc_cause=crash.get("exc_cause"),
crash_exc_vaddr=crash.get("exc_vaddr"),
# A live boot_report is always a new boot. A retained replay is the
# device's last boot, recorded only if we missed it while down.
skip_if_latest=retained,
)