bonamin/bellsystems-cp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(mqtt): store F-070 crash detail and group crashes fleet-wide

Browse Source 
Firmware F-070 adds a fuller `crash` object (backtrace, backtrace_corrupted,
elf_sha256) and a new `pre_crash` snapshot (uptime, heap stats, optional
abort_msg) to boot_report and to telemetry.get_boot_history entries.

- Migration c9d0e1f2a3b4: device_boot_events gains crash / pre_crash JSONB,
  stored verbatim so later additive fields need no migration. The legacy
  crash_* columns are still filled; old rows and old firmware are unchanged.
  JSON is bound as CAST(CAST(:x AS TEXT) AS JSONB): a bare JSONB cast makes
  asyncpg JSON-encode the already-encoded string a second time.
- boot_report ingestion passes both objects through.
- telemetry.get_boot_history replies (control/ack) are merged into boot
  history whoever sent the command: an entry matches an existing row on
  boot_count + reset_reason + time within 30 min (boot_count alone is not
  unique - the counter gets reset), and only fills crash/pre_crash the row
  lacks; unmatched entries are boots we never saw live and are inserted at
  the device's timestamp; entries without ts are skipped.
- GET /api/mqtt/crash-groups: fault boots grouped by abort_msg with hex
  addresses stripped, else task + exception cause, else reset reason. When
  abort_msg is present, pc/exc_cause describe abort() itself and are
  ignored for grouping.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bonamin
2026-09-30 16:28:40 +03:00
co-authored by Claude Opus 5.5
parent 46c5c0a846
commit 963516ece0
7 changed files with 365 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/database/__init__.py
+4
View File
@@ -23,6 +23,8 @@ from database.pg_mqtt import (
insert_boot_event,
get_boot_events,
get_latest_boot_event,
merge_device_boot_history,
get_crash_events,
insert_ping_sample,
get_ping_samples,
get_latest_ping_samples,
@@ -74,6 +76,8 @@ __all__ = [
"insert_boot_event",
"get_boot_events",
"get_latest_boot_event",
"merge_device_boot_history",
"get_crash_events",
"insert_ping_sample",
"get_ping_samples",
"get_latest_ping_samples",
backend/database/pg_mqtt.py
+184 -20
View File
@@ -438,13 +438,56 @@ async def get_latest_alert_event(device_serial: str) -> dict | None:
# Health tab's boot timeline + restart chart.
# ---------------------------------------------------------------------------
_BOOT_COLUMNS = """
id, device_serial, boot_count, reset_reason, is_fault, free_heap,
crash_task, crash_pc, crash_exc_cause, crash_exc_vaddr,
crash::text AS crash, pre_crash::text AS pre_crash,
occurred_at AT TIME ZONE 'UTC' AS occurred_at
"""
# How far apart a device-reported boot timestamp (telemetry.get_boot_history
# "ts", device clock) and our own occurred_at (broker receive time of the live
# boot_report) may be and still count as the same boot. The live report goes
# out on first MQTT connect, so it normally lands within a minute of boot.
_BOOT_MATCH_WINDOW = timedelta(minutes=30)
def _json_or_none(obj: dict | None) -> str | None:
"""Encode for a CAST(CAST(:x AS TEXT) AS JSONB) bind. Passed as text on
purpose: a bare CAST(:x AS JSONB) makes asyncpg type the parameter as jsonb
and JSON-encode our already-encoded string a second time."""
return json.dumps(obj) if obj else None
def _boot_row_to_dict(row) -> dict:
d = _row_to_dict(row)
for key in ("crash", "pre_crash"):
val = d.get(key)
if isinstance(val, str):
try:
d[key] = json.loads(val)
except ValueError:
d[key] = None
return d
def _crash_columns(crash: dict | None) -> dict:
"""The legacy flat crash_* columns, still filled so older readers and the
fleet grouping query keep working without unpacking JSON."""
crash = crash or {}
return {
"crash_task": crash.get("task"),
"crash_pc": crash.get("pc"),
"crash_exc_cause": crash.get("exc_cause"),
"crash_exc_vaddr": crash.get("exc_vaddr"),
}
async def insert_boot_event(device_serial: str, boot_count: int | None,
reset_reason: str | None, is_fault: bool,
free_heap: int | None,
crash_task: str | None = None,
crash_pc: int | None = None,
crash_exc_cause: int | None = None,
crash_exc_vaddr: int | None = None,
crash: dict | None = None,
pre_crash: dict | None = None,
skip_if_latest: bool = False) -> int | None:
"""Insert a boot event. With skip_if_latest=True (a retained replay of the
device's last boot_report), skip it and return None when it matches the
@@ -452,7 +495,10 @@ async def insert_boot_event(device_serial: str, boot_count: int | None,
Only the LATEST row is compared, never "any row with this boot_count": the
firmware's lifetime counter gets reset (reflash / telemetry reset), so the
same boot_count legitimately appears again for a later, different boot."""
same boot_count legitimately appears again for a later, different boot.
`crash` / `pre_crash` are the firmware's objects stored verbatim (F-070);
both are omitted by older firmware and simply stay NULL."""
async with AsyncSessionLocal() as session:
if skip_if_latest and boot_count is not None:
latest = await session.execute(
@@ -471,10 +517,12 @@ async def insert_boot_event(device_serial: str, boot_count: int | None,
text("""
INSERT INTO device_boot_events
(device_serial, boot_count, reset_reason, is_fault, free_heap,
crash_task, crash_pc, crash_exc_cause, crash_exc_vaddr, occurred_at)
crash_task, crash_pc, crash_exc_cause, crash_exc_vaddr,
crash, pre_crash, occurred_at)
VALUES
(:serial, :boot_count, :reset_reason, :is_fault, :free_heap,
:crash_task, :crash_pc, :crash_exc_cause, :crash_exc_vaddr, now())
:crash_task, :crash_pc, :crash_exc_cause, :crash_exc_vaddr,
CAST(CAST(:crash AS TEXT) AS JSONB), CAST(CAST(:pre_crash AS TEXT) AS JSONB), now())
RETURNING id
"""),
{
@@ -483,10 +531,9 @@ async def insert_boot_event(device_serial: str, boot_count: int | None,
"reset_reason": reset_reason,
"is_fault": is_fault,
"free_heap": free_heap,
"crash_task": crash_task,
"crash_pc": crash_pc,
"crash_exc_cause": crash_exc_cause,
"crash_exc_vaddr": crash_exc_vaddr,
**_crash_columns(crash),
"crash": _json_or_none(crash),
"pre_crash": _json_or_none(pre_crash),
},
)
row = result.fetchone()
@@ -494,6 +541,102 @@ async def insert_boot_event(device_serial: str, boot_count: int | None,
return row[0] if row else None
async def merge_device_boot_history(device_serial: str, boots: list[dict]) -> dict:
"""Merge the device's own SD boot log (telemetry.get_boot_history reply)
into device_boot_events.
Each entry is {"ts": epoch s, "boot": n, "reason": str, "crash"?, "pre_crash"?}.
An entry matches an existing row when boot_count and reset_reason are equal
and the times are within _BOOT_MATCH_WINDOW (boot_count alone is not unique,
see insert_boot_event). A match only fills in crash/pre_crash the row is
missing; nothing already stored is overwritten. An entry with no match is a
boot we never saw live and is inserted at the device's timestamp. Entries
without a usable ts can't be placed in time and are skipped.
"""
inserted = updated = skipped = 0
async with AsyncSessionLocal() as session:
for entry in boots:
if not isinstance(entry, dict):
skipped += 1
continue
ts = entry.get("ts")
boot_count = entry.get("boot")
reason = entry.get("reason")
if not isinstance(ts, (int, float)) or ts <= 0:
skipped += 1
continue
at = datetime.fromtimestamp(ts, tz=timezone.utc)
crash = entry.get("crash") if isinstance(entry.get("crash"), dict) else None
pre_crash = entry.get("pre_crash") if isinstance(entry.get("pre_crash"), dict) else None
match = (await session.execute(
text("""
SELECT id, crash IS NULL AS no_crash, pre_crash IS NULL AS no_pre_crash
FROM device_boot_events
WHERE device_serial = :serial
AND boot_count IS NOT DISTINCT FROM :boot_count
AND reset_reason IS NOT DISTINCT FROM :reason
AND occurred_at BETWEEN :lo AND :hi
ORDER BY abs(extract(epoch FROM occurred_at - :at))
LIMIT 1
"""),
{"serial": device_serial, "boot_count": boot_count, "reason": reason,
"lo": at - _BOOT_MATCH_WINDOW, "hi": at + _BOOT_MATCH_WINDOW, "at": at},
)).first()
if match is None:
await session.execute(
text("""
INSERT INTO device_boot_events
(device_serial, boot_count, reset_reason, is_fault, free_heap,
crash_task, crash_pc, crash_exc_cause, crash_exc_vaddr,
crash, pre_crash, occurred_at)
VALUES
(:serial, :boot_count, :reason, :is_fault, NULL,
:crash_task, :crash_pc, :crash_exc_cause, :crash_exc_vaddr,
CAST(CAST(:crash AS TEXT) AS JSONB), CAST(CAST(:pre_crash AS TEXT) AS JSONB), :at)
"""),
{
"serial": device_serial, "boot_count": boot_count, "reason": reason,
"is_fault": bool(entry.get("is_fault", reason in FAULT_RESET_REASONS)),
**_crash_columns(crash),
"crash": _json_or_none(crash), "pre_crash": _json_or_none(pre_crash),
"at": at,
},
)
inserted += 1
continue
fill_crash = crash is not None and match.no_crash
fill_pre = pre_crash is not None and match.no_pre_crash
if not (fill_crash or fill_pre):
continue
sets, params = [], {"id": match.id}
if fill_crash:
sets.append("crash = CAST(CAST(:crash AS TEXT) AS JSONB)")
sets.append("crash_task = COALESCE(crash_task, :crash_task)")
sets.append("crash_pc = COALESCE(crash_pc, :crash_pc)")
sets.append("crash_exc_cause = COALESCE(crash_exc_cause, :crash_exc_cause)")
sets.append("crash_exc_vaddr = COALESCE(crash_exc_vaddr, :crash_exc_vaddr)")
params.update(_crash_columns(crash))
params["crash"] = _json_or_none(crash)
if fill_pre:
sets.append("pre_crash = CAST(CAST(:pre_crash AS TEXT) AS JSONB)")
params["pre_crash"] = _json_or_none(pre_crash)
await session.execute(
text(f"UPDATE device_boot_events SET {', '.join(sets)} WHERE id = :id"),
params,
)
updated += 1
await session.commit()
return {"inserted": inserted, "updated": updated, "skipped": skipped}
# Mirrors the firmware's isFaultReset() — used only when a history entry
# doesn't say is_fault itself.
FAULT_RESET_REASONS = {"PANIC", "TASK_WATCHDOG", "INTERRUPT_WATCHDOG", "OTHER_WATCHDOG", "BROWNOUT"}
async def get_boot_events(device_serial: str, limit: int = 100, offset: int = 0,
since: datetime | None = None,
until: datetime | None = None) -> tuple[list, int]:
@@ -509,9 +652,7 @@ async def get_boot_events(device_serial: str, limit: int = 100, offset: int = 0,
rows_result = await session.execute(
text(f"""
SELECT id, device_serial, boot_count, reset_reason, is_fault, free_heap,
crash_task, crash_pc, crash_exc_cause, crash_exc_vaddr,
occurred_at AT TIME ZONE 'UTC' AS occurred_at
SELECT {_BOOT_COLUMNS}
FROM device_boot_events
WHERE device_serial = :serial{range_clause}
ORDER BY occurred_at DESC
@@ -521,16 +662,14 @@ async def get_boot_events(device_serial: str, limit: int = 100, offset: int = 0,
)
rows = rows_result.mappings().all()
return [_row_to_dict(r) for r in rows], total
return [_boot_row_to_dict(r) for r in rows], total
async def get_latest_boot_event(device_serial: str) -> dict | None:
async with AsyncSessionLocal() as session:
result = await session.execute(
text("""
SELECT id, device_serial, boot_count, reset_reason, is_fault, free_heap,
crash_task, crash_pc, crash_exc_cause, crash_exc_vaddr,
occurred_at AT TIME ZONE 'UTC' AS occurred_at
text(f"""
SELECT {_BOOT_COLUMNS}
FROM device_boot_events
WHERE device_serial = :serial
ORDER BY occurred_at DESC
@@ -540,7 +679,32 @@ async def get_latest_boot_event(device_serial: str) -> dict | None:
)
row = result.mappings().first()
return _row_to_dict(row) if row else None
return _boot_row_to_dict(row) if row else None
async def get_crash_events(since: datetime | None = None,
until: datetime | None = None,
limit: int = 5000) -> list:
"""Fleet-wide fault boots that carry any crash detail (coredump summary
and/or pre-crash snapshot), newest first. Grouped in mqtt/crash_groups.py."""
async with AsyncSessionLocal() as session:
params: dict = {"limit": limit}
range_clause = _time_range_clause("occurred_at", since, until, params)
rows_result = await session.execute(
text(f"""
SELECT {_BOOT_COLUMNS}
FROM device_boot_events
WHERE is_fault
AND (crash IS NOT NULL OR pre_crash IS NOT NULL OR crash_task IS NOT NULL)
{range_clause}
ORDER BY occurred_at DESC
LIMIT :limit
"""),
params,
)
rows = rows_result.mappings().all()
return [_boot_row_to_dict(r) for r in rows]
# ---------------------------------------------------------------------------