feat(mqtt): add device health telemetry and migrate to v2 topic spec
Two efforts that landed together because the v2 topic work extends tables the health-telemetry effort added days earlier in the same files/functions, making them impractical to separate cleanly: Health/diagnostics telemetry (schema, Jul 13-17): - New Postgres tables: device_alert_events, device_boot_events, device_ping_samples, device_diagnostics_reports, plus a `source` column on device_logs to distinguish log origins - Query/service layer in pg_mqtt.py and database/__init__.py for inserting and listing this history, plus a "latest metrics" endpoint combining most-recent diagnostics + ping RTT per device - mqtt/router.py gains list endpoints for alert/boot/ping/diagnostics history, consumed by the upcoming Health tab MQTT v2 topic migration (Sep 21): - Heartbeat payload flattened per vesper_mqtt_topic_spec_v2.md, adding rssi/free_heap/state/ok fields - Command replies move to control/ack, device-initiated events to control/reports; mqtt/client.py subscribes to the new topic set and runs a ping_loop (wired up in main.py) for RTT sampling - mqtt/logger.py and pg_mqtt.py updated to parse and persist the new payload shape alongside the legacy fields for backwards compatibility Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,73 @@
|
||||
"""device diagnostics reports
|
||||
|
||||
Adds device_diagnostics_reports — structured storage for the firmware's
|
||||
diagnostics_report MQTT event (vesper/{uid}/status/info, type="diagnostics_report"),
|
||||
published every 5 minutes. Cleanly separate from device_boot_events (one row per
|
||||
boot, event-driven) and heartbeats (one row every 30s, transport/liveness facts):
|
||||
this table is periodic health telemetry — CPU temperature (min/max/avg over the
|
||||
5-minute window), WiFi reconnect count + last disconnect reason, OTA/firmware
|
||||
check state, and per-task stack high-water marks (bytes free).
|
||||
|
||||
stack_high_water is stored as a JSON-encoded TEXT column rather than flattened
|
||||
columns — unlike the other three groups (fixed field sets), the set of
|
||||
monitored tasks is open-ended on the firmware side (see project-vesper's
|
||||
Telemetry::registerTaskForStackMonitoring), so a fixed column per task would
|
||||
need a migration every time a task is added. TEXT (not JSONB) matches this
|
||||
codebase's existing convention for JSON blobs stored via raw SQL — see
|
||||
commands.command_payload / response_payload.
|
||||
|
||||
Revision ID: a7b8c9d0e1f2
|
||||
Revises: f6a7b8c9d0e1
|
||||
Create Date: 2026-07-17 00:00:00.000000
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "a7b8c9d0e1f2"
|
||||
down_revision: Union[str, None] = "f6a7b8c9d0e1"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"device_diagnostics_reports",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True),
|
||||
sa.Column("device_serial", sa.String(128), nullable=False),
|
||||
|
||||
# cpu_temp — omitted by firmware (all null here) if no samples were
|
||||
# taken yet this window (e.g. very first report shortly after boot).
|
||||
sa.Column("cpu_temp_avg", sa.Float(), nullable=True),
|
||||
sa.Column("cpu_temp_min", sa.Float(), nullable=True),
|
||||
sa.Column("cpu_temp_max", sa.Float(), nullable=True),
|
||||
sa.Column("cpu_temp_samples", sa.Integer(), nullable=True),
|
||||
|
||||
# wifi_reconnects — this-boot-only lifetime count, not device lifetime.
|
||||
sa.Column("wifi_reconnect_count", sa.Integer(), nullable=True),
|
||||
sa.Column("wifi_last_disconnect_reason", sa.String(64), nullable=True),
|
||||
sa.Column("wifi_last_disconnect_uptime_ms", sa.BigInteger(), nullable=True),
|
||||
|
||||
# ota
|
||||
sa.Column("ota_current_version", sa.String(32), nullable=True),
|
||||
sa.Column("ota_update_available", sa.Boolean(), nullable=True),
|
||||
sa.Column("ota_available_version", sa.String(32), nullable=True),
|
||||
sa.Column("ota_last_check_uptime_ms", sa.BigInteger(), nullable=True),
|
||||
sa.Column("ota_last_error", sa.String(32), nullable=True),
|
||||
|
||||
# stack_high_water — open-ended task set, see module docstring.
|
||||
sa.Column("stack_high_water", sa.Text(), nullable=True),
|
||||
|
||||
sa.Column("received_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index(
|
||||
"idx_device_diagnostics_reports_serial_received",
|
||||
"device_diagnostics_reports",
|
||||
["device_serial", sa.text("received_at DESC")],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("idx_device_diagnostics_reports_serial_received", table_name="device_diagnostics_reports")
|
||||
op.drop_table("device_diagnostics_reports")
|
||||
@@ -0,0 +1,78 @@
|
||||
"""mqtt v2 topic migration
|
||||
|
||||
Firmware moved to a new MQTT topic spec (project-vesper's
|
||||
docs/reference/vesper_mqtt_topic_spec_v2.md, feature-catalog F-062). Three
|
||||
schema changes needed to keep ingestion correct against the new payload shapes:
|
||||
|
||||
1. heartbeats.state / heartbeats.ok — the v2 heartbeat payload is flat (no more
|
||||
{"status":"INFO","type":"heartbeat","payload":{...}} wrapper) and adds two
|
||||
new fields: state ("idle"/"playing"/"paused"/"error"/"booting") and ok
|
||||
(overall health), both independently derived on the firmware side — not
|
||||
mirrored from status/playback or system/alerts. Nullable: older rows
|
||||
(pre-migration) and any device still reporting have no value for these.
|
||||
|
||||
2. device_diagnostics_reports gains bell_strikes / bell_loads / cooling_active.
|
||||
The 5-min diagnostics report moved from status/info to system/metrics and
|
||||
picked up per-bell strike/heat data along the way (the topic spec calls out
|
||||
"bell heat ratings" explicitly for this topic). Stored as JSON-encoded TEXT,
|
||||
matching this table's existing stack_high_water column — same reasoning:
|
||||
16 fixed bell slots, but keeping the encoding consistent with the sibling
|
||||
column is simpler than mixing raw JSONB and TEXT in one row shape.
|
||||
|
||||
3. device_reports — new table for the new control/reports topic. Board-
|
||||
initiated, unsolicited, critical events (currently only bell_overload).
|
||||
Distinct from device_alert_events (subsystem health transitions) and
|
||||
device_logs (routine log lines) — this is a narrow, insert-only table for
|
||||
a different kind of signal: urgent, bell-mechanism-specific events the
|
||||
physical tablets act on in real time. The console side is deliberately
|
||||
light for now — store + list for historical/audit purposes; the tablets
|
||||
are the real-time consumer of control/reports, not this console.
|
||||
|
||||
Revision ID: b8c9d0e1f2a3
|
||||
Revises: a7b8c9d0e1f2
|
||||
Create Date: 2026-09-21 00:00:00.000000
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "b8c9d0e1f2a3"
|
||||
down_revision: Union[str, None] = "a7b8c9d0e1f2"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("heartbeats", sa.Column("state", sa.String(16), nullable=True))
|
||||
op.add_column("heartbeats", sa.Column("ok", sa.Boolean(), nullable=True))
|
||||
|
||||
op.add_column("device_diagnostics_reports", sa.Column("bell_strikes", sa.Text(), nullable=True))
|
||||
op.add_column("device_diagnostics_reports", sa.Column("bell_loads", sa.Text(), nullable=True))
|
||||
op.add_column("device_diagnostics_reports", sa.Column("cooling_active", sa.Boolean(), nullable=True))
|
||||
|
||||
op.create_table(
|
||||
"device_reports",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True),
|
||||
sa.Column("device_serial", sa.String(128), nullable=False),
|
||||
sa.Column("report_type", sa.String(64), nullable=False),
|
||||
sa.Column("payload", sa.Text(), nullable=True),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index(
|
||||
"idx_device_reports_serial_occurred",
|
||||
"device_reports",
|
||||
["device_serial", sa.text("occurred_at DESC")],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("idx_device_reports_serial_occurred", table_name="device_reports")
|
||||
op.drop_table("device_reports")
|
||||
|
||||
op.drop_column("device_diagnostics_reports", "cooling_active")
|
||||
op.drop_column("device_diagnostics_reports", "bell_loads")
|
||||
op.drop_column("device_diagnostics_reports", "bell_strikes")
|
||||
|
||||
op.drop_column("heartbeats", "ok")
|
||||
op.drop_column("heartbeats", "state")
|
||||
@@ -0,0 +1,48 @@
|
||||
"""device_alert_events
|
||||
|
||||
Adds an insert-only history log of device alert transitions (WARNING/CRITICAL/
|
||||
FAILED only — CLEARED is not logged here). device_alerts remains the current-
|
||||
state table and is untouched; this is purely additive, used to answer "when
|
||||
was the most recent issue on this device" even after it has been resolved.
|
||||
|
||||
Also adds an optional rssi column to heartbeats, for a signal-strength
|
||||
indicator once firmware starts reporting it on the heartbeat payload.
|
||||
|
||||
Revision ID: d4e5f6a7b8c9
|
||||
Revises: c3d4e5f6a7b8
|
||||
Create Date: 2026-07-13 00:00:00.000000
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "d4e5f6a7b8c9"
|
||||
down_revision: Union[str, None] = "c3d4e5f6a7b8"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"device_alert_events",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True),
|
||||
sa.Column("device_serial", sa.String(128), nullable=False),
|
||||
sa.Column("subsystem", sa.String(128), nullable=False),
|
||||
sa.Column("state", sa.String(64), nullable=False),
|
||||
sa.Column("message", sa.Text(), nullable=True),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index(
|
||||
"idx_device_alert_events_serial_occurred",
|
||||
"device_alert_events",
|
||||
["device_serial", sa.text("occurred_at DESC")],
|
||||
)
|
||||
|
||||
op.add_column("heartbeats", sa.Column("rssi", sa.Integer(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("heartbeats", "rssi")
|
||||
op.drop_index("idx_device_alert_events_serial_occurred", table_name="device_alert_events")
|
||||
op.drop_table("device_alert_events")
|
||||
@@ -0,0 +1,39 @@
|
||||
"""device_logs source column
|
||||
|
||||
Adds a 'source' column to device_logs distinguishing the debug log stream
|
||||
(vesper/{id}/logs, source='log') from the general info stream
|
||||
(vesper/{id}/status/info, source='info'), which was previously discarded
|
||||
after only being written to the server debug log. Existing rows default to
|
||||
'log' since that's the only source ever persisted before this migration.
|
||||
|
||||
Adding a column to a partitioned parent table applies it to all existing
|
||||
and future partitions automatically.
|
||||
|
||||
Revision ID: e5f6a7b8c9d0
|
||||
Revises: d4e5f6a7b8c9
|
||||
Create Date: 2026-07-14 00:00:00.000000
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "e5f6a7b8c9d0"
|
||||
down_revision: Union[str, None] = "d4e5f6a7b8c9"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("""
|
||||
ALTER TABLE device_logs
|
||||
ADD COLUMN source TEXT NOT NULL DEFAULT 'log'
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE INDEX idx_device_logs_source
|
||||
ON device_logs(device_serial, source, received_at DESC)
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS idx_device_logs_source")
|
||||
op.execute("ALTER TABLE device_logs DROP COLUMN source")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""device health tab — boot events, heartbeat free_heap, ping samples
|
||||
|
||||
Adds three pieces of schema needed for the device Health tab:
|
||||
|
||||
1. device_boot_events — structured, insert-only history of the firmware's
|
||||
boot_report MQTT event (vesper/{uid}/status/info, type="boot_report").
|
||||
Previously this payload was flattened into a single device_logs text line
|
||||
with all structured fields (boot_count, crash detail) discarded on arrival.
|
||||
This table gives the console a real timeline to query and chart against,
|
||||
instead of parsing log strings.
|
||||
|
||||
2. heartbeats.free_heap — firmware now includes free_heap on every 30s
|
||||
heartbeat (previously only available once per boot via boot_report), so
|
||||
the console can chart heap trend instead of seeing one point per boot.
|
||||
|
||||
3. device_ping_samples — backend-computed RTT samples. The firmware's ping
|
||||
command now echoes back a caller-supplied timestamp; the backend pings
|
||||
each online device on an interval and records (now - echoed_ts) here.
|
||||
A dedicated table rather than reusing `commands` because `commands` isn't
|
||||
shaped for time-series charting (mixed command types, no fast per-device
|
||||
time-range query path) and pruning ping history independently of other
|
||||
command history is desirable.
|
||||
|
||||
Revision ID: f6a7b8c9d0e1
|
||||
Revises: e5f6a7b8c9d0
|
||||
Create Date: 2026-07-16 00:00:00.000000
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "f6a7b8c9d0e1"
|
||||
down_revision: Union[str, None] = "e5f6a7b8c9d0"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"device_boot_events",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True),
|
||||
sa.Column("device_serial", sa.String(128), nullable=False),
|
||||
sa.Column("boot_count", sa.Integer(), nullable=True),
|
||||
sa.Column("reset_reason", sa.String(64), nullable=True),
|
||||
sa.Column("is_fault", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("free_heap", sa.Integer(), nullable=True),
|
||||
sa.Column("crash_task", sa.String(64), nullable=True),
|
||||
sa.Column("crash_pc", sa.BigInteger(), nullable=True),
|
||||
sa.Column("crash_exc_cause", sa.Integer(), nullable=True),
|
||||
sa.Column("crash_exc_vaddr", sa.BigInteger(), nullable=True),
|
||||
sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index(
|
||||
"idx_device_boot_events_serial_occurred",
|
||||
"device_boot_events",
|
||||
["device_serial", sa.text("occurred_at DESC")],
|
||||
)
|
||||
|
||||
op.add_column("heartbeats", sa.Column("free_heap", sa.Integer(), nullable=True))
|
||||
|
||||
op.create_table(
|
||||
"device_ping_samples",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True),
|
||||
sa.Column("device_serial", sa.String(128), nullable=False),
|
||||
sa.Column("rtt_ms", sa.Integer(), nullable=False),
|
||||
sa.Column("sampled_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index(
|
||||
"idx_device_ping_samples_serial_sampled",
|
||||
"device_ping_samples",
|
||||
["device_serial", sa.text("sampled_at DESC")],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("idx_device_ping_samples_serial_sampled", table_name="device_ping_samples")
|
||||
op.drop_table("device_ping_samples")
|
||||
op.drop_column("heartbeats", "free_heap")
|
||||
op.drop_index("idx_device_boot_events_serial_occurred", table_name="device_boot_events")
|
||||
op.drop_table("device_boot_events")
|
||||
Reference in New Issue
Block a user