diff --git a/frontend/src/components/shared/CrashDetail.jsx b/frontend/src/components/shared/CrashDetail.jsx new file mode 100644 index 0000000..85f4741 --- /dev/null +++ b/frontend/src/components/shared/CrashDetail.jsx @@ -0,0 +1,200 @@ +// frontend/src/components/shared/CrashDetail.jsx +// Crash detail for one fault boot event (firmware F-070): headline, task, +// uptime + heap snapshot at crash, PC/backtrace in ESP serial format, a +// ready-to-run addr2line command, the crashing build's ELF id, and — when the +// server has that build's firmware.elf and the xtensa addr2line binary — +// in-console decoding to function + file:line. +// +// Used by the device Health tab's boot history and the fleet Crash Report. +// +// Props: +// event — a boot event from /mqtt/boot-events (crash / pre_crash objects, +// or only the legacy crash_* columns for older firmware) + +import { useState, useEffect } from 'react' +import api from '@/lib/api' +import Button from '@/components/ui/Button' +import StatusBadge from '@/components/ui/StatusBadge' +import Icon from '@/components/ui/Icon' +import { copyText } from '@/lib/clipboard' +import { + crashOf, preCrashOf, crashHeadline, excCauseName, hex32, isHeapExhaustion, + addressLine, addr2lineCommand, decodeAddresses, fmtBytes, fmtUptime, +} from '@/lib/crashReport' + +const labelStyle = { + fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)', + textTransform: 'uppercase', letterSpacing: 'var(--tracking-wide)', +} +const monoBlockStyle = { + flex: 1, minWidth: 0, margin: 0, + padding: 'var(--space-2) var(--space-3)', + background: 'var(--color-bg-abyss)', border: '1px solid var(--color-border)', + borderRadius: 'var(--radius-md)', + fontFamily: 'var(--font-family-mono)', fontSize: 'var(--font-size-xs)', + color: 'var(--color-text-primary)', + whiteSpace: 'pre-wrap', wordBreak: 'break-all', +} + +function Field({ label, value, muted, mono = true }) { + return ( +
+ {label} + + {value ?? '—'} + +
+ ) +} + +function CopyRow({ label, text }) { + const [copied, setCopied] = useState(false) + const handleCopy = () => { + copyText(text).then(() => { + setCopied(true) + setTimeout(() => setCopied(false), 1500) + }).catch(() => {}) + } + return ( +
+ {label} +
+ 
{text}
+ +
+
+ ) +} + +function DecodePanel({ event, elfSha256 }) { + const [info, setInfo] = useState(null) // /firmware/elf/{sha} + const [results, setResults] = useState(null) // decoded frames + const [decoding, setDecoding] = useState(false) + const [error, setError] = useState('') + + useEffect(() => { + let cancelled = false + setInfo(null); setResults(null); setError('') + api.get(`/firmware/elf/${elfSha256}`) + .then(res => { if (!cancelled) setInfo(res) }) + .catch(() => { if (!cancelled) setInfo({ exists: false, decoder_available: false, unavailable: true }) }) + return () => { cancelled = true } + }, [elfSha256]) + + const handleDecode = async () => { + setDecoding(true); setError('') + try { + const res = await api.post(`/firmware/elf/${elfSha256}/decode`, { addresses: decodeAddresses(event) }) + setResults(res.results || []) + } catch (err) { + setError(err.message || 'Decoding failed.') + } finally { + setDecoding(false) + } + } + + if (!info) return null + + let note = null + if (info.unavailable) note = 'Could not check the symbol store for this build.' + else if (!info.exists) note = `No firmware.elf uploaded for build ${elfSha256}. Attach it to the firmware release to decode here — or use the command above.` + else if (!info.decoder_available) note = 'The ELF for this build is on the server, but the xtensa addr2line decoder is not installed there — use the command above.' + + return ( +
+ Decoded backtrace + {note ? ( + {note} + ) : results ? ( + results.length === 0 ? ( + No addresses to decode. + ) : ( + 
+            {results.map(r => `${r.address}: ${r.frames.join('\n            ')}`).join('\n')}
+
+ ) + ) : ( +
+ +
+ )} + {error && {error}} +
+ ) +} + +export default function CrashDetail({ event }) { + const crash = crashOf(event) + const pre = preCrashOf(event) + if (!crash && !pre) return null + + const abortMsg = pre?.abort_msg || null + const headline = crashHeadline(event) + const heapLow = isHeapExhaustion(pre) + const addrs = addressLine(crash) + const command = addr2lineCommand(event) + + return ( +
+ {/* Headline — abort_msg when present, else the exception cause name */} +
+ + {headline || 'Fault reset (no coredump)'} + +
+ {heapLow && Likely heap exhaustion / fragmentation} + {crash?.backtraceCorrupted && Backtrace corrupted} + {!crash && No coredump} +
+ {abortMsg && crash && ( + + This was an abort. The PC and exception below describe the abort() mechanism, not the cause, + and the first 2–4 backtrace frames are abort internals — the real call site is further down. + + )} +
+ +
+ + + + + + {crash && ( + <> + + + + + + )} +
+ + {addrs && } + {command && } + {crash?.elfSha256 && command && } +
+ ) +} diff --git a/frontend/src/lib/clipboard.js b/frontend/src/lib/clipboard.js new file mode 100644 index 0000000..be19951 --- /dev/null +++ b/frontend/src/lib/clipboard.js @@ -0,0 +1,28 @@ +// frontend/src/lib/clipboard.js +// Copy text to the clipboard, with a textarea fallback for non-secure +// contexts (plain-HTTP LAN access) where navigator.clipboard is undefined. + +export function copyText(text) { + if (navigator.clipboard?.writeText) { + return navigator.clipboard.writeText(text).catch(() => fallbackCopy(text)) + } + return fallbackCopy(text) +} + +function fallbackCopy(text) { + return new Promise((resolve, reject) => { + const ta = document.createElement('textarea') + ta.value = text + ta.setAttribute('readonly', '') + ta.style.cssText = 'position:fixed;opacity:0' + document.body.appendChild(ta) + ta.select() + try { + document.execCommand('copy') ? resolve() : reject(new Error('Copy failed')) + } catch (err) { + reject(err) + } finally { + document.body.removeChild(ta) + } + }) +} diff --git a/frontend/src/lib/crashReport.js b/frontend/src/lib/crashReport.js new file mode 100644 index 0000000..452fded --- /dev/null +++ b/frontend/src/lib/crashReport.js @@ -0,0 +1,124 @@ +// frontend/src/lib/crashReport.js +// Helpers for firmware crash detail (F-070) on boot events: the `crash` +// (coredump summary) and `pre_crash` (heap/uptime snapshot + optional +// abort_msg) objects from boot_report, with fallback to the legacy flat +// crash_* columns for older rows/firmware. +// +// Key rule: when pre_crash.abort_msg is present, pc / exc_cause 29 / +// exc_vaddr 0 describe the abort() mechanism, NOT the real cause — the +// abort_msg is the headline and those fields are de-emphasised. + +// Xtensa EXCCAUSE values the ESP32 actually produces. +export const EXC_CAUSE_NAMES = { + 0: 'IllegalInstruction', + 2: 'InstructionFetchError', + 3: 'LoadStoreError', + 6: 'IntegerDivideByZero', + 9: 'LoadStoreAlignment', + 28: 'LoadProhibited', + 29: 'StoreProhibited', +} + +// Firmware F-055 thresholds: below these the crash was most likely caused by +// heap exhaustion or fragmentation rather than a logic bug. +export const HEAP_LARGEST_BLOCK_WARN = 12000 +export const HEAP_FREE_WARN = 25000 + +export function excCauseName(cause) { + if (cause == null) return 'Unknown exception' + return EXC_CAUSE_NAMES[cause] ?? `Exception cause ${cause}` +} + +// 0x%08X — the ESP serial monitor's address format. +export function hex32(n) { + if (n == null || !Number.isFinite(Number(n))) return '—' + return `0x${(Number(n) >>> 0).toString(16).toUpperCase().padStart(8, '0')}` +} + +// Merge the JSON `crash` object with the legacy flat columns so callers deal +// with one shape. Returns null when the boot has no coredump summary at all. +export function crashOf(event) { + const c = event?.crash || {} + const task = c.task ?? event?.crash_task + const pc = c.pc ?? event?.crash_pc + if (task == null && pc == null && !event?.crash) return null + return { + task, + pc, + excCause: c.exc_cause ?? event?.crash_exc_cause, + excVaddr: c.exc_vaddr ?? event?.crash_exc_vaddr, + backtrace: Array.isArray(c.backtrace) ? c.backtrace : [], + backtraceCorrupted: !!c.backtrace_corrupted, + elfSha256: c.elf_sha256 || null, + } +} + +export function preCrashOf(event) { + return event?.pre_crash || null +} + +export function hasCrashDetail(event) { + return !!(crashOf(event) || preCrashOf(event)) +} + +// "abort() was called at PC 0x400d8a3f on core 0" → 0x400D8A3F (the caller of +// abort(), worth decoding too). Only that phrasing carries a PC. +export function abortCallerPc(abortMsg) { + const m = /abort\(\) was called at PC (0x[0-9a-fA-F]+)/.exec(abortMsg || '') + return m ? parseInt(m[1], 16) : null +} + +export function crashHeadline(event) { + const abortMsg = preCrashOf(event)?.abort_msg + if (abortMsg) return abortMsg + const crash = crashOf(event) + if (crash) return excCauseName(crash.excCause) + return null +} + +export function isHeapExhaustion(preCrash) { + if (!preCrash) return false + return (preCrash.largest_block != null && preCrash.largest_block < HEAP_LARGEST_BLOCK_WARN) + || (preCrash.free_heap != null && preCrash.free_heap < HEAP_FREE_WARN) +} + +// One line, space separated, like the ESP panic handler prints it. +export function addressLine(crash) { + if (!crash) return '' + return [crash.pc, ...crash.backtrace].filter(a => a != null).map(hex32).join(' ') +} + +// Every address worth decoding: PC, backtrace, then the abort() caller PC. +export function decodeAddresses(event) { + const crash = crashOf(event) + const out = [] + if (crash?.pc != null) out.push(Number(crash.pc)) + for (const a of crash?.backtrace || []) out.push(Number(a)) + const caller = abortCallerPc(preCrashOf(event)?.abort_msg) + if (caller != null) out.push(caller) + return out +} + +export function addr2lineCommand(event) { + const addrs = decodeAddresses(event) + if (!addrs.length) return '' + return `xtensa-esp32-elf-addr2line -pfiaC -e firmware.elf ${addrs.map(hex32).join(' ')}` +} + +export function fmtBytes(n) { + if (n == null) return '—' + if (n >= 1024) return `${(n / 1024).toFixed(1)} KB` + return `${n} B` +} + +export function fmtUptime(ms) { + if (ms == null) return '—' + const s = Math.floor(ms / 1000) + const d = Math.floor(s / 86400) + const h = Math.floor((s % 86400) / 3600) + const m = Math.floor((s % 3600) / 60) + const sec = s % 60 + const parts = [d && `${d}d`, h && `${h}h`, m && `${m}m`] + if (!d && !h) parts.push(`${sec}s`) + return parts.filter(Boolean).join(' ') || '0s' +} diff --git a/frontend/src/pages/bellcloud/devices/CrashReport.jsx b/frontend/src/pages/bellcloud/devices/CrashReport.jsx new file mode 100644 index 0000000..08bccf3 --- /dev/null +++ b/frontend/src/pages/bellcloud/devices/CrashReport.jsx @@ -0,0 +1,191 @@ +// frontend/src/pages/bellcloud/devices/CrashReport.jsx +// Fleet-wide crash report (firmware F-070): every fault boot with crash +// detail, grouped so recurring crash types stand out across devices. +// Grouping (backend mqtt/crash_groups.py): abort_msg with hex addresses +// stripped when present, else faulting task + exception cause, else reset +// reason for fault resets with no coredump. + +import { useState, useEffect, useCallback, useMemo } from 'react' +import { Link } from 'react-router-dom' +import api from '@/lib/api' +import PageHeader from '@/components/ui/PageHeader' +import Card from '@/components/ui/Card' +import Button from '@/components/ui/Button' +import StatusBadge from '@/components/ui/StatusBadge' +import Spinner from '@/components/ui/Spinner' +import Icon from '@/components/ui/Icon' +import TimeRangeSelect from '@/components/ui/TimeRangeSelect' +import { resolveRange } from '@/lib/timeRange' +import CrashDetail from '@/components/shared/CrashDetail' +import { fmtDateTimeMedium, fmtRelative } from '@/lib/formatters' +import { isHeapExhaustion } from '@/lib/crashReport' + +const KIND_LABEL = { abort: 'Abort', exception: 'Exception', reason: 'No coredump' } +const KIND_VARIANT = { abort: 'danger', exception: 'danger', reason: 'warning' } + +function deviceLabel(device) { + return device?.console_name || device?.device_name || null +} + +function CrashGroupCard({ group, devicesBySerial }) { + const [showDetail, setShowDetail] = useState(false) + const heapLow = isHeapExhaustion(group.latest?.pre_crash) + + return ( + {KIND_LABEL[group.kind] || group.kind}} + > +
+
+ First seen {fmtDateTimeMedium(group.first_at)} + Last seen {fmtDateTimeMedium(group.last_at)} + {heapLow && Latest: likely heap exhaustion} +
+ +
+ {group.devices.map(d => { + const device = devicesBySerial[d.device_serial] + const name = deviceLabel(device) + return ( +
+ + {device ? ( + + {name || d.device_serial} + + ) : ( + {d.device_serial} + )} + {name && ( + + {d.device_serial} + + )} + + + {d.count}× + +
+ ) + })} +
+ +
+ +
+ {showDetail && ( +
+ + {group.latest.device_serial} · boot #{group.latest.boot_count ?? '—'} · {fmtDateTimeMedium(group.latest.occurred_at)} + + +
+ )} +
+ + ) +} + +export default function CrashReport() { + const [range, setRange] = useState(() => resolveRange('30d')) + const [groups, setGroups] = useState([]) + const [totalCrashes, setTotalCrashes] = useState(0) + const [devices, setDevices] = useState([]) + const [loading, setLoading] = useState(true) + const [error, setError] = useState('') + + const load = useCallback(async () => { + setLoading(true) + setError('') + try { + const p = new URLSearchParams() + if (range.since) p.set('since', range.since.toISOString()) + if (range.until) p.set('until', range.until.toISOString()) + const qs = p.toString() + const [crashRes, deviceRes] = await Promise.all([ + api.get(`/mqtt/crash-groups${qs ? `?${qs}` : ''}`), + // Only for names/links — the report itself doesn't depend on it. + api.get('/devices').catch(() => ({ devices: [] })), + ]) + setGroups(crashRes.groups || []) + setTotalCrashes(crashRes.total_crashes || 0) + setDevices(deviceRes.devices || []) + } catch (err) { + setError(err.message || 'Failed to load crash report.') + } finally { + setLoading(false) + } + }, [range]) + + useEffect(() => { load() }, [load]) + + const devicesBySerial = useMemo(() => { + const map = {} + for (const d of devices) { + const sn = d.serial_number || d.device_id + if (sn) map[sn] = d + } + return map + }, [devices]) + + const affectedDevices = useMemo( + () => new Set(groups.flatMap(g => g.devices.map(d => d.device_serial))).size, + [groups], + ) + + return ( +
+ +
+ +
+ + + {loading ? ( +
+ +
+ ) : error ? ( +
+ {error} +
+ ) : groups.length === 0 ? ( + +
+ No crashes recorded in this range. +
+ + ) : ( + <> + + {totalCrashes} {totalCrashes === 1 ? 'crash' : 'crashes'} in {groups.length} {groups.length === 1 ? 'group' : 'groups'} across {affectedDevices} {affectedDevices === 1 ? 'device' : 'devices'} + +
+ {groups.map(g => ( + + ))} +
+ + )} +
+ ) +} diff --git a/frontend/src/pages/bellcloud/devices/DeviceList.jsx b/frontend/src/pages/bellcloud/devices/DeviceList.jsx index adc1df0..b8eb325 100644 --- a/frontend/src/pages/bellcloud/devices/DeviceList.jsx +++ b/frontend/src/pages/bellcloud/devices/DeviceList.jsx @@ -713,6 +713,9 @@ export default function DeviceList() { : undefined } > + {canEdit && ( + )} + > {bootEvents.length === 0 ? (
No boot events recorded in this range. @@ -888,7 +911,7 @@ function useHeaderStatus(sn, healthSettings, mqttStatus) { }) } -export default function HealthTab({ device, sn, mqttStatus }) { +export default function HealthTab({ device, sn, mqttStatus, isOnline, sendMqttCommand }) { const [subTab, setSubTab] = useState('overview') const [range, setRange] = useState(() => resolveRange('7d')) // Local override after a successful save, so the Settings form and offline- @@ -926,7 +949,7 @@ export default function HealthTab({ device, sn, mqttStatus }) {
{subTab === 'overview' && ( - + )} {subTab === 'logs' && } {subTab === 'settings' && ( diff --git a/frontend/src/pages/engineering/developer/ApiReferencePage.jsx b/frontend/src/pages/engineering/developer/ApiReferencePage.jsx index 8520ed3..8e91458 100644 --- a/frontend/src/pages/engineering/developer/ApiReferencePage.jsx +++ b/frontend/src/pages/engineering/developer/ApiReferencePage.jsx @@ -1946,7 +1946,7 @@ function TransportsV2() {
- +
{[ { field: 'boot_count', type: 'int', notes: 'Lifetime boot counter (NVS-backed, survives SD failure/removal). The authoritative "how many times has this device rebooted, ever" — also readable on demand via telemetry.get_metrics / system.get_telemetry / system.get_all as lifetime_boot_count.' }, @@ -1958,6 +1958,11 @@ function TransportsV2() { { field: 'crash.pc', type: 'int', notes: 'Program counter at the point of the exception.' }, { field: 'crash.exc_cause', type: 'int', notes: 'Xtensa exception cause code (matches EXCCAUSE in the serial register dump).' }, { field: 'crash.exc_vaddr', type: 'int', notes: 'Faulting virtual address (matches EXCVADDR in the serial register dump).' }, + { field: 'crash.backtrace', type: 'int[]?', notes: 'F-070. Up to 16 backtrace PCs (decimal uint32). Decode with xtensa-esp32-elf-addr2line against the matching firmware.elf.' }, + { field: 'crash.backtrace_corrupted', type: 'bool?', notes: 'F-070. true when the stack unwinder hit a corrupted frame, so later frames are unreliable.' }, + { field: 'crash.elf_sha256', type: 'string?', notes: 'F-070. First 16 hex chars of the crashing build\'s ELF SHA-256: identifies which firmware.elf decodes the addresses.' }, + { field: 'pre_crash', type: 'object?', notes: 'F-070. Independent of crash. Only on fault resets (never after power-on/brownout): uptime_ms, free_heap, largest_block, min_free_heap at the time of the crash.' }, + { field: 'pre_crash.abort_msg', type: 'string?', notes: 'F-070. Only for abort/assert/stack-overflow resets. When present, crash.pc / exc_cause 29 / exc_vaddr 0 describe the abort() mechanism, not the cause, and the first 2-4 backtrace frames are abort internals.' }, ].map(row => (
diff --git a/frontend/src/pages/engineering/firmware/FirmwareManager.jsx b/frontend/src/pages/engineering/firmware/FirmwareManager.jsx index 683aa69..36b0160 100644 --- a/frontend/src/pages/engineering/firmware/FirmwareManager.jsx +++ b/frontend/src/pages/engineering/firmware/FirmwareManager.jsx @@ -112,9 +112,11 @@ function FirmwareFormModal({ initial, onClose, onSaved }) { const [changelog, setChangelog] = useState(initial?.changelog ?? '') const [releaseNote, setReleaseNote] = useState(initial?.release_note ?? '') const [file, setFile] = useState(null) + const [elfFile, setElfFile] = useState(null) const [uploading, setUploading] = useState(false) const [error, setError] = useState('') const fileInputRef = useRef(null) + const elfInputRef = useRef(null) const handleSubmit = async (e) => { e.preventDefault() @@ -130,12 +132,14 @@ function FirmwareFormModal({ initial, onClose, onSaved }) { fd.append('changelog', changelog); fd.append('release_note', releaseNote) if (hwType === 'bespoke') fd.append('bespoke_uid', bespokeUid.trim()) fd.append('file', file) + if (elfFile) fd.append('elf_file', elfFile) } else { fd.append('channel', channel); fd.append('version', version) fd.append('update_type', updateType); fd.append('min_fw_version', minFw) fd.append('changelog', changelog); fd.append('release_note', releaseNote) if (hwType === 'bespoke') fd.append('bespoke_uid', bespokeUid.trim()) if (file) fd.append('file', file) + if (elfFile) fd.append('elf_file', elfFile) } const token = localStorage.getItem('access_token') const url = isEdit ? `/api/firmware/${initial.id}` : '/api/firmware/upload' @@ -274,6 +278,30 @@ function FirmwareFormModal({ initial, onClose, onSaved }) { )}
+ {/* Optional symbols for crash decoding — matched to crash reports by + the ELF's SHA-256 prefix (firmware F-070), not by version. */} +
+ { + const f = e.target.files[0] + if (f && !f.name.endsWith('.elf')) { setError('Only .elf files are accepted for symbols.'); return } + setElfFile(f || null) + }} + style={{ display: 'none' }} /> +
+ + {elfFile && ( + + {elfFile.name} · {formatBytes(elfFile.size)} + + )} +
+ + Optional. Lets the console decode crash backtraces from devices running this build. + +
diff --git a/frontend/src/router/index.jsx b/frontend/src/router/index.jsx index 27cab3a..897b2e6 100644 --- a/frontend/src/router/index.jsx +++ b/frontend/src/router/index.jsx @@ -9,6 +9,7 @@ import DashboardPage from '@/pages/dashboard/DashboardPage' import DeviceList from '@/pages/bellcloud/devices/DeviceList' import DeviceDetail from '@/pages/bellcloud/devices/DeviceDetail' import DeviceMapPage from '@/pages/bellcloud/devices/DeviceMapPage' +import CrashReport from '@/pages/bellcloud/devices/CrashReport' import OnboardDevice from '@/pages/bellcloud/devices/OnboardDevice' import StyleGuide from '@/pages/dev/StyleGuide' import CardFontSample from '@/pages/dev/CardFontSample' @@ -145,6 +146,7 @@ export default function V2Router() { } /> } /> {/* Map preview — temporary unlisted route while map view is under construction */} + } /> } /> } /> } />