feat(devices): crash detail view and fleet Crash Report (F-070)
- CrashDetail (components/shared): headline is abort_msg when present, else the Xtensa exception name. For aborts, pc/exception/fault address are shown muted with a note that they describe abort() and that the first 2-4 frames are abort internals. Shows task, uptime at crash, heap stats with a "likely heap exhaustion / fragmentation" badge (largest_block < 12000 or free_heap < 25000, F-055), PC + backtrace as 0x%08X in ESP serial format, a copyable addr2line command (PC, backtrace, then the PC parsed from "abort() was called at PC ..."), elf_sha256, and a backtrace-corrupted flag. When the server has the build's ELF and addr2line, a Decode button shows function + file:line per frame. - Health tab boot history uses it, previews the headline in the row, and gets a "Sync from device" button (telemetry.get_boot_history, merged server-side). - New /devices/crashes page (linked from the Fleet header): crashes grouped across devices with per-device counts and the latest occurrence. - Firmware release modal: optional firmware.elf attachment. - API reference: boot_report payload documents the F-070 fields. - lib/crashReport.js holds the parsing/formatting rules; lib/clipboard.js a copy helper with a non-secure-context fallback. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,200 @@
|
||||
// frontend/src/components/shared/CrashDetail.jsx
|
||||
// Crash detail for one fault boot event (firmware F-070): headline, task,
|
||||
// uptime + heap snapshot at crash, PC/backtrace in ESP serial format, a
|
||||
// ready-to-run addr2line command, the crashing build's ELF id, and — when the
|
||||
// server has that build's firmware.elf and the xtensa addr2line binary —
|
||||
// in-console decoding to function + file:line.
|
||||
//
|
||||
// Used by the device Health tab's boot history and the fleet Crash Report.
|
||||
//
|
||||
// Props:
|
||||
// event — a boot event from /mqtt/boot-events (crash / pre_crash objects,
|
||||
// or only the legacy crash_* columns for older firmware)
|
||||
|
||||
import { useState, useEffect } from 'react'
|
||||
import api from '@/lib/api'
|
||||
import Button from '@/components/ui/Button'
|
||||
import StatusBadge from '@/components/ui/StatusBadge'
|
||||
import Icon from '@/components/ui/Icon'
|
||||
import { copyText } from '@/lib/clipboard'
|
||||
import {
|
||||
crashOf, preCrashOf, crashHeadline, excCauseName, hex32, isHeapExhaustion,
|
||||
addressLine, addr2lineCommand, decodeAddresses, fmtBytes, fmtUptime,
|
||||
} from '@/lib/crashReport'
|
||||
|
||||
const labelStyle = {
|
||||
fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)',
|
||||
textTransform: 'uppercase', letterSpacing: 'var(--tracking-wide)',
|
||||
}
|
||||
const monoBlockStyle = {
|
||||
flex: 1, minWidth: 0, margin: 0,
|
||||
padding: 'var(--space-2) var(--space-3)',
|
||||
background: 'var(--color-bg-abyss)', border: '1px solid var(--color-border)',
|
||||
borderRadius: 'var(--radius-md)',
|
||||
fontFamily: 'var(--font-family-mono)', fontSize: 'var(--font-size-xs)',
|
||||
color: 'var(--color-text-primary)',
|
||||
whiteSpace: 'pre-wrap', wordBreak: 'break-all',
|
||||
}
|
||||
|
||||
function Field({ label, value, muted, mono = true }) {
|
||||
return (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 'var(--space-1)', minWidth: 0 }}>
|
||||
<span style={labelStyle}>{label}</span>
|
||||
<span style={{
|
||||
fontSize: 'var(--font-size-sm)',
|
||||
fontFamily: mono ? 'var(--font-family-mono)' : undefined,
|
||||
color: muted ? 'var(--color-text-muted)' : 'var(--color-text-primary)',
|
||||
wordBreak: 'break-all',
|
||||
}}>
|
||||
{value ?? '—'}
|
||||
</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function CopyRow({ label, text }) {
|
||||
const [copied, setCopied] = useState(false)
|
||||
const handleCopy = () => {
|
||||
copyText(text).then(() => {
|
||||
setCopied(true)
|
||||
setTimeout(() => setCopied(false), 1500)
|
||||
}).catch(() => {})
|
||||
}
|
||||
return (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 'var(--space-1)' }}>
|
||||
<span style={labelStyle}>{label}</span>
|
||||
<div style={{ display: 'flex', alignItems: 'flex-start', gap: 'var(--space-2)' }}>
|
||||
<pre style={monoBlockStyle}>{text}</pre>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
icon={<Icon name={copied ? 'check' : 'copy'} size={14} />}
|
||||
onClick={handleCopy}
|
||||
aria-label={`Copy ${label}`}
|
||||
>
|
||||
{copied ? 'Copied' : 'Copy'}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function DecodePanel({ event, elfSha256 }) {
|
||||
const [info, setInfo] = useState(null) // /firmware/elf/{sha}
|
||||
const [results, setResults] = useState(null) // decoded frames
|
||||
const [decoding, setDecoding] = useState(false)
|
||||
const [error, setError] = useState('')
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
setInfo(null); setResults(null); setError('')
|
||||
api.get(`/firmware/elf/${elfSha256}`)
|
||||
.then(res => { if (!cancelled) setInfo(res) })
|
||||
.catch(() => { if (!cancelled) setInfo({ exists: false, decoder_available: false, unavailable: true }) })
|
||||
return () => { cancelled = true }
|
||||
}, [elfSha256])
|
||||
|
||||
const handleDecode = async () => {
|
||||
setDecoding(true); setError('')
|
||||
try {
|
||||
const res = await api.post(`/firmware/elf/${elfSha256}/decode`, { addresses: decodeAddresses(event) })
|
||||
setResults(res.results || [])
|
||||
} catch (err) {
|
||||
setError(err.message || 'Decoding failed.')
|
||||
} finally {
|
||||
setDecoding(false)
|
||||
}
|
||||
}
|
||||
|
||||
if (!info) return null
|
||||
|
||||
let note = null
|
||||
if (info.unavailable) note = 'Could not check the symbol store for this build.'
|
||||
else if (!info.exists) note = `No firmware.elf uploaded for build ${elfSha256}. Attach it to the firmware release to decode here — or use the command above.`
|
||||
else if (!info.decoder_available) note = 'The ELF for this build is on the server, but the xtensa addr2line decoder is not installed there — use the command above.'
|
||||
|
||||
return (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 'var(--space-2)' }}>
|
||||
<span style={labelStyle}>Decoded backtrace</span>
|
||||
{note ? (
|
||||
<span style={{ fontSize: 'var(--font-size-sm)', color: 'var(--color-text-muted)' }}>{note}</span>
|
||||
) : results ? (
|
||||
results.length === 0 ? (
|
||||
<span style={{ fontSize: 'var(--font-size-sm)', color: 'var(--color-text-muted)' }}>No addresses to decode.</span>
|
||||
) : (
|
||||
<pre style={monoBlockStyle}>
|
||||
{results.map(r => `${r.address}: ${r.frames.join('\n ')}`).join('\n')}
|
||||
</pre>
|
||||
)
|
||||
) : (
|
||||
<div>
|
||||
<Button variant="secondary" size="sm" onClick={handleDecode} loading={decoding}>
|
||||
Decode with firmware.elf
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
{error && <span style={{ fontSize: 'var(--font-size-sm)', color: 'var(--color-danger)' }}>{error}</span>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default function CrashDetail({ event }) {
|
||||
const crash = crashOf(event)
|
||||
const pre = preCrashOf(event)
|
||||
if (!crash && !pre) return null
|
||||
|
||||
const abortMsg = pre?.abort_msg || null
|
||||
const headline = crashHeadline(event)
|
||||
const heapLow = isHeapExhaustion(pre)
|
||||
const addrs = addressLine(crash)
|
||||
const command = addr2lineCommand(event)
|
||||
|
||||
return (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 'var(--space-4)', minWidth: 0 }}>
|
||||
{/* Headline — abort_msg when present, else the exception cause name */}
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 'var(--space-2)' }}>
|
||||
<span style={{
|
||||
fontFamily: 'var(--font-family-mono)', fontSize: 'var(--font-size-md)',
|
||||
fontWeight: 'var(--font-weight-semibold)', color: 'var(--color-danger)', wordBreak: 'break-word',
|
||||
}}>
|
||||
{headline || 'Fault reset (no coredump)'}
|
||||
</span>
|
||||
<div style={{ display: 'flex', flexWrap: 'wrap', gap: 'var(--space-2)' }}>
|
||||
{heapLow && <StatusBadge variant="warning" size="sm">Likely heap exhaustion / fragmentation</StatusBadge>}
|
||||
{crash?.backtraceCorrupted && <StatusBadge variant="warning" size="sm">Backtrace corrupted</StatusBadge>}
|
||||
{!crash && <StatusBadge variant="neutral" size="sm">No coredump</StatusBadge>}
|
||||
</div>
|
||||
{abortMsg && crash && (
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)' }}>
|
||||
This was an abort. The PC and exception below describe the abort() mechanism, not the cause,
|
||||
and the first 2–4 backtrace frames are abort internals — the real call site is further down.
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(150px, 1fr))', gap: 'var(--space-3)' }}>
|
||||
<Field label="Faulting task" value={crash?.task} />
|
||||
<Field label="Uptime at crash" value={pre ? fmtUptime(pre.uptime_ms) : null} />
|
||||
<Field label="Free heap" value={pre ? fmtBytes(pre.free_heap) : null} />
|
||||
<Field label="Largest free block" value={pre ? fmtBytes(pre.largest_block) : null} />
|
||||
<Field label="Min free heap" value={pre ? fmtBytes(pre.min_free_heap) : null} />
|
||||
{crash && (
|
||||
<>
|
||||
<Field
|
||||
label="Exception"
|
||||
value={crash.excCause != null ? `${excCauseName(crash.excCause)} (${crash.excCause})` : null}
|
||||
muted={!!abortMsg}
|
||||
/>
|
||||
<Field label="Program counter" value={hex32(crash.pc)} muted={!!abortMsg} />
|
||||
<Field label="Faulting address" value={hex32(crash.excVaddr)} muted={!!abortMsg} />
|
||||
<Field label="Build (ELF SHA-256)" value={crash.elfSha256} />
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{addrs && <CopyRow label="PC + backtrace" text={addrs} />}
|
||||
{command && <CopyRow label="addr2line command" text={command} />}
|
||||
{crash?.elfSha256 && command && <DecodePanel event={event} elfSha256={crash.elfSha256} />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// frontend/src/lib/clipboard.js
|
||||
// Copy text to the clipboard, with a textarea fallback for non-secure
|
||||
// contexts (plain-HTTP LAN access) where navigator.clipboard is undefined.
|
||||
|
||||
export function copyText(text) {
|
||||
if (navigator.clipboard?.writeText) {
|
||||
return navigator.clipboard.writeText(text).catch(() => fallbackCopy(text))
|
||||
}
|
||||
return fallbackCopy(text)
|
||||
}
|
||||
|
||||
function fallbackCopy(text) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ta = document.createElement('textarea')
|
||||
ta.value = text
|
||||
ta.setAttribute('readonly', '')
|
||||
ta.style.cssText = 'position:fixed;opacity:0'
|
||||
document.body.appendChild(ta)
|
||||
ta.select()
|
||||
try {
|
||||
document.execCommand('copy') ? resolve() : reject(new Error('Copy failed'))
|
||||
} catch (err) {
|
||||
reject(err)
|
||||
} finally {
|
||||
document.body.removeChild(ta)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// frontend/src/lib/crashReport.js
|
||||
// Helpers for firmware crash detail (F-070) on boot events: the `crash`
|
||||
// (coredump summary) and `pre_crash` (heap/uptime snapshot + optional
|
||||
// abort_msg) objects from boot_report, with fallback to the legacy flat
|
||||
// crash_* columns for older rows/firmware.
|
||||
//
|
||||
// Key rule: when pre_crash.abort_msg is present, pc / exc_cause 29 /
|
||||
// exc_vaddr 0 describe the abort() mechanism, NOT the real cause — the
|
||||
// abort_msg is the headline and those fields are de-emphasised.
|
||||
|
||||
// Xtensa EXCCAUSE values the ESP32 actually produces.
|
||||
export const EXC_CAUSE_NAMES = {
|
||||
0: 'IllegalInstruction',
|
||||
2: 'InstructionFetchError',
|
||||
3: 'LoadStoreError',
|
||||
6: 'IntegerDivideByZero',
|
||||
9: 'LoadStoreAlignment',
|
||||
28: 'LoadProhibited',
|
||||
29: 'StoreProhibited',
|
||||
}
|
||||
|
||||
// Firmware F-055 thresholds: below these the crash was most likely caused by
|
||||
// heap exhaustion or fragmentation rather than a logic bug.
|
||||
export const HEAP_LARGEST_BLOCK_WARN = 12000
|
||||
export const HEAP_FREE_WARN = 25000
|
||||
|
||||
export function excCauseName(cause) {
|
||||
if (cause == null) return 'Unknown exception'
|
||||
return EXC_CAUSE_NAMES[cause] ?? `Exception cause ${cause}`
|
||||
}
|
||||
|
||||
// 0x%08X — the ESP serial monitor's address format.
|
||||
export function hex32(n) {
|
||||
if (n == null || !Number.isFinite(Number(n))) return '—'
|
||||
return `0x${(Number(n) >>> 0).toString(16).toUpperCase().padStart(8, '0')}`
|
||||
}
|
||||
|
||||
// Merge the JSON `crash` object with the legacy flat columns so callers deal
|
||||
// with one shape. Returns null when the boot has no coredump summary at all.
|
||||
export function crashOf(event) {
|
||||
const c = event?.crash || {}
|
||||
const task = c.task ?? event?.crash_task
|
||||
const pc = c.pc ?? event?.crash_pc
|
||||
if (task == null && pc == null && !event?.crash) return null
|
||||
return {
|
||||
task,
|
||||
pc,
|
||||
excCause: c.exc_cause ?? event?.crash_exc_cause,
|
||||
excVaddr: c.exc_vaddr ?? event?.crash_exc_vaddr,
|
||||
backtrace: Array.isArray(c.backtrace) ? c.backtrace : [],
|
||||
backtraceCorrupted: !!c.backtrace_corrupted,
|
||||
elfSha256: c.elf_sha256 || null,
|
||||
}
|
||||
}
|
||||
|
||||
export function preCrashOf(event) {
|
||||
return event?.pre_crash || null
|
||||
}
|
||||
|
||||
export function hasCrashDetail(event) {
|
||||
return !!(crashOf(event) || preCrashOf(event))
|
||||
}
|
||||
|
||||
// "abort() was called at PC 0x400d8a3f on core 0" → 0x400D8A3F (the caller of
|
||||
// abort(), worth decoding too). Only that phrasing carries a PC.
|
||||
export function abortCallerPc(abortMsg) {
|
||||
const m = /abort\(\) was called at PC (0x[0-9a-fA-F]+)/.exec(abortMsg || '')
|
||||
return m ? parseInt(m[1], 16) : null
|
||||
}
|
||||
|
||||
export function crashHeadline(event) {
|
||||
const abortMsg = preCrashOf(event)?.abort_msg
|
||||
if (abortMsg) return abortMsg
|
||||
const crash = crashOf(event)
|
||||
if (crash) return excCauseName(crash.excCause)
|
||||
return null
|
||||
}
|
||||
|
||||
export function isHeapExhaustion(preCrash) {
|
||||
if (!preCrash) return false
|
||||
return (preCrash.largest_block != null && preCrash.largest_block < HEAP_LARGEST_BLOCK_WARN)
|
||||
|| (preCrash.free_heap != null && preCrash.free_heap < HEAP_FREE_WARN)
|
||||
}
|
||||
|
||||
// One line, space separated, like the ESP panic handler prints it.
|
||||
export function addressLine(crash) {
|
||||
if (!crash) return ''
|
||||
return [crash.pc, ...crash.backtrace].filter(a => a != null).map(hex32).join(' ')
|
||||
}
|
||||
|
||||
// Every address worth decoding: PC, backtrace, then the abort() caller PC.
|
||||
export function decodeAddresses(event) {
|
||||
const crash = crashOf(event)
|
||||
const out = []
|
||||
if (crash?.pc != null) out.push(Number(crash.pc))
|
||||
for (const a of crash?.backtrace || []) out.push(Number(a))
|
||||
const caller = abortCallerPc(preCrashOf(event)?.abort_msg)
|
||||
if (caller != null) out.push(caller)
|
||||
return out
|
||||
}
|
||||
|
||||
export function addr2lineCommand(event) {
|
||||
const addrs = decodeAddresses(event)
|
||||
if (!addrs.length) return ''
|
||||
return `xtensa-esp32-elf-addr2line -pfiaC -e firmware.elf ${addrs.map(hex32).join(' ')}`
|
||||
}
|
||||
|
||||
export function fmtBytes(n) {
|
||||
if (n == null) return '—'
|
||||
if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`
|
||||
return `${n} B`
|
||||
}
|
||||
|
||||
export function fmtUptime(ms) {
|
||||
if (ms == null) return '—'
|
||||
const s = Math.floor(ms / 1000)
|
||||
const d = Math.floor(s / 86400)
|
||||
const h = Math.floor((s % 86400) / 3600)
|
||||
const m = Math.floor((s % 3600) / 60)
|
||||
const sec = s % 60
|
||||
const parts = [d && `${d}d`, h && `${h}h`, m && `${m}m`]
|
||||
if (!d && !h) parts.push(`${sec}s`)
|
||||
return parts.filter(Boolean).join(' ') || '0s'
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
// frontend/src/pages/bellcloud/devices/CrashReport.jsx
|
||||
// Fleet-wide crash report (firmware F-070): every fault boot with crash
|
||||
// detail, grouped so recurring crash types stand out across devices.
|
||||
// Grouping (backend mqtt/crash_groups.py): abort_msg with hex addresses
|
||||
// stripped when present, else faulting task + exception cause, else reset
|
||||
// reason for fault resets with no coredump.
|
||||
|
||||
import { useState, useEffect, useCallback, useMemo } from 'react'
|
||||
import { Link } from 'react-router-dom'
|
||||
import api from '@/lib/api'
|
||||
import PageHeader from '@/components/ui/PageHeader'
|
||||
import Card from '@/components/ui/Card'
|
||||
import Button from '@/components/ui/Button'
|
||||
import StatusBadge from '@/components/ui/StatusBadge'
|
||||
import Spinner from '@/components/ui/Spinner'
|
||||
import Icon from '@/components/ui/Icon'
|
||||
import TimeRangeSelect from '@/components/ui/TimeRangeSelect'
|
||||
import { resolveRange } from '@/lib/timeRange'
|
||||
import CrashDetail from '@/components/shared/CrashDetail'
|
||||
import { fmtDateTimeMedium, fmtRelative } from '@/lib/formatters'
|
||||
import { isHeapExhaustion } from '@/lib/crashReport'
|
||||
|
||||
const KIND_LABEL = { abort: 'Abort', exception: 'Exception', reason: 'No coredump' }
|
||||
const KIND_VARIANT = { abort: 'danger', exception: 'danger', reason: 'warning' }
|
||||
|
||||
function deviceLabel(device) {
|
||||
return device?.console_name || device?.device_name || null
|
||||
}
|
||||
|
||||
function CrashGroupCard({ group, devicesBySerial }) {
|
||||
const [showDetail, setShowDetail] = useState(false)
|
||||
const heapLow = isHeapExhaustion(group.latest?.pre_crash)
|
||||
|
||||
return (
|
||||
<Card
|
||||
title={group.title}
|
||||
subtitle={`${group.count} ${group.count === 1 ? 'crash' : 'crashes'} on ${group.device_count} ${group.device_count === 1 ? 'device' : 'devices'} · last ${fmtRelative(group.last_at)}`}
|
||||
action={<StatusBadge variant={KIND_VARIANT[group.kind] || 'neutral'} size="sm">{KIND_LABEL[group.kind] || group.kind}</StatusBadge>}
|
||||
>
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 'var(--space-4)' }}>
|
||||
<div style={{ display: 'flex', flexWrap: 'wrap', gap: 'var(--space-2) var(--space-4)', fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)' }}>
|
||||
<span>First seen {fmtDateTimeMedium(group.first_at)}</span>
|
||||
<span>Last seen {fmtDateTimeMedium(group.last_at)}</span>
|
||||
{heapLow && <StatusBadge variant="warning" size="sm">Latest: likely heap exhaustion</StatusBadge>}
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', flexDirection: 'column' }}>
|
||||
{group.devices.map(d => {
|
||||
const device = devicesBySerial[d.device_serial]
|
||||
const name = deviceLabel(device)
|
||||
return (
|
||||
<div
|
||||
key={d.device_serial}
|
||||
style={{
|
||||
display: 'flex', alignItems: 'center', gap: 'var(--space-3)', flexWrap: 'wrap',
|
||||
padding: 'var(--space-2) 0', borderBottom: '1px solid var(--color-border)',
|
||||
}}
|
||||
>
|
||||
<span style={{ flex: 1, minWidth: 0, display: 'flex', flexDirection: 'column' }}>
|
||||
{device ? (
|
||||
<Link to={`/devices/${device.id}?tab=health`} style={{ color: 'var(--color-text-accent)', fontSize: 'var(--font-size-sm)' }}>
|
||||
{name || d.device_serial}
|
||||
</Link>
|
||||
) : (
|
||||
<span style={{ fontSize: 'var(--font-size-sm)', color: 'var(--color-text-primary)' }}>{d.device_serial}</span>
|
||||
)}
|
||||
{name && (
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)', fontFamily: 'var(--font-family-mono)' }}>
|
||||
{d.device_serial}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)', fontFamily: 'var(--font-family-mono)' }}>
|
||||
{d.count}×
|
||||
</span>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
iconRight={<Icon name="chevron_down" size={14} style={{ transform: showDetail ? 'rotate(180deg)' : 'none', transition: 'transform 0.15s' }} />}
|
||||
onClick={() => setShowDetail(v => !v)}
|
||||
>
|
||||
{showDetail ? 'Hide latest occurrence' : 'Show latest occurrence'}
|
||||
</Button>
|
||||
</div>
|
||||
{showDetail && (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 'var(--space-3)' }}>
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)' }}>
|
||||
{group.latest.device_serial} · boot #{group.latest.boot_count ?? '—'} · {fmtDateTimeMedium(group.latest.occurred_at)}
|
||||
</span>
|
||||
<CrashDetail event={group.latest} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
export default function CrashReport() {
|
||||
const [range, setRange] = useState(() => resolveRange('30d'))
|
||||
const [groups, setGroups] = useState([])
|
||||
const [totalCrashes, setTotalCrashes] = useState(0)
|
||||
const [devices, setDevices] = useState([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [error, setError] = useState('')
|
||||
|
||||
const load = useCallback(async () => {
|
||||
setLoading(true)
|
||||
setError('')
|
||||
try {
|
||||
const p = new URLSearchParams()
|
||||
if (range.since) p.set('since', range.since.toISOString())
|
||||
if (range.until) p.set('until', range.until.toISOString())
|
||||
const qs = p.toString()
|
||||
const [crashRes, deviceRes] = await Promise.all([
|
||||
api.get(`/mqtt/crash-groups${qs ? `?${qs}` : ''}`),
|
||||
// Only for names/links — the report itself doesn't depend on it.
|
||||
api.get('/devices').catch(() => ({ devices: [] })),
|
||||
])
|
||||
setGroups(crashRes.groups || [])
|
||||
setTotalCrashes(crashRes.total_crashes || 0)
|
||||
setDevices(deviceRes.devices || [])
|
||||
} catch (err) {
|
||||
setError(err.message || 'Failed to load crash report.')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}, [range])
|
||||
|
||||
useEffect(() => { load() }, [load])
|
||||
|
||||
const devicesBySerial = useMemo(() => {
|
||||
const map = {}
|
||||
for (const d of devices) {
|
||||
const sn = d.serial_number || d.device_id
|
||||
if (sn) map[sn] = d
|
||||
}
|
||||
return map
|
||||
}, [devices])
|
||||
|
||||
const affectedDevices = useMemo(
|
||||
() => new Set(groups.flatMap(g => g.devices.map(d => d.device_serial))).size,
|
||||
[groups],
|
||||
)
|
||||
|
||||
return (
|
||||
<div className="page-wrapper">
|
||||
<PageHeader
|
||||
title="Crash Report"
|
||||
subtitle="Fault resets across the fleet, grouped by crash signature"
|
||||
breadcrumbs={[{ label: 'Device Fleet', href: '/devices' }, { label: 'Crash Report' }]}
|
||||
>
|
||||
<div style={{ width: 'min(220px, 100%)' }}>
|
||||
<TimeRangeSelect value={range} onChange={setRange} />
|
||||
</div>
|
||||
</PageHeader>
|
||||
|
||||
{loading ? (
|
||||
<div style={{ display: 'flex', justifyContent: 'center', padding: 'var(--space-16)' }}>
|
||||
<Spinner size="lg" />
|
||||
</div>
|
||||
) : error ? (
|
||||
<div role="alert" style={{ backgroundColor: 'var(--color-danger-bg)', border: '1px solid var(--color-danger)', borderRadius: 'var(--radius-md)', padding: 'var(--space-4)', color: 'var(--color-danger)' }}>
|
||||
{error}
|
||||
</div>
|
||||
) : groups.length === 0 ? (
|
||||
<Card>
|
||||
<div style={{ padding: 'var(--space-8)', textAlign: 'center', color: 'var(--color-text-muted)', fontSize: 'var(--font-size-sm)' }}>
|
||||
No crashes recorded in this range.
|
||||
</div>
|
||||
</Card>
|
||||
) : (
|
||||
<>
|
||||
<span style={{ fontSize: 'var(--font-size-sm)', color: 'var(--color-text-muted)' }}>
|
||||
{totalCrashes} {totalCrashes === 1 ? 'crash' : 'crashes'} in {groups.length} {groups.length === 1 ? 'group' : 'groups'} across {affectedDevices} {affectedDevices === 1 ? 'device' : 'devices'}
|
||||
</span>
|
||||
<div className="masonry-grid masonry-grid--2">
|
||||
{groups.map(g => (
|
||||
<CrashGroupCard key={g.key} group={g} devicesBySerial={devicesBySerial} />
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -713,6 +713,9 @@ export default function DeviceList() {
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
<Button variant="secondary" icon={<Icon name="warning" size={14} />} onClick={() => navigate('/devices/crashes')}>
|
||||
Crash Report
|
||||
</Button>
|
||||
{canEdit && (
|
||||
<Button variant="primary" onClick={() => navigate('/devices/new')}>
|
||||
Add Device
|
||||
|
||||
@@ -22,6 +22,8 @@ import LogsTab from '@/pages/bellcloud/devices/tabs/LogsTab'
|
||||
import { fmtDateTimeMedium, fmtRelative } from '@/lib/formatters'
|
||||
import { resolveRange } from '@/lib/timeRange'
|
||||
import { useToast } from '@/components/ui/Toast'
|
||||
import CrashDetail from '@/components/shared/CrashDetail'
|
||||
import { hasCrashDetail, crashHeadline, isHeapExhaustion } from '@/lib/crashReport'
|
||||
|
||||
// ─── Reset reason → badge variant ──────────────────────────────────────────
|
||||
|
||||
@@ -189,13 +191,15 @@ function RestartTimeline({ events, rangeStart, rangeEnd }) {
|
||||
// ─── Boot event row (expandable for crash detail) ──────────────────────────
|
||||
|
||||
function BootEventRow({ event, expanded, onToggle }) {
|
||||
const hasCrash = event.crash_task != null
|
||||
const hasCrash = hasCrashDetail(event)
|
||||
const headline = hasCrash ? crashHeadline(event) : null
|
||||
const heapLow = isHeapExhaustion(event.pre_crash)
|
||||
return (
|
||||
<div style={{ borderBottom: '1px solid var(--color-border)' }}>
|
||||
<div
|
||||
onClick={hasCrash ? onToggle : undefined}
|
||||
style={{
|
||||
display: 'flex', alignItems: 'center', gap: 'var(--space-4)',
|
||||
display: 'flex', alignItems: 'center', gap: 'var(--space-4)', flexWrap: 'wrap',
|
||||
padding: 'var(--space-3) var(--space-4)',
|
||||
cursor: hasCrash ? 'pointer' : 'default',
|
||||
}}
|
||||
@@ -209,10 +213,19 @@ function BootEventRow({ event, expanded, onToggle }) {
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)', fontFamily: 'var(--font-family-mono)' }}>
|
||||
boot #{event.boot_count ?? '—'}
|
||||
</span>
|
||||
<span style={{ flex: 1 }} />
|
||||
{headline && (
|
||||
<span style={{
|
||||
flex: 1, minWidth: 0, fontSize: 'var(--font-size-xs)', color: 'var(--color-danger)',
|
||||
fontFamily: 'var(--font-family-mono)', overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap',
|
||||
}}>
|
||||
{headline}
|
||||
</span>
|
||||
)}
|
||||
{!headline && <span style={{ flex: 1 }} />}
|
||||
{heapLow && <StatusBadge variant="warning" size="sm">Low heap</StatusBadge>}
|
||||
{event.free_heap != null && (
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)', fontFamily: 'var(--font-family-mono)' }}>
|
||||
{Math.round(event.free_heap / 1024)} KB free
|
||||
{Math.round(event.free_heap / 1024)} KB free at boot
|
||||
</span>
|
||||
)}
|
||||
{hasCrash && (
|
||||
@@ -225,25 +238,8 @@ function BootEventRow({ event, expanded, onToggle }) {
|
||||
)}
|
||||
</div>
|
||||
{expanded && hasCrash && (
|
||||
<div style={{
|
||||
padding: 'var(--space-3) var(--space-4) var(--space-4) calc(160px + var(--space-4) + var(--space-4))',
|
||||
display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(140px, 1fr))', gap: 'var(--space-3)',
|
||||
}}>
|
||||
{[
|
||||
['Faulting task', event.crash_task],
|
||||
['Program counter', event.crash_pc != null ? `0x${Number(event.crash_pc).toString(16)}` : '—'],
|
||||
['Exception cause', event.crash_exc_cause ?? '—'],
|
||||
['Faulting address', event.crash_exc_vaddr != null ? `0x${Number(event.crash_exc_vaddr).toString(16)}` : '—'],
|
||||
].map(([label, val]) => (
|
||||
<div key={label} style={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)', textTransform: 'uppercase', letterSpacing: 'var(--tracking-wide)' }}>
|
||||
{label}
|
||||
</span>
|
||||
<span style={{ fontSize: 'var(--font-size-sm)', color: 'var(--color-danger)', fontFamily: 'var(--font-family-mono)' }}>
|
||||
{val}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
<div style={{ padding: 'var(--space-2) var(--space-4) var(--space-5)' }}>
|
||||
<CrashDetail event={event} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -298,7 +294,7 @@ function StatusIcon({ status }) {
|
||||
|
||||
// ─── Health overview sub-tab (stats + charts + boot history) ───────────────
|
||||
|
||||
function HealthOverview({ sn, mqttStatus, range, offlineTimeoutSec, healthSettings }) {
|
||||
function HealthOverview({ sn, mqttStatus, range, offlineTimeoutSec, healthSettings, isOnline, sendMqttCommand }) {
|
||||
const [bootEvents, setBootEvents] = useState([])
|
||||
const [heartbeats, setHeartbeats] = useState([])
|
||||
const [pingSamples, setPingSamples] = useState([])
|
||||
@@ -306,6 +302,7 @@ function HealthOverview({ sn, mqttStatus, range, offlineTimeoutSec, healthSettin
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [error, setError] = useState('')
|
||||
const [expandedId, setExpandedId] = useState(null)
|
||||
const [syncing, setSyncing] = useState(false)
|
||||
|
||||
const rangeParams = useMemo(() => {
|
||||
const p = new URLSearchParams()
|
||||
@@ -339,6 +336,23 @@ function HealthOverview({ sn, mqttStatus, range, offlineTimeoutSec, healthSettin
|
||||
|
||||
useEffect(() => { load() }, [load])
|
||||
|
||||
// Asks the device for its own SD boot log. The backend merges the reply
|
||||
// into boot history as it arrives (fills in crash detail for boots we
|
||||
// already have, adds boots the console never saw live), before the ack
|
||||
// reaches us — so a reload right after the reply shows the merged result.
|
||||
const syncFromDevice = async () => {
|
||||
if (!sendMqttCommand) return
|
||||
setSyncing(true)
|
||||
try {
|
||||
await sendMqttCommand('telemetry.get_boot_history')
|
||||
await load()
|
||||
} catch {
|
||||
// the command toast already reports the failure
|
||||
} finally {
|
||||
setSyncing(false)
|
||||
}
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div style={{ display: 'flex', justifyContent: 'center', padding: 'var(--space-16)' }}>
|
||||
@@ -606,7 +620,16 @@ function HealthOverview({ sn, mqttStatus, range, offlineTimeoutSec, healthSettin
|
||||
</Card>
|
||||
|
||||
{/* ── Full boot event history ──────────────────────────────────────── */}
|
||||
<Card title="Boot History" subtitle="Reverse-chronological — click a fault row with crash detail to expand" padding={false}>
|
||||
<Card
|
||||
title="Boot History"
|
||||
subtitle="Reverse-chronological — click a fault row with crash detail to expand"
|
||||
padding={false}
|
||||
action={sendMqttCommand && (
|
||||
<Button variant="secondary" size="sm" icon={<Icon name="refresh" size={14} />} onClick={syncFromDevice} loading={syncing} disabled={!isOnline} title={isOnline ? "Fetch the device's own boot log and merge it into this history" : 'Device is offline'}>
|
||||
Sync from device
|
||||
</Button>
|
||||
)}
|
||||
>
|
||||
{bootEvents.length === 0 ? (
|
||||
<div style={{ padding: 'var(--space-8)', textAlign: 'center', color: 'var(--color-text-muted)', fontSize: 'var(--font-size-sm)', fontStyle: 'italic' }}>
|
||||
No boot events recorded in this range.
|
||||
@@ -888,7 +911,7 @@ function useHeaderStatus(sn, healthSettings, mqttStatus) {
|
||||
})
|
||||
}
|
||||
|
||||
export default function HealthTab({ device, sn, mqttStatus }) {
|
||||
export default function HealthTab({ device, sn, mqttStatus, isOnline, sendMqttCommand }) {
|
||||
const [subTab, setSubTab] = useState('overview')
|
||||
const [range, setRange] = useState(() => resolveRange('7d'))
|
||||
// Local override after a successful save, so the Settings form and offline-
|
||||
@@ -926,7 +949,7 @@ export default function HealthTab({ device, sn, mqttStatus }) {
|
||||
</div>
|
||||
|
||||
{subTab === 'overview' && (
|
||||
<HealthOverview sn={sn} mqttStatus={mqttStatus} range={range} offlineTimeoutSec={offlineTimeoutSec} healthSettings={healthSettings} />
|
||||
<HealthOverview sn={sn} mqttStatus={mqttStatus} range={range} offlineTimeoutSec={offlineTimeoutSec} healthSettings={healthSettings} isOnline={isOnline} sendMqttCommand={sendMqttCommand} />
|
||||
)}
|
||||
{subTab === 'logs' && <LogsTab sn={sn} range={range} />}
|
||||
{subTab === 'settings' && (
|
||||
|
||||
@@ -1946,7 +1946,7 @@ function TransportsV2() {
|
||||
|
||||
<Card title="Boot Report Payload" subtitle="Published once per boot, on first MQTT connect, to vesper/{device_id}/system/info — QoS 1, retained">
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 'var(--space-4)' }}>
|
||||
<MonoBlock code={'{\n "type": "boot_report",\n "payload": {\n "boot_count": 47,\n "reset_reason": "PANIC",\n "is_fault": true,\n "free_heap": 187234,\n "crash": {\n "task": "TelemetryTask",\n "pc": 1074111832,\n "exc_cause": 1,\n "exc_vaddr": 0\n }\n }\n}'} />
|
||||
<MonoBlock code={'{\n "type": "boot_report",\n "payload": {\n "boot_count": 47,\n "reset_reason": "PANIC",\n "is_fault": true,\n "free_heap": 187234,\n "crash": {\n "task": "async_tcp",\n "pc": 1074282197,\n "exc_cause": 29,\n "exc_vaddr": 0,\n "backtrace": [1074282194, 1074340312, 1074348122],\n "backtrace_corrupted": false,\n "elf_sha256": "3f9a1c0be7d2a441"\n },\n "pre_crash": {\n "uptime_ms": 5412000,\n "free_heap": 21480,\n "largest_block": 9716,\n "min_free_heap": 18112,\n "abort_msg": "abort() was called at PC 0x400d8a3f on core 0"\n }\n }\n}'} />
|
||||
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(160px, 1fr))', gap: 'var(--space-3)' }}>
|
||||
{[
|
||||
{ field: 'boot_count', type: 'int', notes: 'Lifetime boot counter (NVS-backed, survives SD failure/removal). The authoritative "how many times has this device rebooted, ever" — also readable on demand via telemetry.get_metrics / system.get_telemetry / system.get_all as lifetime_boot_count.' },
|
||||
@@ -1958,6 +1958,11 @@ function TransportsV2() {
|
||||
{ field: 'crash.pc', type: 'int', notes: 'Program counter at the point of the exception.' },
|
||||
{ field: 'crash.exc_cause', type: 'int', notes: 'Xtensa exception cause code (matches EXCCAUSE in the serial register dump).' },
|
||||
{ field: 'crash.exc_vaddr', type: 'int', notes: 'Faulting virtual address (matches EXCVADDR in the serial register dump).' },
|
||||
{ field: 'crash.backtrace', type: 'int[]?', notes: 'F-070. Up to 16 backtrace PCs (decimal uint32). Decode with xtensa-esp32-elf-addr2line against the matching firmware.elf.' },
|
||||
{ field: 'crash.backtrace_corrupted', type: 'bool?', notes: 'F-070. true when the stack unwinder hit a corrupted frame, so later frames are unreliable.' },
|
||||
{ field: 'crash.elf_sha256', type: 'string?', notes: 'F-070. First 16 hex chars of the crashing build\'s ELF SHA-256: identifies which firmware.elf decodes the addresses.' },
|
||||
{ field: 'pre_crash', type: 'object?', notes: 'F-070. Independent of crash. Only on fault resets (never after power-on/brownout): uptime_ms, free_heap, largest_block, min_free_heap at the time of the crash.' },
|
||||
{ field: 'pre_crash.abort_msg', type: 'string?', notes: 'F-070. Only for abort/assert/stack-overflow resets. When present, crash.pc / exc_cause 29 / exc_vaddr 0 describe the abort() mechanism, not the cause, and the first 2-4 backtrace frames are abort internals.' },
|
||||
].map(row => (
|
||||
<div key={row.field} style={{ padding: 'var(--space-3)', borderRadius: 'var(--radius-md)', backgroundColor: 'var(--color-bg-elevated)', border: '1px solid var(--color-border)' }}>
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: 'var(--space-2)', marginBottom: 'var(--space-1)' }}>
|
||||
|
||||
@@ -112,9 +112,11 @@ function FirmwareFormModal({ initial, onClose, onSaved }) {
|
||||
const [changelog, setChangelog] = useState(initial?.changelog ?? '')
|
||||
const [releaseNote, setReleaseNote] = useState(initial?.release_note ?? '')
|
||||
const [file, setFile] = useState(null)
|
||||
const [elfFile, setElfFile] = useState(null)
|
||||
const [uploading, setUploading] = useState(false)
|
||||
const [error, setError] = useState('')
|
||||
const fileInputRef = useRef(null)
|
||||
const elfInputRef = useRef(null)
|
||||
|
||||
const handleSubmit = async (e) => {
|
||||
e.preventDefault()
|
||||
@@ -130,12 +132,14 @@ function FirmwareFormModal({ initial, onClose, onSaved }) {
|
||||
fd.append('changelog', changelog); fd.append('release_note', releaseNote)
|
||||
if (hwType === 'bespoke') fd.append('bespoke_uid', bespokeUid.trim())
|
||||
fd.append('file', file)
|
||||
if (elfFile) fd.append('elf_file', elfFile)
|
||||
} else {
|
||||
fd.append('channel', channel); fd.append('version', version)
|
||||
fd.append('update_type', updateType); fd.append('min_fw_version', minFw)
|
||||
fd.append('changelog', changelog); fd.append('release_note', releaseNote)
|
||||
if (hwType === 'bespoke') fd.append('bespoke_uid', bespokeUid.trim())
|
||||
if (file) fd.append('file', file)
|
||||
if (elfFile) fd.append('elf_file', elfFile)
|
||||
}
|
||||
const token = localStorage.getItem('access_token')
|
||||
const url = isEdit ? `/api/firmware/${initial.id}` : '/api/firmware/upload'
|
||||
@@ -274,6 +278,30 @@ function FirmwareFormModal({ initial, onClose, onSaved }) {
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
{/* Optional symbols for crash decoding — matched to crash reports by
|
||||
the ELF's SHA-256 prefix (firmware F-070), not by version. */}
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 'var(--space-2)' }}>
|
||||
<input ref={elfInputRef} type="file" accept=".elf"
|
||||
onChange={(e) => {
|
||||
const f = e.target.files[0]
|
||||
if (f && !f.name.endsWith('.elf')) { setError('Only .elf files are accepted for symbols.'); return }
|
||||
setElfFile(f || null)
|
||||
}}
|
||||
style={{ display: 'none' }} />
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: 'var(--space-2)', flexWrap: 'wrap' }}>
|
||||
<Button variant="secondary" size="sm" onClick={() => elfInputRef.current?.click()}>
|
||||
{elfFile ? 'Change firmware.elf' : 'Attach firmware.elf'}
|
||||
</Button>
|
||||
{elfFile && (
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)', wordBreak: 'break-all' }}>
|
||||
{elfFile.name} · {formatBytes(elfFile.size)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--color-text-muted)' }}>
|
||||
Optional. Lets the console decode crash backtraces from devices running this build.
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
@@ -9,6 +9,7 @@ import DashboardPage from '@/pages/dashboard/DashboardPage'
|
||||
import DeviceList from '@/pages/bellcloud/devices/DeviceList'
|
||||
import DeviceDetail from '@/pages/bellcloud/devices/DeviceDetail'
|
||||
import DeviceMapPage from '@/pages/bellcloud/devices/DeviceMapPage'
|
||||
import CrashReport from '@/pages/bellcloud/devices/CrashReport'
|
||||
import OnboardDevice from '@/pages/bellcloud/devices/OnboardDevice'
|
||||
import StyleGuide from '@/pages/dev/StyleGuide'
|
||||
import CardFontSample from '@/pages/dev/CardFontSample'
|
||||
@@ -145,6 +146,7 @@ export default function V2Router() {
|
||||
<Route path="devices" element={<PermissionGate section="devices"><DeviceList /></PermissionGate>} />
|
||||
<Route path="devices/new" element={<PermissionGate section="devices" action="add"><ComingSoon /></PermissionGate>} />
|
||||
{/* Map preview — temporary unlisted route while map view is under construction */}
|
||||
<Route path="devices/crashes" element={<PermissionGate section="devices"><CrashReport /></PermissionGate>} />
|
||||
<Route path="devices/map-preview" element={<PermissionGate section="devices"><DeviceMapPage /></PermissionGate>} />
|
||||
<Route path="devices/onboard" element={<PermissionGate section="devices" action="edit"><OnboardDevice /></PermissionGate>} />
|
||||
<Route path="devices/:id" element={<PermissionGate section="devices"><DeviceDetail /></PermissionGate>} />
|
||||
|
||||
Reference in New Issue
Block a user