48 lines
3.9 KiB
Bash
48 lines
3.9 KiB
Bash
#!/bin/sh
|
|
# aurora-blackbox (B20): logs that survive a reboot or a sudden power loss (everything else lives in /run = RAM).
|
|
# Runs under supervise-daemon (/etc/init.d/aurora-blackbox). Writes to /var/log/aurora/ on p27 (ext4, commit 5 s):
|
|
# status.log one line per $BB_PERIOD s: wall clock, uptime, VBAT/charger/USB, PMIC temp, load, MemAvailable, modem state,
|
|
# netwatch status, WG handshake age + bytes, free-turn pid, Wi-Fi stations. (~200 B/line, ~300 KB/day)
|
|
# kmsg.log the whole kernel log of this boot (replayed from boot, then followed), minus known spam, prefixed with boot number.
|
|
# Each file rotates to .1 at $BB_MAX bytes (one generation kept). Boot markers "=== BOOT n" separate boots; the last lines before a
|
|
# marker show what the board was doing when it died.
|
|
BB_PERIOD=60 BB_MAX=2097152
|
|
[ -f /etc/aurora/blackbox.conf ] && . /etc/aurora/blackbox.conf
|
|
L=/var/log/aurora; mkdir -p $L /var/lib/aurora
|
|
ST=$L/status.log; KM=$L/kmsg.log
|
|
# boot number follows the kernel boot_id, so a service respawn within one boot neither counts a new boot nor re-logs the kernel log
|
|
BID=$(cat /proc/sys/kernel/random/boot_id); N=$(cat /var/lib/aurora/boot-count 2>/dev/null || echo 0); SKIP=-1
|
|
if [ "$(cat /var/lib/aurora/boot-id 2>/dev/null)" = "$BID" ]; then
|
|
SKIP=$(tail -n 50 $KM 2>/dev/null | sed -n "s/^b$N #\([0-9]*\) .*/\1/p" | tail -1); SKIP=${SKIP:--1}
|
|
else N=$((N + 1)); echo $N > /var/lib/aurora/boot-count; echo "$BID" > /var/lib/aurora/boot-id; fi
|
|
|
|
rot() { [ -f "$1" ] && [ $(stat -c %s "$1") -ge $BB_MAX ] && { cp "$1" "$1.1"; : > "$1"; }; }
|
|
kv() { sed -n "s/.*$1 \([^ ]*\).*/\1/p" /sys/kernel/debug/pm8916_lbc/state 2>/dev/null | head -1; }
|
|
temp() { for z in /sys/class/thermal/thermal_zone*; do [ "$(cat $z/type)" = pm8916-thermal ] && { echo $(( $(cat $z/temp) / 1000 )); return; }; done; echo -; }
|
|
wif() { [ "$(cat /run/aurora-vpnsel/mode 2>/dev/null)" = direct ] && echo wg1 || echo wg0; }
|
|
hs() { h=$(wg show $(wif) latest-handshakes 2>/dev/null | awk '{print $2; exit}'); [ -n "$h" ] && [ "$h" != 0 ] && echo $(( $(date +%s) - h )) || echo -; }
|
|
status() {
|
|
v=$(kv vbat_uv); [ -n "$v" ] && v=$(( v / 1000 ))
|
|
printf '%s up=%s vbat=%s chg=%s usb=%s t=%s load=%s memavail=%s modem=%s nw=%s vpn=%s wl=%s wg_hs=%s wg_rxtx=%s ft=%s sta=%s\n' \
|
|
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$(cut -d' ' -f1 /proc/uptime)" "${v:--}" "$(kv state)" \
|
|
"$(cat /sys/class/power_supply/pm8916-lbc-chgr/online 2>/dev/null || echo -)" "$(temp)" "$(cut -d' ' -f1 /proc/loadavg)" \
|
|
"$(awk '/MemAvailable/{print int($2/1024)"M"}' /proc/meminfo)" "$(cat /run/aurora-modem/state 2>/dev/null || echo -)" \
|
|
"$(cut -d' ' -f1,2 /run/aurora-netwatch/status 2>/dev/null | tr ' ' ':' || echo -)" "$(cat /run/aurora-vpnsel/mode 2>/dev/null || echo -)" "$(cut -d' ' -f1 /run/aurora-vpnsel/wl 2>/dev/null || echo -)" "$(hs)" \
|
|
"$(wg show $(wif) transfer 2>/dev/null | awk '{print $2"/"$3; exit}')" "$(pgrep -f /usr/local/bin/free-turn-client | head -1)" \
|
|
"$(timeout 3 iw dev wlan0 station dump 2>/dev/null | grep -c '^Station')"
|
|
}
|
|
|
|
if [ "$SKIP" = -1 ]; then echo "=== BOOT $N kernel $(uname -r) at uptime $(cut -d' ' -f1 /proc/uptime) clock $(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a $ST >> $KM
|
|
else echo "=== blackbox restart in boot $N at uptime $(cut -d' ' -f1 /proc/uptime) (kmsg resumes after #$SKIP)" | tee -a $ST >> $KM; fi
|
|
# kernel log: replay this boot from /dev/kmsg, then follow; "pri,seq,usec,-;msg" -> "b<N> #seq [ sec.usec] msg"
|
|
( exec cat /dev/kmsg ) | awk -v b=$N -v skip=$SKIP '
|
|
{ i = index($0, ";"); split(substr($0, 1, i - 1), h, ","); if (h[2] + 0 <= skip) next }
|
|
/CMA: alloc failed|alloc_contig_range: .* busy|Spurious TX complete/ { next }
|
|
{ printf "b%d #%d [%12.6f] %s\n", b, h[2], h[3] / 1000000, substr($0, i + 1); fflush() }' >> $KM &
|
|
KP=$!
|
|
trap 'kill $KP 2>/dev/null; echo "=== STOP boot $N at uptime $(cut -d" " -f1 /proc/uptime)" | tee -a $ST >> $KM; sync; exit 0' TERM INT
|
|
while :; do
|
|
status >> $ST
|
|
rot $ST; rot $KM
|
|
sleep $BB_PERIOD & wait $!
|
|
done
|