101 lines
8.8 KiB
Bash
101 lines
8.8 KiB
Bash
#!/bin/sh
|
|
# B6J harness. j1: cycles until the first registration failure, snapshot (j-snap.sh) BEFORE stop, then clean stop, end.
|
|
# j3: same cycle, ONE long-lived rmtfs (MODEM_RMTFS_PERSIST=1); stops at the first failure of any kind;
|
|
# after the series: MSS offline -> then rmtfs-stop. RAM only.
|
|
# k: j1 lifecycle + k-at.sh (runtime AT+CEREG=5/CGREG=3, port open from before DMS online until MM start / FAIL queries).
|
|
MODE=$1; N=${2:-6}; O=/tmp/$MODE; mkdir -p $O; M="sh /tmp/aurora-modem"; D=/tmp/aurora-modem.d
|
|
export MODEM_CONF=/tmp/$MODE.conf MODEM_KEEP_ON_FAIL=1
|
|
B=/sys/bus/platform/devices/4080000.remoteproc:bam-dmux; RPS=/sys/class/remoteproc/remoteproc0/state
|
|
mask() { sed -E 's/([0-9]{6,})([0-9]{4})/******\2/g'; }
|
|
up() { cut -d' ' -f1 /proc/uptime; }
|
|
mk() { echo "<5>[$MODE] $*" > /dev/kmsg; echo "[$(up)] $*" | tee -a $O/run.log; }
|
|
nv() { for want in modemst1 modemst2 fsg fsc persist; do for p in /sys/class/block/mmcblk0p*; do
|
|
[ "$(sed -n 's/^PARTNAME=//p' $p/uevent)" = "$want" ] && echo "$want $(basename $p) $(sha256sum /dev/$(basename $p) | cut -d' ' -f1)"; done; done; }
|
|
cnt() { grep -cE "$1" $2; }
|
|
f() { grep -m1 -oE "$1: *'[^']*'" $2 2>/dev/null | sed -E "s/.*'([^']*)'/\1/"; }
|
|
[ -f $O/nv-before.txt ] || { nv > $O/nv-before.txt; cat /sys/block/mmcblk0/stat > $O/stat-before.txt; }
|
|
[ -f $O/table.tsv ] || printf 'cycle\treg\treg_latency_s\treg_state\tps\trat_band_ch\ttac_pci_eci\tbearer\tipv4\tdata\tbam\tmss\trmtfs_pid\trmtfs_writes\tfr_110_22_eagain_ssr\tusb\temmc_w\tresult\n' > $O/table.tsv
|
|
for c in $(seq 1 $N); do
|
|
X=$O/c$c; mkdir -p $X; dmesg | wc -l > $X/dmesg-off
|
|
wc -l < $D/lifecycle.log > $X/log-off 2>/dev/null || echo 0 > $X/log-off
|
|
wc -l < $D/rmtfs.log > $X/rmtfs-off 2>/dev/null || echo 0 > $X/rmtfs-off
|
|
rm -f $D/reg-*.txt $D/qmi-ss.txt $D/bearer.txt
|
|
mk "CYCLE $c begin: emmc $(awk '{print "w="$5" s="$7}' /sys/block/mmcblk0/stat) mss=$(cat $RPS) rmtfs='$(pidof rmtfs)'"
|
|
[ $MODE = k ] && { sh /tmp/k-at.sh run $X/at > /dev/null 2>&1 & }
|
|
$M start > $X/start.out 2>&1; SR=$?; mk "CYCLE $c start rc=$SR"
|
|
cp $D/qmi-ss.txt $D/reg-*.txt $X/ 2>/dev/null; RPID=$(pidof rmtfs); cp $D/reg-trans.tsv $D/reg-denied-count $X/ 2>/dev/null
|
|
$M status 2>&1 | mask > $X/status-up.txt
|
|
if [ $SR = 0 ]; then
|
|
cp $D/bearer.txt $X/ 2>/dev/null; cp $D/ipcfg $X/ipcfg 2>/dev/null
|
|
sh /tmp/aurora-modem-datatest $X/dt > $X/dt.out 2>&1; DR=$?
|
|
else
|
|
DR=9; mk "CYCLE $c start FAILED in $(cat $D/state) - snapshot BEFORE stop"
|
|
[ $MODE = k ] && { sh /tmp/k-at.sh failq > $X/k-failq.out 2>&1; export AT_PORT=/dev/none-at-held-by-k-at; }
|
|
[ $MODE = l ] && export AT_PORT=/dev/none-l-mode
|
|
sh /tmp/j-snap.sh $X/snap > $X/snap.out 2>&1
|
|
if [ $MODE = l ]; then mk "CYCLE $c B6L passive observation 900 s (no cleanup, MSS/rmtfs/radio untouched)"
|
|
sh /tmp/l-obs.sh $X/obs 900 15 > $X/obs.out 2>&1; mk "CYCLE $c observation: $(cat $X/obs/result.txt)"
|
|
sh /tmp/j-snap.sh $X/snap-end > $X/snap-end.out 2>&1; fi; mk "CYCLE $c snapshot done"
|
|
fi
|
|
mk "CYCLE $c $(cat $X/dt/verdict.txt 2>/dev/null || echo DATATEST not-run) rc=$DR"
|
|
[ $MODE = k ] && sh /tmp/k-at.sh close > $X/k-close.out 2>&1
|
|
$M stop > $X/stop.out 2>&1; PR=$?; mk "CYCLE $c stop rc=$PR"; sleep 2
|
|
$M status 2>&1 | mask > $X/status-down.txt
|
|
cat /sys/block/mmcblk0/stat > $X/stat.txt
|
|
dmesg | tail -n +$(( $(cat $X/dmesg-off) + 1 )) > $X/dmesg-delta.txt
|
|
tail -n +$(( $(cat $X/log-off) + 1 )) $D/lifecycle.log | mask > $X/lifecycle.log
|
|
tail -n +$(( $(cat $X/rmtfs-off) + 1 )) $D/rmtfs.log > $X/rmtfs-delta.log
|
|
E1=$(cnt 'Failed to resume' $X/dmesg-delta.txt); E2=$(cnt '[-(]110\b|-110' $X/dmesg-delta.txt); E3=$(cnt '[ (=]-22\b' $X/dmesg-delta.txt)
|
|
E4=$(cnt 'EAGAIN|Resource temporarily' $X/dmesg-delta.txt); E5=$(cnt 'crash detected|fatal error|[Ww]atchdog|wdog|SSR|Oops|Kernel panic|Unable to handle|BUG:' $X/dmesg-delta.txt)
|
|
grep -iE 'remoteproc.*(fail|error)|Failed to resume|-110|EAGAIN|crash|fatal|atchdog|Oops|BUG:' $X/dmesg-delta.txt > $X/dmesg-errs.txt
|
|
F=""
|
|
[ $PR = 0 ] || F="$F stop"; [ "$(cat $RPS)" = offline ] || F="$F mss-$(cat $RPS)"
|
|
if [ $MODE = j3 ]; then [ -n "$(pidof rmtfs)" ] || F="$F rmtfs-died"; [ -z "$R0" ] && R0=$RPID; [ "$(pidof rmtfs)" = "$R0" ] || F="$F rmtfs-pid-changed"
|
|
else [ -z "$(pidof rmtfs)" ] || F="$F rmtfs-alive"; fi
|
|
[ -z "$(pidof ModemManager)" ] || F="$F mm"; [ -z "$(pidof qmi-proxy)" ] || F="$F qmi-proxy"
|
|
for d in /sys/class/net/wwan*; do [ -e $d ] || continue; w=${d##*/}
|
|
[ $(( $(cat $d/flags) & 1 )) -ne 0 ] && F="$F $w-up"; ip -4 addr show $w | grep -q "inet " && F="$F $w-addr"; done
|
|
ip route | grep -q " dev wwan" && F="$F routes"; ip route | grep -q "^default" && F="$F default-route"
|
|
BS=$(cat $B/power/runtime_status); [ "$BS" = suspended ] || F="$F bam-$BS"
|
|
[ $((E1+E2+E4+E5)) -gt 0 ] && F="$F dmesg-errs"
|
|
awk '{exit !($5==0 && $7==0)}' $X/stat.txt || F="$F emmc-writes"
|
|
ping -c 2 -W 2 172.16.42.2 > $X/usb-ping.txt 2>&1 && U=ok || { U=FAIL; F="$F usb-ncm"; }
|
|
# table row
|
|
if grep -q "registered: " $X/lifecycle.log; then REG=PASS; LAT=$(grep -m1 -oE "registered: .* after [0-9.]+s" $X/lifecycle.log | grep -oE "[0-9.]+s$")
|
|
elif grep -q "FAIL in RADIO_ONLINE" $X/lifecycle.log; then REG=FAIL; LAT=">180s"; else REG=n/a; LAT=-; fi
|
|
SS=$X/qmi-ss.txt; [ -s $SS ] || SS=$X/snap/qmi-ss.txt
|
|
RB=$(f "Active Band Class" $X/reg-nas-get-rf-band-info.txt)/$(f "Active Channel" $X/reg-nas-get-rf-band-info.txt)
|
|
[ "$REG" = FAIL ] && RB=$(f "Active Band Class" $X/snap/qmi.txt)/$(f "Active Channel" $X/snap/qmi.txt)
|
|
CL=$X/reg-nas-get-cell-location-info.txt; [ "$REG" = FAIL ] && CL=$X/snap/qmi.txt
|
|
TPE=$(f "Tracking Area Code" $CL)/$(f "Serving Cell ID" $CL)/$(f "Global Cell ID" $CL)
|
|
BR=$( [ $SR = 0 ] && echo connected || echo none ); IP=$(sed -n 's/^A=//p' $X/ipcfg 2>/dev/null)
|
|
DT=$(cut -d' ' -f2 $X/dt/verdict.txt 2>/dev/null || echo not-run)
|
|
RW=$(grep -c "write " $X/rmtfs-delta.log)
|
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' $c $REG "$LAT" "$(f 'Registration state' $SS)" "$(f 'PS' $SS)" \
|
|
"lte:$RB" "$TPE" $BR "${IP:--}" $DT $BS $(cat $RPS) "${RPID:--}" $RW "$E1/$E2/$E3/$E4/$E5" $U "$(awk '{print $5"/"$7}' $X/stat.txt)" "${F:-ok}" >> $O/table.tsv
|
|
[ $MODE = k ] && { grep -E "\+C(E|G)?REG:|ERROR" $X/at/at.clean.txt > $X/urc.txt; mk "CYCLE $c AT: $(grep -cE '< \+CEREG:' $X/at/at.clean.txt) CEREG lines, last '$(grep -E '< \+CEREG:' $X/at/at.clean.txt | tail -1 | sed 's/.*< //')', ERROR x$(grep -c '< ERROR' $X/at/at.clean.txt)"; }
|
|
if [ $MODE = m ]; then
|
|
[ -f $O/m-table.tsv ] || printf 'cycle\treg\tlatency_s\ttransitions\tdenied\tsearching_rows\tinit_pci\tfinal_pci\treselection\tps\tbearer\tdns\tping\thttp\thttps\tstop\tchecks\n' > $O/m-table.tsv
|
|
TR=$X/reg-trans.tsv; NT=$(($(wc -l < $TR 2>/dev/null || echo 1)-1)); IP0=$(awk -F'\t' 'NR==2{print $10}' $TR); IP1=$(awk -F'\t' 'END{print $10}' $TR)
|
|
PCIS=$(awk -F'\t' 'NR>1 && $10!=""{print $10}' $TR | uniq | tr '\n' '>' | sed 's/>$//')
|
|
NS=$(awk -F'\t' 'NR>1 && $3=="not-registered-searching"' $TR | wc -l)
|
|
RL=$(grep -m1 -oE "registered: .* after [0-9.]+s" $X/lifecycle.log | grep -oE "[0-9.]+s$"); [ -n "$RL" ] || RL=$(grep -m1 -oE "REGISTRATION_TIMEOUT|LOCAL FAILURE[^(]*" $X/lifecycle.log)
|
|
V=$(cat $X/dt/verdict.txt 2>/dev/null)
|
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' $c $REG "${RL:--}" $NT "$(cat $X/reg-denied-count 2>/dev/null)" $NS "${IP0:--}" "${IP1:--}" \
|
|
"$( [ "$(echo $PCIS | tr '>' '\n' | sort -u | wc -l)" -gt 1 ] && echo "yes:$PCIS" || echo "no:$PCIS")" "$(awk -F'\t' 'END{print $4}' $TR)" $BR \
|
|
"$(echo "$V" | grep -oE 'dns=[0-9]')" "$(echo "$V" | grep -oE 'ping=[0-9+]*')" "$(echo "$V" | grep -oE 'http=code=[0-9]*')" "$(echo "$V" | grep -oE 'https=\[[^]]*\]\([^)]*\)')" "rc=$PR" "${F:-ok}" >> $O/m-table.tsv
|
|
fi
|
|
mk "CYCLE $c RESULT reg=$REG lat=$LAT data=$DT checks=${F:-ok} rmtfs=$RPID->$(pidof rmtfs) writes=$RW bam=$BS"
|
|
[ $SR = 0 ] && [ $DR = 0 ] || { mk "SERIES STOP: cycle $c start/data failure"; break; }
|
|
[ -z "$F" ] || { mk "SERIES STOP: cycle $c new failure:$F"; break; }
|
|
sleep 5
|
|
done
|
|
if [ $MODE = j3 ]; then
|
|
mk "series end: MSS $(cat $RPS) -> rmtfs-stop"; [ "$(cat $RPS)" = offline ] && $M rmtfs-stop > $O/rmtfs-stop.out 2>&1; mk "rmtfs-stop rc=$? rmtfs='$(pidof rmtfs)'"
|
|
fi
|
|
nv > $O/nv-after.txt; cat /sys/block/mmcblk0/stat > $O/stat-after.txt
|
|
cmp -s $O/nv-before.txt $O/nv-after.txt && NVR=unchanged || NVR=CHANGED
|
|
mk "FINAL: nv $NVR, emmc $(awk '{print "w="$5" s="$7}' $O/stat-after.txt), mss=$(cat $RPS), rmtfs='$(pidof rmtfs)', mm='$(pidof ModemManager)', bam=$(cat $B/power/runtime_status), wwan-up=$(for d in /sys/class/net/wwan*; do [ $(( $(cat $d/flags) & 1 )) -ne 0 ] && echo ${d##*/}; done | tr '\n' ' '), usb $(ping -c 2 -W 2 172.16.42.2 >/dev/null 2>&1 && echo ok || echo FAIL)"
|
|
for f in $D/mm-*.log; do [ -f $f ] && mask < $f > $O/$(basename $f); done
|
|
cp $D/rmtfs.log $O/; mask < $D/lifecycle.log > $O/lifecycle.log; dmesg > $O/dmesg-full.txt
|
|
tar -C /tmp -cf /tmp/$MODE.tar $MODE; mk "$MODE-RUN-END"
|