#!/bin/sh
# B6J harness. j1: cycles until the first registration failure, snapshot (j-snap.sh) BEFORE stop, then clean stop, end.
#              j3: same cycle, ONE long-lived rmtfs (MODEM_RMTFS_PERSIST=1); stops at the first failure of any kind;
#                  after the series: MSS offline -> then rmtfs-stop. RAM only.
#              k:  j1 lifecycle + k-at.sh (runtime AT+CEREG=5/CGREG=3, port open from before DMS online until MM start / FAIL queries).
MODE=$1; N=${2:-6}; O=/tmp/$MODE; mkdir -p $O; M="sh /tmp/aurora-modem"; D=/tmp/aurora-modem.d
export MODEM_CONF=/tmp/$MODE.conf MODEM_KEEP_ON_FAIL=1
B=/sys/bus/platform/devices/4080000.remoteproc:bam-dmux; RPS=/sys/class/remoteproc/remoteproc0/state
mask() { sed -E 's/([0-9]{6,})([0-9]{4})/******\2/g'; }
up() { cut -d' ' -f1 /proc/uptime; }
mk() { echo "<5>[$MODE] $*" > /dev/kmsg; echo "[$(up)] $*" | tee -a $O/run.log; }
nv() { for want in modemst1 modemst2 fsg fsc persist; do for p in /sys/class/block/mmcblk0p*; do
	[ "$(sed -n 's/^PARTNAME=//p' $p/uevent)" = "$want" ] && echo "$want $(basename $p) $(sha256sum /dev/$(basename $p) | cut -d' ' -f1)"; done; done; }
cnt() { grep -cE "$1" $2; }
f() { grep -m1 -oE "$1: *'[^']*'" $2 2>/dev/null | sed -E "s/.*'([^']*)'/\1/"; }
[ -f $O/nv-before.txt ] || { nv > $O/nv-before.txt; cat /sys/block/mmcblk0/stat > $O/stat-before.txt; }
[ -f $O/table.tsv ] || printf 'cycle\treg\treg_latency_s\treg_state\tps\trat_band_ch\ttac_pci_eci\tbearer\tipv4\tdata\tbam\tmss\trmtfs_pid\trmtfs_writes\tfr_110_22_eagain_ssr\tusb\temmc_w\tresult\n' > $O/table.tsv
for c in $(seq 1 $N); do
	X=$O/c$c; mkdir -p $X; dmesg | wc -l > $X/dmesg-off
	wc -l < $D/lifecycle.log > $X/log-off 2>/dev/null || echo 0 > $X/log-off
	wc -l < $D/rmtfs.log > $X/rmtfs-off 2>/dev/null || echo 0 > $X/rmtfs-off
	rm -f $D/reg-*.txt $D/qmi-ss.txt $D/bearer.txt
	mk "CYCLE $c begin: emmc $(awk '{print "w="$5" s="$7}' /sys/block/mmcblk0/stat) mss=$(cat $RPS) rmtfs='$(pidof rmtfs)'"
	[ $MODE = k ] && { sh /tmp/k-at.sh run $X/at > /dev/null 2>&1 & }
	$M start > $X/start.out 2>&1; SR=$?; mk "CYCLE $c start rc=$SR"
	cp $D/qmi-ss.txt $D/reg-*.txt $X/ 2>/dev/null; RPID=$(pidof rmtfs); cp $D/reg-trans.tsv $D/reg-denied-count $X/ 2>/dev/null
	$M status 2>&1 | mask > $X/status-up.txt
	if [ $SR = 0 ]; then
		cp $D/bearer.txt $X/ 2>/dev/null; cp $D/ipcfg $X/ipcfg 2>/dev/null
		sh /tmp/aurora-modem-datatest $X/dt > $X/dt.out 2>&1; DR=$?
	else
		DR=9; mk "CYCLE $c start FAILED in $(cat $D/state) - snapshot BEFORE stop"
		[ $MODE = k ] && { sh /tmp/k-at.sh failq > $X/k-failq.out 2>&1; export AT_PORT=/dev/none-at-held-by-k-at; }
		[ $MODE = l ] && export AT_PORT=/dev/none-l-mode
		sh /tmp/j-snap.sh $X/snap > $X/snap.out 2>&1
		if [ $MODE = l ]; then mk "CYCLE $c B6L passive observation 900 s (no cleanup, MSS/rmtfs/radio untouched)"
			sh /tmp/l-obs.sh $X/obs 900 15 > $X/obs.out 2>&1; mk "CYCLE $c observation: $(cat $X/obs/result.txt)"
			sh /tmp/j-snap.sh $X/snap-end > $X/snap-end.out 2>&1; fi; mk "CYCLE $c snapshot done"
	fi
	mk "CYCLE $c $(cat $X/dt/verdict.txt 2>/dev/null || echo DATATEST not-run) rc=$DR"
	[ $MODE = k ] && sh /tmp/k-at.sh close > $X/k-close.out 2>&1
	$M stop > $X/stop.out 2>&1; PR=$?; mk "CYCLE $c stop rc=$PR"; sleep 2
	$M status 2>&1 | mask > $X/status-down.txt
	cat /sys/block/mmcblk0/stat > $X/stat.txt
	dmesg | tail -n +$(( $(cat $X/dmesg-off) + 1 )) > $X/dmesg-delta.txt
	tail -n +$(( $(cat $X/log-off) + 1 )) $D/lifecycle.log | mask > $X/lifecycle.log
	tail -n +$(( $(cat $X/rmtfs-off) + 1 )) $D/rmtfs.log > $X/rmtfs-delta.log
	E1=$(cnt 'Failed to resume' $X/dmesg-delta.txt); E2=$(cnt '[-(]110\b|-110' $X/dmesg-delta.txt); E3=$(cnt '[ (=]-22\b' $X/dmesg-delta.txt)
	E4=$(cnt 'EAGAIN|Resource temporarily' $X/dmesg-delta.txt); E5=$(cnt 'crash detected|fatal error|[Ww]atchdog|wdog|SSR|Oops|Kernel panic|Unable to handle|BUG:' $X/dmesg-delta.txt)
	grep -iE 'remoteproc.*(fail|error)|Failed to resume|-110|EAGAIN|crash|fatal|atchdog|Oops|BUG:' $X/dmesg-delta.txt > $X/dmesg-errs.txt
	F=""
	[ $PR = 0 ] || F="$F stop"; [ "$(cat $RPS)" = offline ] || F="$F mss-$(cat $RPS)"
	if [ $MODE = j3 ]; then [ -n "$(pidof rmtfs)" ] || F="$F rmtfs-died"; [ -z "$R0" ] && R0=$RPID; [ "$(pidof rmtfs)" = "$R0" ] || F="$F rmtfs-pid-changed"
	else [ -z "$(pidof rmtfs)" ] || F="$F rmtfs-alive"; fi
	[ -z "$(pidof ModemManager)" ] || F="$F mm"; [ -z "$(pidof qmi-proxy)" ] || F="$F qmi-proxy"
	for d in /sys/class/net/wwan*; do [ -e $d ] || continue; w=${d##*/}
		[ $(( $(cat $d/flags) & 1 )) -ne 0 ] && F="$F $w-up"; ip -4 addr show $w | grep -q "inet " && F="$F $w-addr"; done
	ip route | grep -q " dev wwan" && F="$F routes"; ip route | grep -q "^default" && F="$F default-route"
	BS=$(cat $B/power/runtime_status); [ "$BS" = suspended ] || F="$F bam-$BS"
	[ $((E1+E2+E4+E5)) -gt 0 ] && F="$F dmesg-errs"
	awk '{exit !($5==0 && $7==0)}' $X/stat.txt || F="$F emmc-writes"
	ping -c 2 -W 2 172.16.42.2 > $X/usb-ping.txt 2>&1 && U=ok || { U=FAIL; F="$F usb-ncm"; }
	# table row
	if grep -q "registered: " $X/lifecycle.log; then REG=PASS; LAT=$(grep -m1 -oE "registered: .* after [0-9.]+s" $X/lifecycle.log | grep -oE "[0-9.]+s$")
	elif grep -q "FAIL in RADIO_ONLINE" $X/lifecycle.log; then REG=FAIL; LAT=">180s"; else REG=n/a; LAT=-; fi
	SS=$X/qmi-ss.txt; [ -s $SS ] || SS=$X/snap/qmi-ss.txt
	RB=$(f "Active Band Class" $X/reg-nas-get-rf-band-info.txt)/$(f "Active Channel" $X/reg-nas-get-rf-band-info.txt)
	[ "$REG" = FAIL ] && RB=$(f "Active Band Class" $X/snap/qmi.txt)/$(f "Active Channel" $X/snap/qmi.txt)
	CL=$X/reg-nas-get-cell-location-info.txt; [ "$REG" = FAIL ] && CL=$X/snap/qmi.txt
	TPE=$(f "Tracking Area Code" $CL)/$(f "Serving Cell ID" $CL)/$(f "Global Cell ID" $CL)
	BR=$( [ $SR = 0 ] && echo connected || echo none ); IP=$(sed -n 's/^A=//p' $X/ipcfg 2>/dev/null)
	DT=$(cut -d' ' -f2 $X/dt/verdict.txt 2>/dev/null || echo not-run)
	RW=$(grep -c "write " $X/rmtfs-delta.log)
	printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' $c $REG "$LAT" "$(f 'Registration state' $SS)" "$(f 'PS' $SS)" \
		"lte:$RB" "$TPE" $BR "${IP:--}" $DT $BS $(cat $RPS) "${RPID:--}" $RW "$E1/$E2/$E3/$E4/$E5" $U "$(awk '{print $5"/"$7}' $X/stat.txt)" "${F:-ok}" >> $O/table.tsv
	[ $MODE = k ] && { grep -E "\+C(E|G)?REG:|ERROR" $X/at/at.clean.txt > $X/urc.txt; mk "CYCLE $c AT: $(grep -cE '< \+CEREG:' $X/at/at.clean.txt) CEREG lines, last '$(grep -E '< \+CEREG:' $X/at/at.clean.txt | tail -1 | sed 's/.*< //')', ERROR x$(grep -c '< ERROR' $X/at/at.clean.txt)"; }
	if [ $MODE = m ]; then
		[ -f $O/m-table.tsv ] || printf 'cycle\treg\tlatency_s\ttransitions\tdenied\tsearching_rows\tinit_pci\tfinal_pci\treselection\tps\tbearer\tdns\tping\thttp\thttps\tstop\tchecks\n' > $O/m-table.tsv
		TR=$X/reg-trans.tsv; NT=$(($(wc -l < $TR 2>/dev/null || echo 1)-1)); IP0=$(awk -F'\t' 'NR==2{print $10}' $TR); IP1=$(awk -F'\t' 'END{print $10}' $TR)
		PCIS=$(awk -F'\t' 'NR>1 && $10!=""{print $10}' $TR | uniq | tr '\n' '>' | sed 's/>$//')
		NS=$(awk -F'\t' 'NR>1 && $3=="not-registered-searching"' $TR | wc -l)
		RL=$(grep -m1 -oE "registered: .* after [0-9.]+s" $X/lifecycle.log | grep -oE "[0-9.]+s$"); [ -n "$RL" ] || RL=$(grep -m1 -oE "REGISTRATION_TIMEOUT|LOCAL FAILURE[^(]*" $X/lifecycle.log)
		V=$(cat $X/dt/verdict.txt 2>/dev/null)
		printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' $c $REG "${RL:--}" $NT "$(cat $X/reg-denied-count 2>/dev/null)" $NS "${IP0:--}" "${IP1:--}" \
			"$( [ "$(echo $PCIS | tr '>' '\n' | sort -u | wc -l)" -gt 1 ] && echo "yes:$PCIS" || echo "no:$PCIS")" "$(awk -F'\t' 'END{print $4}' $TR)" $BR \
			"$(echo "$V" | grep -oE 'dns=[0-9]')" "$(echo "$V" | grep -oE 'ping=[0-9+]*')" "$(echo "$V" | grep -oE 'http=code=[0-9]*')" "$(echo "$V" | grep -oE 'https=\[[^]]*\]\([^)]*\)')" "rc=$PR" "${F:-ok}" >> $O/m-table.tsv
	fi
	mk "CYCLE $c RESULT reg=$REG lat=$LAT data=$DT checks=${F:-ok} rmtfs=$RPID->$(pidof rmtfs) writes=$RW bam=$BS"
	[ $SR = 0 ] && [ $DR = 0 ] || { mk "SERIES STOP: cycle $c start/data failure"; break; }
	[ -z "$F" ] || { mk "SERIES STOP: cycle $c new failure:$F"; break; }
	sleep 5
done
if [ $MODE = j3 ]; then
	mk "series end: MSS $(cat $RPS) -> rmtfs-stop"; [ "$(cat $RPS)" = offline ] && $M rmtfs-stop > $O/rmtfs-stop.out 2>&1; mk "rmtfs-stop rc=$? rmtfs='$(pidof rmtfs)'"
fi
nv > $O/nv-after.txt; cat /sys/block/mmcblk0/stat > $O/stat-after.txt
cmp -s $O/nv-before.txt $O/nv-after.txt && NVR=unchanged || NVR=CHANGED
mk "FINAL: nv $NVR, emmc $(awk '{print "w="$5" s="$7}' $O/stat-after.txt), mss=$(cat $RPS), rmtfs='$(pidof rmtfs)', mm='$(pidof ModemManager)', bam=$(cat $B/power/runtime_status), wwan-up=$(for d in /sys/class/net/wwan*; do [ $(( $(cat $d/flags) & 1 )) -ne 0 ] && echo ${d##*/}; done | tr '\n' ' '), usb $(ping -c 2 -W 2 172.16.42.2 >/dev/null 2>&1 && echo ok || echo FAIL)"
for f in $D/mm-*.log; do [ -f $f ] && mask < $f > $O/$(basename $f); done
cp $D/rmtfs.log $O/; mask < $D/lifecycle.log > $O/lifecycle.log; dmesg > $O/dmesg-full.txt
tar -C /tmp -cf /tmp/$MODE.tar $MODE; mk "$MODE-RUN-END"