#!/bin/sh # som-test-nrf91 -- test an Emcraft SOM-NRF9151 over its Zephyr shell console: # is the bench sound, did the right firmware boot, is the modem healthy, and is # the board on the network with a data path. Runs in a brintOS machine (BusyBox) # and natively on a Mac, the same script. # # The suites carry the names Kent's bot uses for its native expect suites # ("Zephyr app", "AT command set", "LTE link"), and follow its rules: # * read-only by default: every line sent to the board passes an allowlist, # and AT set commands are refused unless --allow-writes names them; # * a check runs only if the board has the command (`help`) or the device # (`device list`) it needs; # * a bench condition (no SIM, no coverage, no battery) is a SKIP carrying # its reason; a FAIL means something that should have worked did not; # * IMEI, IMSI and ICCID are masked unless --identity is given; # * never sent, whatever the flags: AT%XFACTORYRESET, AT&F, AT%CMNG=3, # AT+CFUN=0; the watchdog is only ever read. # # The qa suite, run only when named, is the unattended console tests of the # NRF-SOM QA Test Plan v23: 3.1 serial console, 3.2 version, 3.4 thread stacks # and 3.6 SPI flash. It resets the board, so it needs --allow-reset, and 3.6 # writes the board's LittleFS only with --allow-fs-writes. Its resets are soft # (the shell's `kernel reboot cold`, or openocd's `reset run`), not the plan's # power cycle. # # Every check prints one line, `. PASS|FAIL|SKIP|XFAIL -- detail`, # and the run ends with one PASS or FAIL line. Exit status: 0 PASS, 1 FAIL, # 2 usage, 3 the rig failed (no console, no prompt), so nothing was tested. # # Why the console is driven this way (all measured in brintOS, 2026-10-05): # * this image's expect cannot spawn (brintos/hardwarejs#50), so the shell # drives the port: one open on fd 3, a background dd records everything the # board says, and printf writes to fd 3; # * waits are deadlines on `date`: a sleep right after the shell reaps a # child can return at once (brintos/linux-in-the-browser#120); # * nothing here uses cat or cp (sendfile never completes, glibc#14), nor # BusyBox head -N. set -u PROG=som-test-nrf91 CONSOLE_ID=0403:6015 # the BSB's FT230X SCRIPTS="${SOM_OPENOCD_SCRIPTS:-/usr/share/openocd/scripts}" BOARD_CFG="${SOM_BOARD_CFG:-$SCRIPTS/board/emcraft-som-nrf9151.cfg}" FW_DIR="${SOM_FW_DIR:-/usr/share/brintos-som/nrf9151}" # the releases this machine carries suites="" quick=0 profile="" console="" known_add="" lte_wait=180 cloud_wait=120 writes=0 identity=0 out="" send_line="" reset_ok=0 reset_mode="" fs_writes=0 qa_version="" qa_cloud_wait=300 speed="" adapter="" usage() { printf '%s\n' \ "usage: $PROG [--suite zephyr,at,lte,qa] [--quick] [--profile NAME] [--console DEV]" \ " [--known CHECK,...] [--lte-wait SEC] [--allow-writes]" \ " [--cloud-wait SEC]" \ " [--identity] [--out DIR]" \ " $PROG --suite qa --allow-reset [--reset shell|swd] [--allow-fs-writes]" \ " [--qa-version VER] [--qa-cloud-wait SEC] [--speed KHZ]" \ " [--adapter NAME] [--console DEV] [--out DIR]" \ " $PROG --send LINE [--console DEV]" \ "" \ " --suite LIST suites to run (default: zephyr,at,lte; qa runs only when named)" \ " --quick only the checks a flash needs: identity, uptime, devices," \ " watchdog, AT, modem firmware (under a minute)" \ " --profile NAME asset_tracker | hello_mcuboot | generic (default: from \`version\`)" \ " --console DEV the SOM's console (default: som-console --find --id $CONSOLE_ID," \ " or the only /dev/cu.usbserial-* on a Mac), or a directory" \ " holding the FIFOs in and out of a stand-in board" \ " --known LIST checks known to fail on this unit: a FAIL there is an XFAIL" \ " --lte-wait SEC how long lte.registered waits for registration (default 180)" \ " --allow-writes also run the checks that change modem state and restore it:" \ " a flight-mode round trip (AT+CFUN=4, then back to 1), the" \ " re-attach, and the app reconnecting to nRF Cloud, with the" \ " modem's data counters started (AT%XCONNSTAT=1; left on)" \ " --cloud-wait SEC how long lte.cloud waits for the reconnect (default 120)" \ " --allow-reset let the qa suite reset the board (required by --suite qa)" \ " --reset MODE shell (kernel reboot cold) or swd (openocd reset run)" \ " (default: shell if \`kernel\` lists reboot, else swd)" \ " --speed KHZ SWD clock for --reset swd (default: the board config's 1000)" \ " --adapter NAME another openocd interface config for --reset swd (default: jlink)" \ " --allow-fs-writes also run 3.6, which writes atest and test1 to the board's" \ " /lfs1 and removes them at the end" \ " --qa-version VER the version 3.1 and 3.2 expect, e.g. 1.2-114-release (default:" \ " the PROVENANCE of the release the board runs, if this machine" \ " carries it, else what \`version\` reports)" \ " --qa-cloud-wait SEC how long 3.4 waits for CLOUD_EVT_CONNECTED (default 300)" \ " --identity print IMEI, IMSI and ICCID in full" \ " --out DIR where results.tsv and transcript.txt go" \ " (default /tmp/som-test-)" \ " --send LINE send one allowlisted line, print the reply, and exit" } while [ $# -gt 0 ]; do case "$1" in --suite) suites=${2:-}; shift 2 || exit 2 ;; --quick) quick=1; shift ;; --profile) profile=${2:-}; shift 2 || exit 2 ;; --console) console=${2:-}; shift 2 || exit 2 ;; --known) known_add=${2:-}; shift 2 || exit 2 ;; --lte-wait) lte_wait=${2:-}; shift 2 || exit 2 ;; --cloud-wait) cloud_wait=${2:-}; shift 2 || exit 2 ;; --allow-writes) writes=1; shift ;; --allow-reset) reset_ok=1; shift ;; --reset) reset_mode=${2:-}; shift 2 || exit 2 ;; --speed) speed=${2:-}; shift 2 || exit 2 ;; --adapter) adapter=${2:-}; shift 2 || exit 2 ;; --allow-fs-writes) fs_writes=1; shift ;; --qa-version) qa_version=${2:-}; shift 2 || exit 2 ;; --qa-cloud-wait) qa_cloud_wait=${2:-}; shift 2 || exit 2 ;; --identity) identity=1; shift ;; --out) out=${2:-}; shift 2 || exit 2 ;; --send) send_line=${2:-}; shift 2 || exit 2 ;; -h|--help) usage; exit 0 ;; *) echo "$PROG: unknown argument: $1" >&2; usage >&2; exit 2 ;; esac done case "$lte_wait$cloud_wait" in ''|*[!0-9]*) echo "$PROG: --lte-wait and --cloud-wait take seconds" >&2; exit 2 ;; esac case "$qa_cloud_wait" in ''|*[!0-9]*) echo "$PROG: --qa-cloud-wait takes seconds" >&2; exit 2 ;; esac [ -n "$suites" ] || suites="zephyr,at,lte" for s in $(echo "$suites" | tr ',' ' '); do case "$s" in zephyr|at|lte|qa) ;; *) echo "$PROG: unknown suite: $s" >&2; exit 2 ;; esac done case "$profile" in ''|asset_tracker|hello_mcuboot|generic) ;; *) echo "$PROG: unknown profile: $profile" >&2; exit 2 ;; esac case "$reset_mode" in ''|shell|swd) ;; *) echo "$PROG: --reset takes shell or swd" >&2; exit 2 ;; esac case "$speed" in *[!0-9]*|0*) echo "$PROG: bad --speed '$speed': give kHz as a whole number" >&2; exit 2 ;; esac case "$qa_version" in *[[:cntrl:]]*|*[[:space:]]*) echo "$PROG: bad --qa-version '$qa_version'" >&2; exit 2 ;; esac say() { echo "$PROG: $*"; } now() { date +%s; } want() { case ",$suites," in *",$1,"*) return 0 ;; esac; return 1; } if want qa && [ "$reset_ok" -eq 0 ]; then echo "$PROG: --suite qa resets the board: give --allow-reset (nothing was opened)" >&2 exit 2 fi # --- what may be sent --------------------------------------------------------------- # Read-only lines, one per line. Taken from the allowlist of a native survey of # the same board (2026-09-28), plus the reads the LTE suite needs. # `at AT+BOGUS` is the negative test: the modem must refuse it. ALLOW_RO=' help version kernel version kernel uptime kernel thread list device list date get fs statvfs /lfs1 rtc_app get pmic_charger status pmic_regulator status gpio_interrupt net iface watchdog status nrf_provisioning uuid ff show softsim status lis2dw12 get lps22hh get at AT at AT+BOGUS at AT+CMEE? at AT+CGMI at AT+CGMM at AT+CGMR at AT%SHORTSWVER at AT%HWVERSION at AT%XMODEMUUID at AT+CGSN=1 at AT+CFUN? at AT%XSYSTEMMODE? at AT+CEMODE? at AT%XTEMP? at AT%XVBAT at AT+CPIN? at AT%XICCID at AT+CIMI at AT+CEREG? at AT+COPS? at AT%XMONITOR at AT+CESQ at AT%CONEVAL at AT+CGDCONT? at AT+CGACT? at AT+CGPADDR at AT+CPSMS? at AT+CEDRXS? at AT%XBANDLOCK? at AT%CMNG=1 at AT+CCLK? at AT%XCONNSTAT? kernel device kernel thread stacks fs fs ls fs ls lfs1/ fs ls /lfs1 fs cat lfs1/test1 fs read lfs1/atest ' # With --allow-writes, these as well. Each is undone by the check that sends it. ALLOW_RW=' at AT+CFUN=4 at AT+CFUN=1 at AT%XCONNSTAT=1 ' # With --allow-reset. ALLOW_RESET=' kernel reboot cold ' # With --allow-fs-writes: QA Test Plan 3.6, and the removal of its two files. ALLOW_FS=' fs cd lfs1 fs erase_write_test atest 300 2 fs read_test atest 2 fs write test1 54 65 73 74 fs trunc test1 fs read test1 fs cat test1 fs rm /lfs1/atest fs rm /lfs1/test1 ' allowed() { # grep -F reads a pattern with a newline as several, and printf sends each line. case "$1" in *[[:cntrl:]]*) return 1 ;; esac case "$1" in # never, whatever the flags *[Aa][Tt]%[Xx][Ff][Aa][Cc][Tt][Oo][Rr][Yy]*|*[Aa][Tt]'&'[Ff]*|*%[Cc][Mm][Nn][Gg]=3*|*+[Cc][Ff][Uu][Nn]=0*) return 1 ;; esac printf '%s\n' "$ALLOW_RO" | grep -Fxq -- "$1" && return 0 [ "$writes" -eq 1 ] && printf '%s\n' "$ALLOW_RW" | grep -Fxq -- "$1" && return 0 [ "$reset_ok" -eq 1 ] && printf '%s\n' "$ALLOW_RESET" | grep -Fxq -- "$1" && return 0 [ "$fs_writes" -eq 1 ] && printf '%s\n' "$ALLOW_FS" | grep -Fxq -- "$1" && return 0 return 1 } if [ -n "$send_line" ] && ! allowed "$send_line"; then say "refused, nothing sent: '$send_line' is not on the read-only allowlist" exit 2 fi # --- the console ---------------------------------------------------------------------- if [ -z "$console" ]; then if command -v som-console >/dev/null 2>&1; then console=$(som-console --find --id "$CONSOLE_ID" 2>/dev/null) || console="" else for d in /dev/cu.usbserial-*; do [ -e "$d" ] || continue [ -n "$console" ] && { console=""; say "more than one /dev/cu.usbserial-*: name one with --console"; break; } console=$d done fi fi T0=$(now) [ -n "$out" ] || out=/tmp/som-test-$(date -u +%Y%m%dT%H%M%SZ) mkdir -p "$out" 2>/dev/null || { echo "$PROG: cannot create $out" >&2; exit 2; } W=$out/.work mkdir -p "$W" || exit 2 RAW=$W/raw : >"$RAW" : >"$out/results.tsv" : >"$W/ids" RD="" if sleep 0.1 2>/dev/null; then NAP=0.2; else NAP=1; fi nap() { sleep "$NAP" 2>/dev/null || sleep 1; } clean() { tr -d '\r\033\000' | sed 's/\[[0-9;]*[A-Za-z]//g'; } fsize() { echo $(( $(wc -c <"$1") )); } slice() { # everything the board said after byte $1, cleaned if [ "$1" -gt 0 ]; then dd if="$RAW" bs="$1" skip=1 2>/dev/null | clean else clean <"$RAW" fi } PROMPT_RE='^[A-Za-z0-9_-]*:~\$ *$' PN="" # the prompt itself ("uart:~$"), as rig.prompt saw it PROMPT_START='^[A-Za-z0-9_-]*:~\$' SEQ=0 [ -n "${SOM_TEST_DEBUG:-}" ] && mkdir -p "$out/debug" finish() { if [ -n "$RD" ]; then kill "$RD" 2>/dev/null; wait "$RD" 2>/dev/null; RD=""; fi exec 3<&- 2>/dev/null # The transcript, cleaned and with identity numbers masked. if [ -s "$RAW" ]; then clean <"$RAW" >"$W/transcript" if [ "$identity" -eq 0 ] && [ -s "$W/ids" ]; then while read -r v; do [ -n "$v" ] || continue m=$(mask "$v") sed "s/$v/$m/g" "$W/transcript" >"$W/t2" && mv "$W/t2" "$W/transcript" done <"$W/ids" fi mv "$W/transcript" "$out/transcript.txt" fi if [ -n "${SOM_TEST_DEBUG:-}" ]; then mv "$RAW" "$out/debug/raw.bin"; fi rm -rf "$W" } trap 'finish' EXIT trap 'exit 130' INT TERM mask() { # all but the last 4 digits echo "$1" | sed 's/^$.*$$....$$/\1#\2/' | sed 's/^[^#]*#/***********/' } remember_id() { echo "$1" >>"$W/ids"; } show_id() { if [ "$identity" -eq 1 ]; then echo "$1"; else mask "$1"; fi; } con_open() { [ -n "$console" ] && [ -e "$console" ] || return 1 if [ -d "$console" ]; then [ -p "$console/in" ] && [ -p "$console/out" ] || return 1 exec 3<>"$console/in" || return 1 dd bs=256 <"$console/out" >>"$RAW" 2>/dev/null & RD=$! return 0 fi exec 3<>"$console" || return 1 stty 115200 raw -echo -echoe -echok clocal <&3 2>/dev/null || return 1 dd bs=256 <&3 >>"$RAW" 2>/dev/null & RD=$! return 0 } # send LINE [SECONDS]: send one allowlisted line, wait for the shell's prompt, # and leave the reply in $W/full (with log lines) and $W/reply (without the # echo, prompts and log lines). Returns 0 on a prompt, 1 on a timeout, 2 if the # line is refused (then nothing is sent). send() { s_line=$1; s_tmo=${2:-6} : >"$W/full"; : >"$W/reply" allowed "$s_line" || { say "refused, nothing sent: $s_line"; return 2; } s_off=$(fsize "$RAW") [ "$s_off" -gt 0 ] || { printf '\r' >&3; nap; s_off=$(fsize "$RAW"); [ "$s_off" -gt 0 ] || s_off=1; } s_n=0 until printf '%s\r' "$s_line" >&3 2>/dev/null; do # EINTR on a tty (brintos/linux#73) s_n=$((s_n + 1)); [ "$s_n" -ge 3 ] && return 1 done s_at=0 case "$s_line" in "at "*) s_at=1 ;; esac s_end=$(( $(now) + s_tmo )) s_last=-1 s_ok=1 s_rc=1 # The shell prints its prompt only once a command has finished: log lines # that arrive while one runs carry no prompt. So any prompt after the echo # ends the reply -- also "uart:~$ [log line]", which is what an idle shell # prints when a log arrives, and after which it may not redraw a bare # prompt until the next keystroke (measured 2026-10-06). The reply is done # once that is seen and nothing more has arrived since. # # Every process costs about 0.15 s in a brintOS guest (measured # 2026-10-06), so a poll that finds nothing new costs one wc; only new # bytes cost a dd | tr | awk, which cleans them, keeps them in $W/full and # decides, in one pass. nap while :; do s_size=$(fsize "$RAW") if [ "$s_size" = "$s_last" ]; then [ "$s_ok" -eq 0 ] && { s_rc=0; break; } else s_last=$s_size dd if="$RAW" bs="$s_off" skip=1 2>/dev/null | tr -d '\r\033\000' | awk -v c="$s_line" -v at="$s_at" -v out="$W/full" -v pn="$PN" ' { gsub(/\[[0-9;]*[A-Za-z]/, "") # Output without a final newline (`fs cat` of a 4-byte file) runs into the prompt. if (e && pn != "" && (k = index($0, pn)) > 1) { print substr($0, 1, k - 1) > out; $0 = substr($0, k) } print > out if (!e && (c == "" || index($0, c))) { e = NR; if (pn == "" && match($0, /[A-Za-z0-9_-]*:~\$/)) pn = substr($0, RSTART, RLENGTH); next } if (e && $0 ~ /^[A-Za-z0-9_-]*:~\$/) p = 1 if ($0 ~ /^(OK|ERROR|\+CM[ES] ERROR.*)$/) ok = 1 } END { if (NR == 0) printf "" > out exit !(e && p && (at == 0 || ok)) }' s_ok=$? fi [ "$(now)" -ge "$s_end" ] && break nap done if [ -n "${SOM_TEST_DEBUG:-}" ]; then SEQ=$((SEQ + 1)) sed -n p "$W/full" >"$out/debug/$SEQ-$(echo "$s_line" | tr -c 'A-Za-z0-9_\n' '_').txt" fi s_echo=$(grep -n -F -- "$s_line" "$W/full" | sed -n '1s/:.*//p') sed -e "1,${s_echo:-1}d" -e "/$PROMPT_RE/d" -e '/^[A-Za-z0-9_-]*:~\$ /d' \ -e '/^\[[0-9]*:[0-9][0-9]:[0-9][0-9]\.[0-9]*,[0-9]*\] "$W/reply" return "$s_rc" } reply_has() { grep -q -E "$1" "$W/reply"; } reply_val() { # the rest of the first reply line starting with $1 sed -n "/^$1/{s/^$1[[:space:]]*//p;q;}" "$W/reply" } at_ok() { grep -qx 'OK' "$W/reply"; } at_err() { grep -q -E '^(ERROR|\+CM[ES] ERROR.*)$' "$W/reply"; } # --- results ------------------------------------------------------------------------- KNOWN="" known() { case ",$KNOWN," in *",$1,"*) return 0 ;; esac; return 1; } tally() { eval "c_$1_$2=\$(( \${c_$1_$2:-0} + 1 ))"; } count() { eval "echo \${c_$1_$2:-0}"; } FAILS=0 result() { # CHECK STATE DETAIL r_id=$1; r_st=$2; r_det=${3:-} if [ "$r_st" = FAIL ] && known "$r_id"; then r_st=XFAIL; r_det="$r_det (known issue)"; fi [ "$r_st" = PASS ] && known "$r_id" && r_det="$r_det (listed as known, but passed)" echo "$PROG: $r_id $r_st -- $r_det" printf '%s\t%s\t%s\n' "$r_id" "$r_st" "$r_det" >>"$out/results.tsv" r_suite=${r_id%%.*} case "$r_st" in PASS) r_lc=pass ;; FAIL) r_lc=fail ;; SKIP) r_lc=skip ;; *) r_lc=xfail ;; esac tally "$r_suite" "$r_lc" [ "$r_st" = FAIL ] && FAILS=$((FAILS + 1)) return 0 } rig_fail() { result "rig.$1" FAIL "$2" say "FAIL -- rig: $2; the board was not tested" exit 3 } # --- one line, for a person or an AI ------------------------------------------------- if [ -n "$send_line" ]; then con_open || { say "cannot open the console ${console:-(none found)}"; exit 3; } send "" 3 >/dev/null 2>&1 || true if send "$send_line" 15; then sed -n p "$W/reply" exit 0 fi say "no prompt within 15 s after: $send_line" sed -n p "$W/full" exit 3 fi # --- rig ------------------------------------------------------------------------------- say "console: ${console:-none found}" say "out: $out" [ -n "$console" ] || rig_fail console "no console: grant the FT230X ($CONSOLE_ID) or name it with --console" con_open || rig_fail console "cannot open or set up $console" result rig.console PASS "$console at 115200 8N1" # Wake the shell: Enter until a prompt answers. The prompt may end a line that # holds something else, e.g. "uart:~$ uart:~$ " when a log line was redrawn # (seen 2026-10-07), so only the end of the line is matched here. The write is # retried like send()'s (EINTR on a tty, brintos/linux#73). r=1 for i in 1 2 3; do s_off=$(fsize "$RAW") n=0 until printf '\r' >&3 2>/dev/null; do n=$((n + 1)); [ "$n" -ge 3 ] && break done e=$(( $(now) + 3 )) while [ "$(now)" -lt "$e" ]; do if slice "$s_off" | grep -q ':~\$ *$'; then r=0; break; fi nap done [ "$r" -eq 0 ] && break done if [ "$r" -ne 0 ]; then if [ -s "$RAW" ]; then rig_fail prompt "the console sends data but no shell prompt (wrong speed, or not a Zephyr shell?)" fi rig_fail prompt "no answer from the board's shell in 9 s. Powered? A probe on the debug header can silence the console" fi PN=$(slice "$s_off" | grep -o '[A-Za-z0-9_-]*:~\$' | tail -1) result rig.prompt PASS "the Zephyr shell answers" send help 8 || rig_fail help "no complete reply to help" sed -n 's/^ $[a-z_][a-z0-9_]*$ *:.*/\1/p' "$W/reply" >"$W/cmds" ncmd=$(sed -n '$=' "$W/cmds"); ncmd=${ncmd:-0} [ "$ncmd" -gt 0 ] || rig_fail help "help lists no commands" result rig.help PASS "$ncmd shell commands: $(tr '\n' ' ' <"$W/cmds" | sed 's/ $//')" has() { grep -qx "$1" "$W/cmds"; } U0="" if has kernel && send "kernel uptime"; then U0=$(reply_val 'Uptime:' | sed 's/ ms.*//'); fi # --- profile --------------------------------------------------------------------------- VERSION="" if has version && send version; then VERSION=$(reply_val 'Firmware version:'); fi if [ -z "$profile" ]; then case "$VERSION" in *asset_tracker_v2*v1.2-114*) profile=asset_tracker ;; *hello_mcuboot*) profile=hello_mcuboot ;; *) profile=generic ;; esac fi # P_CFUN is the modem's functional mode the firmware should leave it in: # 1 for an app that attaches to LTE, 0 for one that never starts the modem. P_NAME=""; P_VERSION=""; P_MFW=""; P_WDT=""; P_DISABLED=""; P_KNOWN=""; P_CFUN=1 case "$profile" in asset_tracker) # Emcraft release 1.2-114, debug or release build. P_NAME=asset_tracker_v2-1.2-114 P_VERSION="asset_tracker_v2 v1.2-114" P_MFW=mfw_nrf91x1_2.0.2 P_WDT=enabled # This build's date_time_work_q (1280 B) reaches 1212 B (94%) whenever # the date_time library falls back to NTP (no %XTIME from the network # before the update runs); 44% otherwise. Reproduced 2026-10-06 with # AT%XTIME=0 before attach. The fix is Emcraft's: a bigger # CONFIG_DATE_TIME_THREAD_STACK_SIZE. qa.stacks reads the same stacks. P_KNOWN=zephyr.threads,qa.stacks ;; hello_mcuboot) # Emcraft's MCUboot demo: a shell and the `at` command, but it never # starts the modem, so +CFUN: 0 and no SIM or LTE (seen 2026-10-09). P_NAME=hello_mcuboot P_VERSION=hello_mcuboot P_CFUN=0 ;; generic) P_NAME=generic ;; esac KNOWN=$(echo "$P_KNOWN,$known_add" | sed 's/^,//; s/,$//') say "profile: $P_NAME${KNOWN:+ (known: $KNOWN)}" # --- Zephyr app ----------------------------------------------------------------------- run_zephyr() { # Identity and build if [ -z "$VERSION" ]; then if has version; then result zephyr.version FAIL "\`version\` gave no firmware version" else result zephyr.version SKIP "no version command"; fi elif [ -n "$P_VERSION" ]; then case "$VERSION" in *"$P_VERSION"*) result zephyr.version PASS "$VERSION" ;; *) result zephyr.version FAIL "expected '$P_VERSION', the board runs: $VERSION" ;; esac else result zephyr.version PASS "$VERSION (no expectation in profile $P_NAME)" fi if has kernel; then if [ "$quick" -eq 0 ]; then if send "kernel version" && reply_has '^Zephyr version'; then result zephyr.kernel_version PASS "Zephyr $(reply_val 'Zephyr version')" else result zephyr.kernel_version FAIL "no Zephyr version from \`kernel version\`" fi fi u1="" if send "kernel uptime"; then u1=$(reply_val 'Uptime:' | sed 's/ ms.*//'); fi case "$u1" in *[!0-9]*|'') result zephyr.uptime FAIL "\`kernel uptime\` gave no number" ;; *) if [ -z "$U0" ] || [ "$u1" -gt "$U0" ]; then result zephyr.uptime PASS "running, up $((u1 / 1000)) s" else result zephyr.uptime FAIL "uptime did not grow ($U0 then $u1 ms): the board restarted?" fi ;; esac if [ "$quick" -eq 0 ]; then if send "kernel thread list" 10 && reply_has '^Threads:'; then # The busiest stack: its percentage, thread and bytes. A # high-water mark, so it only grows over a boot. set -- $(awk '$1 ~ /^\*?0x[0-9a-fA-F]+$/ { n = ($2 == "" ? "(unnamed)" : $2) } /usage [0-9]+ \/ [0-9]+ $/ { t++; p = $(NF - 1); sub(/\(/, "", p) if (p + 0 >= w) { w = p + 0; wn = n; u = $0 sub(/.*usage /, "", u); sub(/ \(.*/, "", u); gsub(/ /, "", u) } } END { print t + 0, w + 0, wn, u }' "$W/reply") nth=$1; worst=$2 if [ "$worst" -ge 90 ]; then result zephyr.threads FAIL "$nth threads; $3 is at ${worst}% of its stack ($4 B, 90% or more)" else result zephyr.threads PASS "$nth threads, highest stack use ${worst}% ($3, $4 B)" fi elif reply_has 'Subcommands:|wrong parameter|command not found'; then result zephyr.threads SKIP "kernel thread list is not in this build" else result zephyr.threads FAIL "no thread list from \`kernel thread list\`" fi fi else result zephyr.uptime SKIP "no kernel command" fi # Devices if has device && send "device list" 8 && reply_has '^- '; then ndev=$(grep -c '^- ' "$W/reply") nready=$(grep -c '^- .*(READY)$' "$W/reply") sed -n 's/^- \(.*$ ($.*$)$/\1 \2/p' "$W/reply" | grep -v ' READY$' >"$W/notready" bad=""; exp="" while read -r dn st; do case " $P_DISABLED " in *" $dn "*) exp="$exp $dn=$st" ;; *) bad="$bad $dn=$st" ;; esac done <"$W/notready" if [ -z "$bad" ]; then result zephyr.devices PASS "$ndev devices, $nready READY${exp:+; expected:$exp}" else result zephyr.devices FAIL "$ndev devices, $nready READY; not ready:$bad" fi elif has device; then result zephyr.devices FAIL "no device list" else result zephyr.devices SKIP "no device command" fi # Watchdog: read only. Enabled after a fresh boot of the app. if has watchdog; then send "watchdog status" w=$(sed -n 's/.*Watchdog status:[[:space:]]*//p' "$W/reply" | sed -n 1p) if [ -z "$w" ]; then result zephyr.watchdog FAIL "no status from \`watchdog status\`" elif [ -n "$P_WDT" ] && [ "$w" != "$P_WDT" ]; then result zephyr.watchdog FAIL "watchdog is $w, expected $P_WDT (disabled since boot? a reset turns it back on)" else result zephyr.watchdog PASS "$w" fi else result zephyr.watchdog SKIP "no watchdog command" fi [ "$quick" -eq 1 ] && return 0 # Filesystem if has fs; then if send "fs statvfs /lfs1" && reply_has 'blocks'; then fv=$(sed -n 1p "$W/reply") blocks=$(echo "$fv" | sed -n 's/.* blocks $[0-9]*$.*/\1/p') bfree=$(echo "$fv" | sed -n 's/.* bfree $[0-9]*$.*/\1/p') if [ "${blocks:-0}" -gt 0 ]; then result zephyr.fs PASS "LittleFS /lfs1 mounted: $blocks blocks, $bfree free" else result zephyr.fs FAIL "/lfs1: $fv" fi else result zephyr.fs FAIL "/lfs1 not mounted: $(sed -n 1p "$W/reply")" fi else result zephyr.fs SKIP "no fs command" fi # External RTC: it must answer. Holding no time since power-up (no backup # cell) is a bench condition, reported as a SKIP of the timekeeping check. if has rtc_app; then send "rtc_app get" if grep -q -E '[0-9]{4}-[0-9]{2}-[0-9]{2}' "$W/reply" && ! grep -q -E '^19[0-9]{2}-' "$W/reply"; then result zephyr.rtc PASS "pcf85263a keeps time: $(grep -E '[0-9]{4}-[0-9]{2}-[0-9]{2}' "$W/reply" | sed -n 1p)" elif grep -q -E 'oscillator stopped|RTC not set|err: -61' "$W/full"; then result zephyr.rtc SKIP "the RTC answers but holds no valid time (oscillator-stop flag since power loss; no backup cell?)" else result zephyr.rtc FAIL "no answer from the RTC: $(sed -n 1p "$W/full")" fi else result zephyr.rtc SKIP "no rtc_app command" fi # PMIC if has pmic_charger; then if send "pmic_charger status" && reply_has 'Battery'; then if reply_has 'Battery not detected'; then result zephyr.pmic_charger PASS "nPM1300 charger answers; no battery fitted (bench condition)" else result zephyr.pmic_charger PASS "nPM1300 charger answers: $(sed -n 's/^[[:space:]]*Battery voltage:[[:space:]]*//p' "$W/reply")" fi else result zephyr.pmic_charger FAIL "no charger status" fi fi if has pmic_regulator; then if send "pmic_regulator status" && reply_has 'Regulator:'; then result zephyr.pmic_regulator PASS "$(grep -c 'Regulator:' "$W/reply") regulators report" else result zephyr.pmic_regulator FAIL "no regulator status" fi fi if has gpio_interrupt; then if send gpio_interrupt && reply_has '^P[0-9]+\.[0-9]+:'; then result zephyr.gpio_interrupt PASS "$(grep -c -E '^P[0-9]+\.[0-9]+:' "$W/reply") pin counters" else result zephyr.gpio_interrupt FAIL "no counters from gpio_interrupt" fi fi # Sensors on the SOM, read once. The board is assumed at rest on the bench. if has lis2dw12; then if send "lis2dw12 get" && reply_has '^accel x:'; then a=$(reply_val 'accel' | sed 's/[xyz]://g; s/ms\/2//g') g=$(echo "$a" | awk '{ printf "%.1f", sqrt($1 * $1 + $2 * $2 + $3 * $3) }') if echo "$g" | awk '{ exit !($1 >= 8.0 && $1 <= 11.6) }'; then result zephyr.accel PASS "lis2dw12 at rest: |a| = $g m/s2 ($(echo "$a" | awk '{ printf "x %.1f y %.1f z %.1f", $1, $2, $3 }'))" else result zephyr.accel FAIL "lis2dw12: |a| = $g m/s2, expected about 9.8 at rest ($a)" fi else result zephyr.accel FAIL "no data from \`lis2dw12 get\`: $(sed -n 1p "$W/reply")" fi fi if has lps22hh; then if send "lps22hh get" && reply_has '^Pressure:'; then kpa=$(reply_val 'Pressure:' | sed 's/ kPa//') tc=$(reply_val 'Temperature:' | sed 's/ C//') if echo "$kpa $tc" | awk '{ exit !($1 >= 80 && $1 <= 110 && $2 >= -20 && $2 <= 85) }'; then result zephyr.pressure PASS "lps22hh: $kpa kPa, $tc C" else result zephyr.pressure FAIL "lps22hh out of range: $kpa kPa, $tc C" fi else result zephyr.pressure FAIL "no data from \`lps22hh get\`: $(sed -n 1p "$W/reply")" fi fi if has net; then if send "net iface" && reply_has '^Interface '; then ifs=$(sed -n 's/^Interface $[^ ]*$ .*/\1/p' "$W/reply" | tr '\n' ' ' | sed 's/ $//') ndown=$(grep -c 'Interface is down' "$W/reply") result zephyr.net_iface PASS "interfaces: $ifs ($ndown down; the LTE link is offloaded to the modem)" else result zephyr.net_iface FAIL "no interfaces from \`net iface\`" fi fi if has nrf_provisioning; then if send "nrf_provisioning uuid" && reply_has '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; then result zephyr.provisioning PASS "device UUID $(sed -n 1p "$W/reply")" else result zephyr.provisioning FAIL "no device UUID" fi fi } # --- AT command set --------------------------------------------------------------------- SIM=unknown SIM_WHY="" # why the SIM is not ready, for the checks that skip on it cfun_name() { case "$1" in 0) echo "modem off" ;; 1) echo "normal" ;; 4) echo "flight mode" ;; *) echo "mode $1" ;; esac } at_q() { # CHECK AT-COMMAND PREFIX [quiet]: send, and leave the value in $V. # 0 OK; 1 the modem said ERROR; 2 no reply or no OK. On 1 and 2 a FAIL is # recorded, unless "quiet" says the caller reports it. V="" if ! send "at $2" 10; then [ "${4:-}" = quiet ] || result "$1" FAIL "no reply to $2 in 10 s" return 2 fi if at_err; then [ "${4:-}" = quiet ] || result "$1" FAIL "$2: $(sed -n 1p "$W/reply")" return 1 fi if [ -n "$3" ]; then V=$(reply_val "$3"); else V=$(grep -v -x 'OK' "$W/reply" | sed -n 1p); fi at_ok || { [ "${4:-}" = quiet ] || result "$1" FAIL "$2: no OK"; return 2; } return 0 } at_info() { # CHECK AT-COMMAND PREFIX: PASS if the modem answers at_q "$1" "$2" "$3" && result "$1" PASS "${V:-OK}" } run_at() { if ! has at; then result at.at SKIP "no \`at\` command in this build" return 0 fi if send "at AT" 10 && at_ok; then result at.at PASS "OK" else result at.at FAIL "bare AT: $(sed -n 1p "$W/reply")"; return 0; fi if [ "$quick" -eq 0 ]; then if send "at AT+BOGUS" 10 && at_err; then result at.reject_unknown PASS "an unknown command is refused: $(sed -n 1p "$W/reply")" else result at.reject_unknown FAIL "AT+BOGUS was not refused: $(sed -n 1p "$W/reply")" fi at_info at.cmee "AT+CMEE?" "+CMEE:" at_q at.cgmi "AT+CGMI" "" && case "$V" in *Nordic*) result at.cgmi PASS "$V" ;; *) result at.cgmi FAIL "manufacturer: $V" ;; esac at_q at.cgmm "AT+CGMM" "" && case "$V" in nRF91*) result at.cgmm PASS "$V" ;; *) result at.cgmm FAIL "model: $V" ;; esac fi if at_q at.cgmr "AT+CGMR" ""; then if [ -n "$P_MFW" ] && [ "$V" != "$P_MFW" ]; then result at.cgmr FAIL "modem firmware $V, expected $P_MFW" else result at.cgmr PASS "$V" fi fi CFUN0="" if at_q at.cfun "AT+CFUN?" "+CFUN:"; then if [ "$V" = "$P_CFUN" ]; then result at.cfun PASS "$V ($(cfun_name "$V"))" else result at.cfun FAIL "+CFUN: $V ($(cfun_name "$V")), expected $P_CFUN ($(cfun_name "$P_CFUN"))" fi CFUN0=$V fi # With the modem off the SIM is powered down too, so +CPIN? says ERROR: # that is the firmware's choice, not a missing SIM. if at_q at.sim "AT+CPIN?" "+CPIN:" quiet; then case "$V" in READY) SIM=ready; result at.sim PASS "READY" ;; *) SIM=locked; SIM_WHY="SIM not ready ($V)"; result at.sim FAIL "SIM needs: $V" ;; esac elif [ $? -eq 1 ]; then if [ -n "$CFUN0" ] && [ "$CFUN0" != 1 ]; then SIM=off; SIM_WHY="modem not in normal mode (+CFUN: $CFUN0, $(cfun_name "$CFUN0")), so the SIM is not read" else SIM=none; SIM_WHY="no SIM (+CPIN? ERROR): bench condition" fi result at.sim SKIP "$SIM_WHY" else result at.sim FAIL "no reply to AT+CPIN?" fi [ "$quick" -eq 1 ] && return 0 at_info at.shortswver "AT%SHORTSWVER" "%SHORTSWVER:" at_info at.hwversion "AT%HWVERSION" "%HWVERSION:" at_info at.modemuuid "AT%XMODEMUUID" "%XMODEMUUID:" if at_q at.imei "AT+CGSN=1" "+CGSN:"; then imei=$(echo "$V" | tr -d '"') remember_id "$imei" case "$imei" in [0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]) result at.imei PASS "IMEI $(show_id "$imei")" ;; *) result at.imei FAIL "not an IMEI: $(show_id "$imei")" ;; esac fi if at_q at.systemmode "AT%XSYSTEMMODE?" "%XSYSTEMMODE:"; then m="" set -- $(echo "$V" | tr ',' ' ') [ "${1:-0}" = 1 ] && m="$m LTE-M" [ "${2:-0}" = 1 ] && m="$m NB-IoT" [ "${3:-0}" = 1 ] && m="$m GNSS" result at.systemmode PASS "$V:${m:- none}" fi at_info at.cemode "AT+CEMODE?" "+CEMODE:" if at_q at.temp "AT%XTEMP?" "%XTEMP:"; then if [ "$V" -ge 0 ] 2>/dev/null && [ "$V" -le 85 ]; then result at.temp PASS "$V C" else result at.temp FAIL "modem temperature $V C, outside 0..85"; fi fi if at_q at.vbat "AT%XVBAT" "%XVBAT:"; then if [ "$V" -ge 3000 ] 2>/dev/null && [ "$V" -le 5500 ]; then result at.vbat PASS "$V mV" else result at.vbat FAIL "modem supply $V mV, outside 3000..5500"; fi fi if [ "$SIM" = ready ]; then if at_q at.iccid "AT%XICCID" "%XICCID:"; then remember_id "$V"; result at.iccid PASS "ICCID $(show_id "$V")" fi if at_q at.imsi "AT+CIMI" ""; then remember_id "$V"; result at.imsi PASS "IMSI $(show_id "$V")" fi else result at.iccid SKIP "${SIM_WHY:-no SIM}" result at.imsi SKIP "${SIM_WHY:-no SIM}" fi at_info at.cops "AT+COPS?" "+COPS:" at_info at.xmonitor "AT%XMONITOR" "%XMONITOR:" at_info at.cesq "AT+CESQ" "+CESQ:" # %CONEVAL measures the radio link, so the modem refuses it unless it is on. if [ -z "$CFUN0" ] || [ "$CFUN0" = 1 ]; then at_info at.coneval "AT%CONEVAL" "%CONEVAL:" else result at.coneval SKIP "needs the modem in normal mode (+CFUN: $CFUN0, $(cfun_name "$CFUN0"))" fi if at_q at.cgdcont "AT+CGDCONT?" "+CGDCONT:"; then result at.cgdcont PASS "${V:-no PDP context defined}"; fi at_info at.cpsms "AT+CPSMS?" "+CPSMS:" if at_q at.cedrxs "AT+CEDRXS?" "+CEDRXS:"; then result at.cedrxs PASS "${V:-no eDRX requested}"; fi at_info at.bandlock "AT%XBANDLOCK?" "%XBANDLOCK:" if has softsim; then if send "softsim status" && reply_has 'SoftSIM'; then result at.softsim PASS "$(grep 'SoftSIM' "$W/reply" | tr '\n' ';' | sed 's/;$//; s/;/; /g')" else result at.softsim FAIL "no SoftSIM status" fi fi if at_q at.cmng "AT%CMNG=1" ""; then result at.cmng PASS "$(grep -c '^%CMNG:' "$W/reply") stored credentials (listed, not read)" fi } # --- LTE link ------------------------------------------------------------------------------ decode_stat() { case "$1" in 0) echo "not registered, not searching" ;; 1) echo "registered, home" ;; 2) echo "searching" ;; 3) echo "registration denied" ;; 4) echo "unknown" ;; 5) echo "registered, roaming" ;; 90) echo "not registered, UICC failure" ;; *) echo "status $1" ;; esac } decode_act() { case "$1" in 7) echo "LTE-M" ;; 9) echo "NB-IoT" ;; 14) echo "NB-NTN (satellite)" ;; *) echo "AcT $1" ;; esac } field() { echo "$1" | cut -d, -f"$2" | tr -d '"'; } run_lte() { if ! has at; then result lte.registered SKIP "no \`at\` command in this build"; return 0; fi if [ "$SIM" = unknown ]; then if at_q lte.sim "AT+CPIN?" "+CPIN:" quiet && [ "$V" = READY ]; then SIM=ready; else SIM=none; fi fi if [ "$SIM" != ready ]; then result lte.registered SKIP "${SIM_WHY:-no SIM ready: bench condition}" return 0 fi if [ "$writes" -eq 1 ] && send "at AT%XCONNSTAT=1" 10 && at_ok; then say "data counters started (AT%XCONNSTAT=1); they stay on" fi # Registration, waited for. t=$(now); e=$(( t + lte_wait )); st=""; cereg="" while :; do if send "at AT+CEREG?" 10; then cereg=$(reply_val '+CEREG:') st=$(field "$cereg" 2) case "$st" in 1|5) break ;; esac fi [ "$(now)" -ge "$e" ] && break e2=$(( $(now) + 5 )); while [ "$(now)" -lt "$e2" ]; do nap; done done case "$st" in 1|5) act=$(field "$cereg" 5) result lte.registered PASS "$(decode_stat "$st") in $(( $(now) - t )) s" if [ -n "$act" ]; then result lte.rat PASS "$(decode_act "$act")" else result lte.rat SKIP "no access technology in +CEREG (unsolicited level below 2)"; fi ;; *) result lte.registered FAIL "not registered after ${lte_wait} s: $(decode_stat "${st:-?}") (+CEREG: $cereg)" return 0 ;; esac if at_q lte.operator "AT%XMONITOR" "%XMONITOR:"; then result lte.operator PASS "$(field "$V" 2) (PLMN $(field "$V" 4)), band $(field "$V" 7), cell $(field "$V" 8), TAC $(field "$V" 5)" fi # Signal: %CONEVAL measures only while the modem is awake; in PSM it says so. if at_q lte.signal "AT%CONEVAL" "%CONEVAL:"; then if [ "$(field "$V" 1)" = 0 ]; then rsrp=$(field "$V" 4); rsrq=$(field "$V" 5); snr=$(field "$V" 6) q=$(( rsrq * 5 - 195 )); sg="" [ "$q" -lt 0 ] && { sg=-; q=$(( -q )); } result lte.signal PASS "RSRP $((rsrp - 140)) dBm, RSRQ $sg$((q / 10)).$((q % 10)) dB, SNR $((snr - 24)) dB, energy estimate $(field "$V" 3) (5 worst .. 9 best)" else result lte.signal SKIP "no measurement now (%CONEVAL result $(field "$V" 1); the modem may be asleep in PSM)" fi fi # PDN apn=""; act="" if at_q lte.pdn "AT+CGDCONT?" "+CGDCONT:" quiet; then apn=$(field "$V" 3); fi if at_q lte.pdn "AT+CGACT?" "+CGACT:" quiet; then act=$(field "$V" 2); fi if [ -n "$apn" ] && [ "$act" = 1 ]; then result lte.pdn PASS "APN $apn, context active" else result lte.pdn FAIL "PDN not up (APN '${apn}', +CGACT state '${act}')" fi if at_q lte.ip "AT+CGPADDR" "+CGPADDR:"; then a=$(field "$V" 2) case "$a" in *.*.*.*) result lte.ip PASS "IPv4 $a" ;; *) result lte.ip FAIL "no IPv4 address (+CGPADDR: $V)" ;; esac fi # Network time if at_q lte.time "AT+CCLK?" "+CCLK:"; then yy=$(echo "$V" | tr -d '"' | cut -c1-2) if [ "${yy:-0}" -ge 25 ] 2>/dev/null; then result lte.time PASS "network time $(echo "$V" | tr -d '"')" else result lte.time FAIL "no network time (+CCLK: $V)"; fi fi if has date && send "date get"; then d=$(grep -E '^[0-9]{4}-' "$W/reply" | sed -n 1p) case "$d" in 19*|'') result lte.app_time FAIL "the app's clock is not set from the network: ${d:-no date}" ;; *) result lte.app_time PASS "app clock $d" ;; esac fi # Data path. With --allow-writes it is proven after the re-attach below; # read-only, by the modem's counters if someone started them, or by the # app's own cloud traffic if some happens during the run. if [ "$writes" -eq 0 ]; then check_data clean <"$RAW" >"$W/all" if grep -q -E 'CLOUD_EVT_CONNECTED|CLOUD_EVT_DATA_SEND|CLOUD_EVT_DATA_ACK' "$W/all"; then result lte.cloud PASS "the app exchanged data with nRF Cloud during the run: $(grep -o -E 'CLOUD_EVT_(CONNECTED|DATA_SEND[A-Z_]*|DATA_ACK)' "$W/all" | sort -u | tr '\n' ' ' | sed 's/ $//')" else result lte.cloud SKIP "no cloud traffic during the run (--allow-writes makes the app reconnect, and waits for it)" fi fi } check_data() { # the modem's byte counters if at_q lte.data "AT%XCONNSTAT?" "%XCONNSTAT:" quiet; then tx=$(field "$V" 3); rx=$(field "$V" 4); pmax=$(field "$V" 5) if [ "${tx:-0}" -gt 0 ] || [ "${rx:-0}" -gt 0 ] || [ "${pmax:-0}" -gt 0 ]; then result lte.data PASS "modem counted traffic: $tx kB sent, $rx kB received, largest packet $pmax B" elif [ "$writes" -eq 1 ]; then result lte.data FAIL "the modem counted no traffic since its counters started this run" else result lte.data SKIP "no traffic counted (counters off, or no traffic since they started)" fi else result lte.data SKIP "AT%XCONNSTAT? gave no counters" fi } wait_log() { # PATTERN OFFSET SECONDS: wait for a console line, sending nothing w_end=$(( $(now) + $3 )) while [ "$(now)" -lt "$w_end" ]; do slice "$2" | grep -q -E "$1" && return 0 nap; nap done slice "$2" | grep -q -E "$1" } # --- state-changing checks (--allow-writes) ------------------------------------------------ run_writes() { has at || return 0 if [ "${CFUN0:-}" != 1 ]; then result at.flight_mode SKIP "the modem was not in normal mode (+CFUN: ${CFUN0:-?}), so there is nothing to restore to" return 0 fi f_off=$(fsize "$RAW") if ! { send "at AT+CFUN=4" 15 && at_ok; }; then result at.flight_mode FAIL "AT+CFUN=4 refused: $(sed -n 1p "$W/reply")" else at_q at.flight_mode "AT+CFUN?" "+CFUN:" quiet; fm=$V send "at AT+CFUN=1" 15 at_q at.flight_mode "AT+CFUN?" "+CFUN:" quiet; back=$V if [ "$fm" = 4 ] && [ "$back" = 1 ]; then result at.flight_mode PASS "+CFUN=4 took effect and normal mode (1) is restored" else result at.flight_mode FAIL "flight mode read $fm, after restore $back" fi fi if [ "$SIM" = ready ]; then t=$(now); e=$(( t + lte_wait )); st="" while [ "$(now)" -lt "$e" ]; do if send "at AT+CEREG?" 10; then st=$(field "$(reply_val '+CEREG:')" 2) case "$st" in 1|5) break ;; esac fi e2=$(( $(now) + 2 )); while [ "$(now)" -lt "$e2" ]; do nap; done done case "$st" in 1|5) result lte.reattach PASS "registered again $(( $(now) - t )) s after leaving flight mode" ;; *) result lte.reattach FAIL "not registered ${lte_wait} s after leaving flight mode ($(decode_stat "${st:-?}"))" return 0 ;; esac # The app notices the outage and reconnects to nRF Cloud by itself: # DNS, TCP, TLS and MQTT over the new attach. That is the data path. if slice "$f_off" | grep -q 'CLOUD_EVT'; then t=$(now) if wait_log 'CLOUD_EVT_CONNECTED' "$f_off" "$cloud_wait"; then result lte.cloud PASS "the app reconnected to nRF Cloud $(( $(now) - t )) s after the re-attach" else result lte.cloud FAIL "no nRF Cloud reconnect within ${cloud_wait} s of the re-attach ($(slice "$f_off" | grep -o -E 'CLOUD_EVT_[A-Z_]*|err: -?[0-9]+' | sort -u | tr '\n' ' '))" fi else result lte.cloud SKIP "the app logs no cloud events (no CLOUD_EVT after flight mode)" fi check_data fi } # --- QA Test Plan (--suite qa) ------------------------------------------------------------ # Each check names the plan test it runs; the expected lines are copied from v23. QA_PLAN="QA Test Plan v23" QA_SINCE="" # the console offset after the last reset's boot; end.stable scans from it QA_HOW="" QA_CONSOLE="" qa_result() { # CHECK STATE TEST DETAIL result "$1" "$2" "$QA_PLAN $3: $4${QA_HOW:+ ($QA_HOW)}" } qa_reset() { # MODE: reset the board; QA_RESET and QA_T are the console offset and time QA_RESET=$(fsize "$RAW"); QA_T=$(now) QA_SINCE=$QA_RESET; U0="" case "$1" in shell) QA_HOW="soft reset over the shell, not the plan's power cycle" send "kernel reboot cold" 2 [ $? -ne 2 ] || { QA_BAD="kernel reboot cold was refused"; return 1; } ;; swd) QA_HOW="soft reset over SWD, not the plan's power cycle" command -v openocd >/dev/null 2>&1 || { QA_BAD="openocd is not installed"; return 1; } [ -r "$BOARD_CFG" ] || { QA_BAD="no board config at $BOARD_CFG"; return 1; } : >"$W/prelude.tcl" [ -n "$speed" ] && echo "set BRINTOS_JTAG_SPEED $speed" >>"$W/prelude.tcl" if [ -n "$adapter" ]; then iface="" case "$adapter" in */*) iface=$adapter ;; *.cfg) name=$adapter ;; *) name=$adapter.cfg ;; esac if [ -z "$iface" ]; then iface=interface/ftdi/$name [ -r "$SCRIPTS/interface/$name" ] && iface=interface/$name fi [ -r "$SCRIPTS/$iface" ] || { QA_BAD="no such adapter config: $SCRIPTS/$iface"; return 1; } echo "set BRINTOS_JTAG_IFACE $iface" >>"$W/prelude.tcl" fi # Nothing after the reset: a debugger read during the boot stalled 1.2-114's application. if ! openocd -f "$W/prelude.tcl" -f "$BOARD_CFG" -c "init; reset run; shutdown" >"$W/ocd.log" 2>&1; then QA_BAD="openocd could not reset the board: $(grep -E '^Error' "$W/ocd.log" | sed -n 1p)" return 1 fi ;; esac return 0 } qa_boot() { # SECONDS TEXT...: each TEXT on a console line after the reset, in order, within SECONDS of it b_end=$(( QA_T + $1 )); shift printf '%s\n' "$@" >"$W/boot" while :; do rm -f "$W/hit" QA_MISSING=$(slice "$QA_RESET" | awk -v pats="$W/boot" -v hit="$W/hit" ' BEGIN { while ((getline l < pats) > 0) p[++n] = l; k = 1 } k <= n && index($0, p[k]) { if (k == n) print > hit; k++ } END { if (k <= n) print p[k] }') [ -z "$QA_MISSING" ] && return 0 [ "$(now)" -ge "$b_end" ] && return 1 nap; nap done } qa_booted() { # after a reset's boot window: end.stable and its uptime start here QA_SINCE=$(fsize "$RAW") if has kernel && send "kernel uptime"; then U0=$(reply_val 'Uptime:' | sed 's/ ms.*//'); fi } qa_up() { # after qa_reset: the banner and the release's Start line within 30 s (3.1) if ! qa_boot 30 '*** Booting nRF Connect SDK' '*** Using Zephyr OS' 'main: Start'; then qa_booted QA_BAD="no '$QA_MISSING' line within 30 s of the reset" return 1 fi qa_booted st=$(sed -n 's/.*main: Start //p' "$W/hit") [ "$st" = "asset_tracker_v2 v$QAV on nrf9151som" ] && return 0 QA_BAD="the board started '$st', want 'asset_tracker_v2 v$QAV on nrf9151som'" return 1 } qa_step() { # LINE SECONDS [ERE...]: send LINE; each ERE must match a reply line, and no error q_l=$1; q_t=$2; shift 2 send "$q_l" "$q_t" || { QA_BAD="no prompt within $q_t s after '$q_l'"; return 1; } if reply_has '[Ff]ail|[Ee]rror|not found|[Ww]rong|[Ii]nvalid'; then QA_BAD="'$q_l': $(grep -E '[Ff]ail|[Ee]rror|not found|[Ww]rong|[Ii]nvalid' "$W/reply" | sed -n 1p)" return 1 fi for q_re; do reply_has "$q_re" || { QA_BAD="'$q_l' printed no line matching '$q_re': '$(sed -n 1p "$W/reply")'"; return 1; } done return 0 } qa_ramp() { # the rows `fs read` prints for the erase-write test's 300 bytes awk 'BEGIN { for (i = 0; i < 300; i++) { r = int(i / 16) row[r] = row[r] (i % 16 ? " " : "") sprintf("%02X", i % 256 % 255) } for (r = 0; r < 19; r++) print row[r] }' } qa_ramp_check() { # the hex columns of the reply against qa_ramp, row by row qa_ramp >"$W/ramp" QA_BAD=$(awk -v want="$W/ramp" ' BEGIN { while ((getline l < want) > 0) w[++n] = l } /^File size:/ { on = 1; next } on && /[0-9A-F][0-9A-F] [0-9A-F][0-9A-F] / { r++; g[r] = $0 if (r <= n && !bad && !index(" " $0 " ", " " w[r] " ")) bad = r } END { if (bad) printf "row %d (offset 0x%X) of lfs1/atest: want %s, the board printed: %s", bad, (bad - 1) * 16, w[bad], g[bad] else if (r != n) printf "lfs1/atest read back as %d rows, want %d", r, n }' "$W/reply") [ -z "$QA_BAD" ] } qa_stacks_parse() { # FILE: an OVER line per thread above 90 % of its stack, then a summary awk ' /$real size[[:space:]]*[0-9]+$:[[:space:]]*unused[[:space:]]+[0-9]+[[:space:]]+usage[[:space:]]+[0-9]+[[:space:]]*\/[[:space:]]*[0-9]+[[:space:]]*$[[:space:]]*[0-9]+[[:space:]]*%$/ { n = $0; sub(/^[[:space:]]*/, "", n); a = n; sub(/[[:space:]].*/, "", a) sub(/^0x[0-9a-fA-F]+[[:space:]]*/, "", n); sub(/[[:space:]]*\(real size.*/, "", n) if (n == "") n = a u = $0; sub(/.*usage[[:space:]]+/, "", u); s = u sub(/[[:space:]]*\/.*/, "", u); sub(/^[0-9]+[[:space:]]*\/[[:space:]]*/, "", s); sub(/[^0-9].*/, "", s) u += 0; s += 0; t++ if (s > 0 && 100 * u > 90 * s) printf "OVER %.1f %% %s (%d / %d)\n", 100 * u / s, n, u, s if (s > 0 && (t == 1 || u / s > w)) { w = u / s; wn = n; wu = u; ws = s } } END { printf "%d threads, highest %.1f %% %s (%d / %d)\n", t, 100 * w, wn, wu, ws }' "$1" } qa_console() { # 3.1 Serial Console Test QA_CONSOLE=FAIL t=$(now) if ! { qa_reset "$QA_MODE" && qa_up; }; then qa_result qa.console FAIL 3.1 "$QA_BAD"; return 0 fi up=$(( $(now) - t )) if ! { send kernel && reply_has '^kernel - Kernel commands' && reply_has '^Subcommands:'; }; then qa_result qa.console FAIL 3.1 "\`kernel\` did not list the kernel commands: '$(sed -n 1p "$W/reply")'"; return 0 fi if ! { send device && reply_has '^device - Device commands' && reply_has '^Subcommands:' && reply_has 'list +: List configured devices'; }; then qa_result qa.console FAIL 3.1 "\`device\` did not list 'list : List configured devices': '$(sed -n 1p "$W/reply")'"; return 0 fi QA_CONSOLE=PASS qa_result qa.console PASS 3.1 "the banner and 'main: Start asset_tracker_v2 v$QAV on nrf9151som' within $up s of the reset; kernel and device list their commands" } qa_version() { # 3.2 Version Test [ "$QA_CONSOLE" = PASS ] || { qa_result qa.version SKIP 3.2 "depends on qa.console (3.1), which did not pass"; return 0; } fv="Firmware version: asset_tracker_v2 v$QAV on nrf9151som" if ! send version; then qa_result qa.version FAIL 3.2 "no reply to \`version\`" elif ! grep -qxF -- "$fv" "$W/reply"; then qa_result qa.version FAIL 3.2 "want '$fv', the board printed '$(grep 'Firmware version:' "$W/reply" | sed -n 1p)'" elif ! grep -qx 'Hardware revision: 0' "$W/reply"; then hw=$(grep 'Hardware revision' "$W/reply" | sed -n 1p) qa_result qa.version FAIL 3.2 "want 'Hardware revision: 0', the board printed ${hw:+'$hw'}${hw:-no Hardware revision line}" else qa_result qa.version PASS 3.2 "$fv; Hardware revision: 0" fi } qa_stacks() { # 3.4 Thread Stack Utilization has at || { qa_result qa.stacks SKIP 3.4 "no \`at\` command to list the credentials (state:provisioned)"; return 0; } ty="" at_q qa.stacks "AT%CMNG=1" "" quiet && ty=$(sed -n 's/^%CMNG: *[0-9]*, *$[0-9]*$.*/\1/p' "$W/reply" | sort -u | tr '\n' ' ') for k in 0 1 2; do case " $ty" in *" $k "*) ;; *) qa_result qa.stacks SKIP 3.4 "not provisioned (state:provisioned): AT%CMNG=1 lists key types '${ty% }', want 0, 1 and 2" return 0 ;; esac done if ! { qa_reset "$QA_MODE" && qa_up; }; then qa_result qa.stacks FAIL 3.4 "$QA_BAD"; return 0 fi t=$(now) if ! wait_log 'CLOUD_EVT_CONNECTED' "$QA_RESET" "$qa_cloud_wait"; then qa_result qa.stacks FAIL 3.4 "booted, but no CLOUD_EVT_CONNECTED within $qa_cloud_wait s"; return 0 fi cl=$(( $(now) - t )) if ! send "kernel thread stacks" 10; then qa_result qa.stacks FAIL 3.4 "no complete reply to \`kernel thread stacks\` in 10 s"; return 0 fi qa_stacks_parse "$W/reply" >"$W/stacks" nth=$(sed -n '$s/ .*//p' "$W/stacks") if [ "${nth:-0}" -eq 0 ]; then qa_result qa.stacks FAIL 3.4 "no thread listing from \`kernel thread stacks\`: '$(sed -n 1p "$W/reply")'" elif grep -q '^OVER' "$W/stacks"; then qa_result qa.stacks FAIL 3.4 "$(grep '^OVER' "$W/stacks" | tr '\n' ';' | sed 's/;$//; s/;/; /g'); limit 90 %" else qa_result qa.stacks PASS 3.4 "CLOUD_EVT_CONNECTED $cl s after the boot; $(sed -n '$p' "$W/stacks"); limit 90 %" fi } qa_fs_steps() { # 3.6 as the plan writes it; QA_WROTE once the board's LittleFS may have changed qa_step "fs ls" 6 '^lfs1/$' || return 1 qa_step "fs cd lfs1" 6 || return 1 QA_WROTE=1 qa_step "fs erase_write_test atest 300 2" 30 '^Loop #1 done in' '^Loop #2 done in' '^Total: .*Per loop: .*Speed: ' || return 1 qa_step "fs read_test atest 2" 30 '^File size: 300\.0B$' '^Loop #1 done in' '^Loop #2 done in' '^Total: ' || return 1 qa_step "fs write test1 54 65 73 74" 6 || return 1 qa_step "fs trunc test1" 6 || return 1 qa_step "fs write test1 54 65 73 74" 6 || return 1 qa_step "fs read test1" 6 '^File size: 4$' '54 65 73 74 .*Test' || return 1 qa_step "fs cat test1" 6 '^Test$' || return 1 qa_reset "$QA_MODE" && qa_up || return 1 qa_step "fs ls lfs1/" 6 '^atest$' '^test1$' || return 1 qa_step "fs cat lfs1/test1" 6 '^Test$' || return 1 qa_step "fs read lfs1/atest" 10 '^File size: 300$' || return 1 qa_ramp_check } qa_fs() { # 3.6 SPI Flash Test [ "$fs_writes" -eq 1 ] || { qa_result qa.fs SKIP 3.6 "writes the board's LittleFS: --allow-fs-writes"; return 0; } [ "$QA_CONSOLE" = PASS ] || { qa_result qa.fs SKIP 3.6 "depends on qa.console (3.1), which did not pass"; return 0; } has fs || { qa_result qa.fs SKIP 3.6 "no fs command"; return 0; } send fs || { qa_result qa.fs FAIL 3.6 "no reply to \`fs\`"; return 0; } for c in erase_write_test read_test; do grep -q "^ *$c *:" "$W/reply" || { qa_result qa.fs SKIP 3.6 "this build's fs shell has no $c"; return 0; } done QA_WROTE=0; QA_BAD="" qa_fs_steps; ok=$? left="" if [ "$QA_WROTE" -eq 1 ]; then # The plan leaves both files; removed so the board is as it was found. send "fs rm /lfs1/atest" 10; send "fs rm /lfs1/test1" 10 if send "fs ls /lfs1"; then left=$(grep -x -E 'atest|test1' "$W/reply" | tr '\n' ' ' | sed 's/ $//') else left="unknown (no listing of /lfs1)"; fi fi if [ "$ok" -ne 0 ]; then qa_result qa.fs FAIL 3.6 "$QA_BAD${left:+; left on the board: $left}" elif [ -n "$left" ]; then qa_result qa.fs FAIL 3.6 "the steps passed, but left on the board: $left" else qa_result qa.fs PASS 3.6 "lfs1 mounted; erase-write and read tests ran; test1 written, truncated, rewritten and read back; test1 and the 300-byte atest kept across a reset; both removed" fi } run_qa() { if [ -n "$qa_version" ]; then QAV=$qa_version; qav_from="--qa-version" else prov=$FW_DIR/${VERSION%% *}/PROVENANCE exp=$(sed -n 's/^expect: *//p' "$prov" 2>/dev/null | sed -n 1p) rel=$(sed -n 's/^release: *$[^ ,]*$.*/\1/p' "$prov" 2>/dev/null | sed -n 1p) typ=$(sed -n 's/^origin:.*_$[a-z]*$\.hex$/\1/p' "$prov" 2>/dev/null | sed -n 1p) QAV="" if [ -n "$exp" ] && [ -n "$rel" ] && [ -n "$typ" ]; then case "$VERSION" in *"$exp"*) QAV=$rel-$typ; qav_from="the PROVENANCE in $FW_DIR/${VERSION%% *}" ;; esac fi [ -n "$QAV" ] || { QAV=$(echo "$VERSION" | sed -n 's/^[^ ]* v$[^ ]*$.*/\1/p'); qav_from="the board's \`version\`"; } fi if [ -z "$QAV" ]; then qa_result qa.console FAIL 3.1 "no version to expect: \`version\` gave none; give --qa-version" for c in qa.version:3.2 qa.stacks:3.4 qa.fs:3.6; do qa_result "${c%%:*}" SKIP "${c#*:}" "no version to expect"; done return 0 fi say "qa: expects asset_tracker_v2 v$QAV (from $qav_from)" QA_MODE=$reset_mode if [ -z "$QA_MODE" ]; then QA_MODE=swd if has kernel && send kernel && grep -q '^ *reboot *:' "$W/reply"; then QA_MODE=shell; fi fi say "qa: resets over $QA_MODE" qa_console qa_version qa_stacks qa_fs } # --- run ------------------------------------------------------------------------------------ CFUN0="" want zephyr && run_zephyr want at && run_at [ "$quick" -eq 0 ] && want lte && run_lte [ "$writes" -eq 1 ] && [ "$quick" -eq 0 ] && run_writes want qa && run_qa # End of run: still the same boot, and no fault on the console. The console is # scanned whether or not the last query was answered, from the last reset's boot # if the qa suite reset the board. last=none; u2="" if has kernel; then if send "kernel uptime"; then last=answered; u2=$(reply_val 'Uptime:' | sed 's/ ms.*//'); else last=unanswered; fi fi slice "${QA_SINCE:-0}" >"$W/all" fault=$(grep -E '\*\*\* Booting|FATAL ERROR|>>> ZEPHYR FATAL|HardFault|ASSERTION FAIL' "$W/all" | sed -n 1p) if [ -n "$fault" ]; then result end.stable FAIL "the console shows a boot or a fault during the run${QA_SINCE:+ (since the last reset)}: $fault" elif [ "$last" = unanswered ]; then result end.stable FAIL "no reply to the last query (kernel uptime): the board hung or restarted at the end of the run" elif [ "$last" = answered ]; then if [ -n "$U0" ] && [ -n "$u2" ] && [ "$u2" -gt "$U0" ] 2>/dev/null; then nerr=$(grep -c '' "$W/all") result end.stable PASS "same boot throughout (up $((u2 / 1000)) s)$( [ "${nerr:-0}" -gt 0 ] && echo "; $nerr log lines, see transcript")" else result end.stable FAIL "uptime went from $U0 to ${u2:-?} ms: the board restarted during the run" fi fi # --- summary --------------------------------------------------------------------------------- line() { # SUITE LABEL p=$(count "$1" pass); f=$(count "$1" fail); s=$(count "$1" skip); x=$(count "$1" xfail) [ $((p + f + s + x)) -gt 0 ] || return 0 printf '%s: %-16s %3d pass / %d fail / %d skip%s\n' "$PROG" "$2" "$p" "$f" "$s" "$( [ "$x" -gt 0 ] && echo " / $x xfail")" } line rig "Rig" line zephyr "Zephyr app" line at "AT command set" line lte "LTE link" line qa "QA Test Plan" line end "End of run" tp=0; tf=0; ts=0; tx=0 for s in rig zephyr at lte qa end; do tp=$((tp + $(count $s pass))); tf=$((tf + $(count $s fail))) ts=$((ts + $(count $s skip))); tx=$((tx + $(count $s xfail))) done el=$(( $(now) - T0 )) summary="$tp pass, $tf fail, $ts skip$( [ "$tx" -gt 0 ] && echo ", $tx xfail") across $((tp + tf + ts + tx)) checks (profile $P_NAME, $((el / 60)) min $((el % 60)) s; $out)" if [ "$FAILS" -eq 0 ]; then say "PASS -- $summary" exit 0 fi say "FAIL -- $summary" exit 1