From 80236a682a50a49fa949e80ff00ff55460121078 Mon Sep 17 00:00:00 2001 From: Srikanth Muppandam Date: Mon, 9 Mar 2026 23:07:13 +0530 Subject: [PATCH 1/2] lib(module-reload): improve timeout handling and target-specific evidence capture Avoid wedging the test harness when unload or load commands time out by returning immediately after timeout evidence collection. Also narrow module state capture to the target module and improve timeout-path stdout reporting for easier debugging. Signed-off-by: Srikanth Muppandam --- Runner/utils/lib_module_reload.sh | 597 ++++++++++++++++++++++++++++++ 1 file changed, 597 insertions(+) create mode 100755 Runner/utils/lib_module_reload.sh diff --git a/Runner/utils/lib_module_reload.sh b/Runner/utils/lib_module_reload.sh new file mode 100755 index 00000000..b481441a --- /dev/null +++ b/Runner/utils/lib_module_reload.sh @@ -0,0 +1,597 @@ +#!/bin/sh +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear + +MRV_LAST_CMD_PID="" +MRV_LAST_CMD_ELAPSED="0" +MRV_LAST_TIMEOUT_DIR="" +MRV_PROFILE_REASON="" + +mrv_reset_profile_vars() { + PROFILE_NAME="" + PROFILE_DESCRIPTION="" + MODULE_NAME="" + MODULE_LOAD_CMD="" + MODULE_UNLOAD_CMD="" + MODULE_RELOAD_SUPPORTED="yes" + PROFILE_MODE_DEFAULT="basic" + PROFILE_REQUIRED_CMDS="" + PROFILE_SERVICES="" + PROFILE_DEVICE_PATTERNS="" + PROFILE_SYSFS_PATTERNS="" + PROFILE_PREPARE_HOOK="profile_prepare" + PROFILE_WARMUP_HOOK="profile_warmup" + PROFILE_QUIESCE_HOOK="profile_quiesce" + PROFILE_POST_UNLOAD_HOOK="profile_post_unload" + PROFILE_POST_LOAD_HOOK="profile_post_load" + PROFILE_SMOKE_HOOK="profile_smoke" + PROFILE_FINALIZE_HOOK="profile_finalize" + PROFILE_SELECTED_MODE="" +} + +mrv_module_lsmod_name() { + printf '%s\n' "$1" | tr '-' '_' +} + +mrv_module_loaded() { + name="$(mrv_module_lsmod_name "$1")" + lsmod 2>/dev/null | awk -v mod="$name" 'NR > 1 && $1 == mod { found=1; exit } END { exit !found }' +} + +mrv_module_builtin() { + if [ -f "/lib/modules/$(uname -r)/modules.builtin" ]; then + mod_pat="$(printf '%s\n' "$1" | sed 's/_/[-_]/g')" + grep -Eq "/${mod_pat}(\\.ko(\\.[^.]+)*)?$" "/lib/modules/$(uname -r)/modules.builtin" 2>/dev/null + return $? + fi + return 1 +} + +mrv_wait_module_state() { + want="$1" + name="$2" + timeout_s="$3" + elapsed=0 + + while [ "$elapsed" -lt "$timeout_s" ] 2>/dev/null; do + if [ "$want" = "present" ]; then + if mrv_module_loaded "$name"; then + return 0 + fi + else + if ! mrv_module_loaded "$name"; then + return 0 + fi + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + + if [ "$want" = "present" ]; then + mrv_module_loaded "$name" + return $? + fi + + if mrv_module_loaded "$name"; then + return 1 + fi + return 0 +} + +mrv_capture_patterns() { + patterns="$1" + outfile="$2" + + : > "$outfile" + + if [ -z "$patterns" ]; then + return 0 + fi + + # shellcheck disable=SC2086 + set -- $patterns + for pat in "$@"; do + found=0 + # shellcheck disable=SC2086 + for path in $pat; do + if [ -e "$path" ] || [ -L "$path" ]; then + found=1 + printf 'PATH: %s\n' "$path" >> "$outfile" + ls -ld "$path" >> "$outfile" 2>&1 || true + fi + done + if [ "$found" -eq 0 ] 2>/dev/null; then + printf 'PATH: %s (not present)\n' "$pat" >> "$outfile" + fi + done +} + +mrv_capture_services() { + services="$1" + outfile="$2" + + : > "$outfile" + + if [ -z "$services" ]; then + printf 'No profile services defined\n' >> "$outfile" + return 0 + fi + + if ! command -v systemctl >/dev/null 2>&1; then + printf 'systemctl not available\n' >> "$outfile" + return 0 + fi + + # shellcheck disable=SC2086 + set -- $services + for svc in "$@"; do + printf '===== %s =====\n' "$svc" >> "$outfile" + systemctl status "$svc" --no-pager >> "$outfile" 2>&1 || true + if command -v journalctl >/dev/null 2>&1; then + printf '\n----- journalctl -u %s -----\n' "$svc" >> "$outfile" + journalctl -u "$svc" --no-pager -n 200 >> "$outfile" 2>&1 || true + fi + done +} + +mrv_capture_module_state() { + outdir="$1" + mkdir -p "$outdir" + + { + printf 'profile=%s\n' "${PROFILE_NAME:-unknown}" + printf 'description=%s\n' "${PROFILE_DESCRIPTION:-unknown}" + printf 'module=%s\n' "${MODULE_NAME:-unknown}" + printf 'mode=%s\n' "${PROFILE_SELECTED_MODE:-unknown}" + printf 'kernel=%s\n' "$(uname -r)" + printf 'date=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || echo unknown)" + } > "$outdir/state.txt" + + sys_mod="$(mrv_module_lsmod_name "$MODULE_NAME")" + + { + printf 'Module-focused lsmod view for %s\n' "$sys_mod" + lsmod 2>/dev/null | awk -v mod="$sys_mod" ' + NR == 1 { print; next } + $1 == mod { found=1; print } + END { + if (!found) { + printf "module %s not present in lsmod\n", mod + } + } + ' + } > "$outdir/lsmod.log" 2>&1 || true + + modinfo "$MODULE_NAME" > "$outdir/modinfo.log" 2>&1 || true + ps -ef > "$outdir/ps.log" 2>&1 || true + dmesg > "$outdir/dmesg.log" 2>&1 || true + + if [ -d "/sys/module/$sys_mod" ]; then + find "/sys/module/$sys_mod" -maxdepth 3 -print > "$outdir/sys_module_tree.log" 2>&1 || true + if [ -d "/sys/module/$sys_mod/holders" ]; then + ls -la "/sys/module/$sys_mod/holders" > "$outdir/holders.log" 2>&1 || true + fi + else + printf '/sys/module/%s not present\n' "$sys_mod" > "$outdir/holders.log" + fi + + mrv_capture_services "$PROFILE_SERVICES" "$outdir/services.log" + mrv_capture_patterns "$PROFILE_DEVICE_PATTERNS" "$outdir/devices.log" + mrv_capture_patterns "$PROFILE_SYSFS_PATTERNS" "$outdir/profile_sysfs.log" + + if command -v journalctl >/dev/null 2>&1; then + journalctl -b --no-pager -n 300 > "$outdir/journal.log" 2>&1 || true + fi +} + +mrv_capture_pid_timeout_snapshot() { + pid="$1" + outdir="$2" + + mkdir -p "$outdir" + + if [ -d "/proc/$pid" ]; then + cat "/proc/$pid/status" > "$outdir/proc_status.log" 2>&1 || true + cat "/proc/$pid/wchan" > "$outdir/proc_wchan.log" 2>&1 || true + cat "/proc/$pid/stack" > "$outdir/proc_stack.log" 2>&1 || true + ps -T -p "$pid" > "$outdir/ps_threads.log" 2>&1 || true + fi +} + +mrv_exec_with_timeout() { + timeout_s="$1" + logfile="$2" + shift 2 + cmd="$*" + elapsed=0 + + : > "$logfile" + printf 'CMD: %s\n' "$cmd" >> "$logfile" + + sh -c "$cmd" >> "$logfile" 2>&1 & + cmd_pid=$! + MRV_LAST_CMD_PID="$cmd_pid" + MRV_LAST_CMD_ELAPSED=0 + + while kill -0 "$cmd_pid" 2>/dev/null; do + if [ "$elapsed" -ge "$timeout_s" ] 2>/dev/null; then + MRV_LAST_CMD_ELAPSED="$elapsed" + + printf 'TIMEOUT: command exceeded %ss (pid=%s)\n' "$timeout_s" "$cmd_pid" >> "$logfile" + + if [ -n "$MRV_LAST_TIMEOUT_DIR" ]; then + mrv_capture_pid_timeout_snapshot "$cmd_pid" "$MRV_LAST_TIMEOUT_DIR" + fi + + kill -TERM "$cmd_pid" 2>/dev/null || true + sleep 2 + + if kill -0 "$cmd_pid" 2>/dev/null; then + printf 'TIMEOUT: pid %s ignored SIGTERM, sending SIGKILL\n' "$cmd_pid" >> "$logfile" + kill -KILL "$cmd_pid" 2>/dev/null || true + sleep 1 + fi + + if kill -0 "$cmd_pid" 2>/dev/null; then + printf 'TIMEOUT: pid %s still present after SIGKILL (likely stuck in kernel)\n' "$cmd_pid" >> "$logfile" + fi + + return 124 + fi + + sleep 1 + elapsed=$((elapsed + 1)) + done + + wait "$cmd_pid" + rc=$? + MRV_LAST_CMD_ELAPSED="$elapsed" + return "$rc" +} + +mrv_capture_hang_evidence() { + phase="$1" + outdir="$2" + + mkdir -p "$outdir" + mrv_capture_module_state "$outdir" + + if [ "$ENABLE_SYSRQ_HANG_DUMP" -eq 1 ] 2>/dev/null; then + if [ -w /proc/sysrq-trigger ]; then + printf 'Triggered sysrq-t and sysrq-w\n' > "$outdir/sysrq_actions.log" + echo t > /proc/sysrq-trigger 2>/dev/null || true + echo w > /proc/sysrq-trigger 2>/dev/null || true + dmesg > "$outdir/dmesg_after_sysrq.log" 2>&1 || true + fi + fi + + if command -v fuser >/dev/null 2>&1; then + : > "$outdir/fuser.log" + # shellcheck disable=SC2086 + set -- $PROFILE_DEVICE_PATTERNS + for pat in "$@"; do + # shellcheck disable=SC2086 + for path in $pat; do + if [ -e "$path" ] || [ -L "$path" ]; then + printf '===== %s =====\n' "$path" >> "$outdir/fuser.log" + fuser -vm "$path" >> "$outdir/fuser.log" 2>&1 || true + fi + done + done + fi + + if command -v lsof >/dev/null 2>&1; then + lsof > "$outdir/lsof.log" 2>&1 || true + fi + + { + printf 'phase=%s\n' "$phase" + printf 'pid=%s\n' "$MRV_LAST_CMD_PID" + printf 'elapsed=%s\n' "$MRV_LAST_CMD_ELAPSED" + } > "$outdir/timeout_summary.log" +} + +mrv_run_hook() { + hook_name="$1" + shift + + if [ -n "$hook_name" ] && command -v "$hook_name" >/dev/null 2>&1; then + "$hook_name" "$@" + return $? + fi + + return 0 +} + +mrv_profile_check() { + requested_mode="$1" + + if [ -z "$PROFILE_NAME" ] || [ -z "$MODULE_NAME" ]; then + MRV_PROFILE_REASON="profile missing PROFILE_NAME or MODULE_NAME" + return 2 + fi + + if [ "${MODULE_RELOAD_SUPPORTED:-yes}" != "yes" ]; then + MRV_PROFILE_REASON="profile marks module as non-reloadable" + return 2 + fi + + if mrv_module_builtin "$MODULE_NAME"; then + MRV_PROFILE_REASON="module is built-in and cannot be unloaded" + return 2 + fi + + if ! modinfo "$MODULE_NAME" >/dev/null 2>&1; then + if ! mrv_module_loaded "$MODULE_NAME"; then + MRV_PROFILE_REASON="module not present on image" + return 2 + fi + fi + + if ! command -v modprobe >/dev/null 2>&1; then + MRV_PROFILE_REASON="modprobe not available" + return 2 + fi + + if ! command -v rmmod >/dev/null 2>&1; then + MRV_PROFILE_REASON="rmmod not available" + return 2 + fi + + required="${PROFILE_REQUIRED_CMDS:-}" + if [ -n "$required" ]; then + # shellcheck disable=SC2086 + set -- $required + for req in "$@"; do + if ! command -v "$req" >/dev/null 2>&1; then + MRV_PROFILE_REASON="required command missing: $req" + return 2 + fi + done + fi + + MODULE_LOAD_CMD="${MODULE_LOAD_CMD:-modprobe $MODULE_NAME}" + MODULE_UNLOAD_CMD="${MODULE_UNLOAD_CMD:-rmmod $MODULE_NAME}" + PROFILE_SELECTED_MODE="${requested_mode:-${PROFILE_MODE_DEFAULT:-basic}}" + + return 0 +} + +mrv_post_unload_validate() { + if ! mrv_wait_module_state absent "$MODULE_NAME" "$TIMEOUT_SETTLE"; then + log_fail "[$PROFILE_NAME] module still present after unload settle timeout" + return 1 + fi + return 0 +} + +mrv_post_load_validate() { + if ! mrv_wait_module_state present "$MODULE_NAME" "$TIMEOUT_SETTLE"; then + log_fail "[$PROFILE_NAME] module did not reappear after load settle timeout" + return 1 + fi + return 0 +} + +mrv_run_iteration() { + iter="$1" + iter_dir="$RESULT_ROOT/$PROFILE_NAME/iter_$(printf '%02d' "$iter")" + preload_log="$iter_dir/preload.log" + unload_log="$iter_dir/unload.log" + load_log="$iter_dir/load.log" + unload_elapsed=0 + load_elapsed=0 + + mkdir -p "$iter_dir" + + if ! mrv_module_loaded "$MODULE_NAME"; then + log_info "[$PROFILE_NAME] module not loaded before iteration $iter, creating baseline loaded state" + MRV_LAST_TIMEOUT_DIR="$iter_dir/preload_timeout" + mrv_exec_with_timeout "$TIMEOUT_LOAD" "$preload_log" "$MODULE_LOAD_CMD" + preload_rc=$? + if [ "$preload_rc" -ne 0 ]; then + log_fail "[$PROFILE_NAME] failed to create baseline loaded state before iteration $iter (rc=$preload_rc)" + log_info "[$PROFILE_NAME] preload log: $preload_log" + return 1 + fi + if ! mrv_post_load_validate; then + log_info "[$PROFILE_NAME] baseline post-load validation failed" + return 1 + fi + fi + + if ! mrv_run_hook "$PROFILE_WARMUP_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] warmup hook failed in iteration $iter" + return 1 + fi + + mrv_capture_module_state "$iter_dir/pre_state" + + if ! mrv_run_hook "$PROFILE_QUIESCE_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] quiesce hook failed in iteration $iter" + return 1 + fi + + log_info "[$PROFILE_NAME] iter $iter/$ITERATIONS exec: $MODULE_UNLOAD_CMD" + MRV_LAST_TIMEOUT_DIR="$iter_dir/unload_timeout" + mrv_exec_with_timeout "$TIMEOUT_UNLOAD" "$unload_log" "$MODULE_UNLOAD_CMD" + unload_rc=$? + unload_elapsed="$MRV_LAST_CMD_ELAPSED" + + if [ "$unload_rc" -eq 124 ]; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS unload timed out after ${unload_elapsed}s (pid=$MRV_LAST_CMD_PID)" + log_info "[$PROFILE_NAME] unload timeout log: $unload_log" + log_info "[$PROFILE_NAME] collecting hang evidence in: $iter_dir/hang_evidence" + mrv_capture_hang_evidence unload "$iter_dir/hang_evidence" + log_fail "[$PROFILE_NAME] hang evidence captured; failing iteration $iter/$ITERATIONS" + return 1 + fi + + if [ "$unload_rc" -ne 0 ]; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS unload failed (rc=$unload_rc)" + log_info "[$PROFILE_NAME] unload log: $unload_log" + mrv_capture_module_state "$iter_dir/unload_failure_state" + return 1 + fi + + if ! mrv_post_unload_validate; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS post-unload validation failed" + mrv_capture_module_state "$iter_dir/post_unload_invalid_state" + return 1 + fi + + mrv_capture_module_state "$iter_dir/post_unload_state" + + if ! mrv_run_hook "$PROFILE_POST_UNLOAD_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] post-unload hook failed in iteration $iter" + return 1 + fi + + log_info "[$PROFILE_NAME] iter $iter/$ITERATIONS exec: $MODULE_LOAD_CMD" + MRV_LAST_TIMEOUT_DIR="$iter_dir/load_timeout" + mrv_exec_with_timeout "$TIMEOUT_LOAD" "$load_log" "$MODULE_LOAD_CMD" + load_rc=$? + load_elapsed="$MRV_LAST_CMD_ELAPSED" + + if [ "$load_rc" -eq 124 ]; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS load timed out after ${load_elapsed}s (pid=$MRV_LAST_CMD_PID)" + log_info "[$PROFILE_NAME] load timeout log: $load_log" + log_info "[$PROFILE_NAME] collecting hang evidence in: $iter_dir/load_hang_evidence" + mrv_capture_hang_evidence load "$iter_dir/load_hang_evidence" + log_fail "[$PROFILE_NAME] hang evidence captured, failing iteration $iter/$ITERATIONS" + return 1 + fi + + if [ "$load_rc" -ne 0 ]; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS load failed (rc=$load_rc)" + log_info "[$PROFILE_NAME] load log: $load_log" + mrv_capture_module_state "$iter_dir/load_failure_state" + return 1 + fi + + if ! mrv_post_load_validate; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS post-load validation failed" + mrv_capture_module_state "$iter_dir/post_load_invalid_state" + return 1 + fi + + if ! mrv_run_hook "$PROFILE_POST_LOAD_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] post-load hook failed in iteration $iter" + mrv_capture_module_state "$iter_dir/post_load_hook_failure_state" + return 1 + fi + + if ! mrv_run_hook "$PROFILE_SMOKE_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] smoke hook failed in iteration $iter" + mrv_capture_module_state "$iter_dir/smoke_failure_state" + return 1 + fi + + mrv_capture_module_state "$iter_dir/post_load_state" + + if command -v diff >/dev/null 2>&1; then + diff -u "$iter_dir/pre_state/dmesg.log" "$iter_dir/post_load_state/dmesg.log" > "$iter_dir/dmesg.diff" 2>&1 || true + fi + + { + printf 'iter=%s\n' "$iter" + printf 'unload_elapsed=%s\n' "$unload_elapsed" + printf 'load_elapsed=%s\n' "$load_elapsed" + } > "$iter_dir/iteration_metrics.log" + + log_pass "[$PROFILE_NAME] iteration $iter/$ITERATIONS passed" + return 0 +} + +mrv_run_one_profile() ( + profile_file="$1" + requested_mode="$2" + + mrv_reset_profile_vars + + # shellcheck disable=SC1090 + . "$profile_file" + + mrv_profile_check "$requested_mode" + check_rc=$? + if [ "$check_rc" -eq 2 ]; then + log_skip "[$(basename "$profile_file" .profile)] SKIP - $MRV_PROFILE_REASON" + exit 2 + fi + if [ "$check_rc" -ne 0 ]; then + log_fail "[$(basename "$profile_file" .profile)] FAIL - profile validation error" + exit 1 + fi + + profile_root="$RESULT_ROOT/$PROFILE_NAME" + mkdir -p "$profile_root" + + { + printf 'profile=%s\n' "$PROFILE_NAME" + printf 'description=%s\n' "${PROFILE_DESCRIPTION:-unknown}" + printf 'module=%s\n' "$MODULE_NAME" + printf 'mode=%s\n' "$PROFILE_SELECTED_MODE" + } > "$profile_root/profile_info.txt" + + log_info "[$PROFILE_NAME] module=$MODULE_NAME mode=$PROFILE_SELECTED_MODE iterations=$ITERATIONS" + + if ! mrv_run_hook "$PROFILE_PREPARE_HOOK" "$profile_root"; then + log_fail "[$PROFILE_NAME] prepare hook failed" + exit 1 + fi + + iter=1 + while [ "$iter" -le "$ITERATIONS" ] 2>/dev/null; do + if ! mrv_run_iteration "$iter"; then + mrv_run_hook "$PROFILE_FINALIZE_HOOK" "$profile_root" >/dev/null 2>&1 || true + exit 1 + fi + iter=$((iter + 1)) + done + + if ! mrv_run_hook "$PROFILE_FINALIZE_HOOK" "$profile_root"; then + log_fail "[$PROFILE_NAME] finalize hook failed" + exit 1 + fi + + exit 0 +) + +mrv_resolve_profiles() { + target_module="$1" + profile_dir="$2" + profile_list_file="$3" + + if [ -n "$target_module" ]; then + candidate="$profile_dir/$target_module.profile" + if [ ! -f "$candidate" ]; then + log_error "Profile not found: $candidate" + return 1 + fi + printf '%s\n' "$candidate" + return 0 + fi + + if [ -f "$profile_list_file" ]; then + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|'#'*) continue ;; + esac + candidate="$profile_dir/$line.profile" + if [ -f "$candidate" ]; then + printf '%s\n' "$candidate" + else + log_warn "Enabled profile listed but missing: $candidate" + fi + done < "$profile_list_file" + return 0 + fi + + for candidate in "$profile_dir"/*.profile; do + if [ -f "$candidate" ]; then + printf '%s\n' "$candidate" + fi + done + + return 0 +} From a7ba5d893be635d08f2097e4f1f478eae7c1b69c Mon Sep 17 00:00:00 2001 From: Srikanth Muppandam Date: Mon, 9 Mar 2026 23:07:34 +0530 Subject: [PATCH 2/2] test(module-reload): harden fastrpc profile daemon quiesce flow Update the fastrpc profile to stop and mask rpc daemons, verify remaining daemon processes, and provide clearer stdout context when quiesce or unload paths fail. Signed-off-by: Srikanth Muppandam --- .../Module_Reload_Validation.yaml | 28 ++ .../Module_Reload_Validation_README.md | 227 +++++++++ .../profiles/fastrpc.profile | 438 ++++++++++++++++++ .../Baseport/Module_Reload_Validation/run.sh | 229 +++++++++ 4 files changed, 922 insertions(+) create mode 100755 Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation.yaml create mode 100644 Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation_README.md create mode 100755 Runner/suites/Kernel/Baseport/Module_Reload_Validation/profiles/fastrpc.profile create mode 100755 Runner/suites/Kernel/Baseport/Module_Reload_Validation/run.sh diff --git a/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation.yaml b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation.yaml new file mode 100755 index 00000000..b98d8170 --- /dev/null +++ b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation.yaml @@ -0,0 +1,28 @@ +metadata: + name: Module_Reload_Validation + format: "Lava-Test Test Definition 1.0" + description: "Generic profiled kernel module unload/reload regression validation" + maintainer: + - srikanth kumar + os: + - linux + scope: + - functional + +params: + PROFILE: "" + ITERATIONS: "3" + MODE: "" + TIMEOUT_UNLOAD: "30" + TIMEOUT_LOAD: "30" + TIMEOUT_SETTLE: "20" + ENABLE_SYSRQ_HANG_DUMP: "1" + +run: + steps: + - REPO_PATH=$PWD + - cd Runner/suites/Kernel/Baseport/Module_Reload_Validation + - SYSRQ_ARG="" + - if [ "${ENABLE_SYSRQ_HANG_DUMP}" = "0" ]; then SYSRQ_ARG="--disable-sysrq-hang-dump"; fi + - ./run.sh --module "${PROFILE}" --iterations "${ITERATIONS}" --mode "${MODE}" --timeout-unload "${TIMEOUT_UNLOAD}" --timeout-load "${TIMEOUT_LOAD}" --timeout-settle "${TIMEOUT_SETTLE}" ${SYSRQ_ARG} || true + - $REPO_PATH/Runner/utils/send-to-lava.sh Module_Reload_Validation.res diff --git a/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation_README.md b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation_README.md new file mode 100644 index 00000000..98d7ab90 --- /dev/null +++ b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation_README.md @@ -0,0 +1,227 @@ +# Module_Reload_Validation + +## Overview +`Module_Reload_Validation` is a generic, profile-driven kernel module unload/reload regression suite. + +It is intended to catch issues such as: +- module unload hangs, +- failed reloads, +- service/device rebind regressions after reload, +- issues that reproduce on the 1st, 2nd, or later reload iteration. + +The suite uses: +- a **generic engine** in `run.sh`, +- shared helper logic in `Runner/utils/lib_module_reload.sh`, +- **module-specific profiles** under `profiles/`. + +## Folder layout + +```text +Runner/suites/Kernel/Baseport/Module_Reload_Validation/ +├── run.sh +├── Module_Reload_Validation.yaml +├── profiles/ +│ ├── enabled.list +│ └── fastrpc.profile + +Runner/utils/ +└── lib_module_reload.sh +``` + +## Main components + +### `run.sh` +Thin orchestration layer that: +- parses CLI arguments, +- resolves the selected profile(s), +- invokes the generic library engine, +- writes `Module_Reload_Validation.res`. + +### `lib_module_reload.sh` +Shared module reload engine that handles: +- module state checks, +- timeout-controlled unload/load execution, +- per-iteration evidence collection, +- timeout-path hang evidence, +- profile hook dispatch, +- result handling. + +### `profiles/*.profile` +Each profile provides module-specific metadata and optional hook logic. + +Example profile fields: +- `PROFILE_NAME` +- `PROFILE_DESCRIPTION` +- `MODULE_NAME` +- `PROFILE_MODE_DEFAULT` +- `PROFILE_REQUIRED_CMDS` +- `PROFILE_SERVICES` +- `PROFILE_DEVICE_PATTERNS` +- `PROFILE_SYSFS_PATTERNS` + +Optional hooks: +- `profile_prepare` +- `profile_warmup` +- `profile_quiesce` +- `profile_post_unload` +- `profile_post_load` +- `profile_smoke` +- `profile_finalize` + +## Current starter profile + +### `fastrpc.profile` +Current profile covers FastRPC unload/reload validation and supports service lifecycle-based testing. + +Default mode: +- `daemon_lifecycle` + +Relevant services: +- `adsprpcd.service` +- `cdsprpcd.service` + +## Execution flow +For each selected profile: + +1. Validate the profile. +2. Ensure the module is loaded before starting iteration work. +3. Run warmup hook. +4. Capture pre-state logs. +5. Run quiesce hook. +6. Attempt module unload with timeout. +7. Validate module absence. +8. Run post-unload hook. +9. Attempt module reload with timeout. +10. Validate module presence. +11. Run post-load hook. +12. Run smoke hook. +13. Capture post-load state. +14. Repeat for all iterations. +15. Run finalize hook. + +## Hang handling policy +Sysrq dump is **not** triggered on normal passing iterations. + +It is triggered only when an unload/load action actually times out. + +Current behavior: +- normal pass -> no sysrq dump +- quick non-timeout failure -> normal failure evidence only +- timeout / hang -> hang evidence bundle + optional sysrq dump + +Default behavior in current suite: +- sysrq hang dump enabled +- but only used on timeout paths + +## Evidence collected +Per profile / iteration, the suite can capture: +- command logs, +- `lsmod`, +- `modinfo`, +- `ps`, +- `dmesg`, +- service status and recent journal, +- profiled device path presence, +- profiled sysfs path presence, +- `/sys/module//holders`, +- timeout PID `/proc` snapshots, +- optional sysrq task/block dumps. + +Results are stored under: + +```text +results/Module_Reload_Validation//iter_XX/ +``` + +## CLI usage + +### Run one profile +```sh +./run.sh --module fastrpc +``` + +### Run one profile with more iterations +```sh +./run.sh --module fastrpc --iterations 5 +``` + +### Override mode +```sh +./run.sh --module fastrpc --mode daemon_lifecycle +./run.sh --module fastrpc --mode basic +``` + +### Override timeouts +```sh +./run.sh --module fastrpc --timeout-unload 60 --timeout-load 60 --timeout-settle 30 +``` + +### Disable sysrq timeout-path dumps +```sh +./run.sh --module fastrpc --disable-sysrq-hang-dump +``` + +### Run all enabled profiles +```sh +./run.sh +``` + +If `--module` is empty or not given, the suite runs all profiles listed in `profiles/enabled.list`. + +If `--mode` is empty or not given, the profile default mode is used. + +## YAML usage in LAVA +Current YAML is generic and takes profile input from the test plan. + +Important params: +- `PROFILE` +- `ITERATIONS` +- `MODE` +- `TIMEOUT_UNLOAD` +- `TIMEOUT_LOAD` +- `TIMEOUT_SETTLE` +- `ENABLE_SYSRQ_HANG_DUMP` + +Behavior: +- `PROFILE="fastrpc"` -> runs only `fastrpc.profile` +- `PROFILE=""` -> runs all enabled profiles +- `MODE=""` -> uses the profile default mode + +## How to add a new profile +1. Create a new file under `profiles/`, for example: + - `profiles/ath11k_pci.profile` +2. Define the required metadata: + - `PROFILE_NAME` + - `MODULE_NAME` +3. Add hooks only if module-specific lifecycle handling is needed. +4. Add the profile basename to `profiles/enabled.list`. +5. Run locally with: + +```sh +./run.sh --module ath11k_pci +``` + +No YAML duplication is needed for new profiles. + +## Result policy + +### PASS +- all requested iterations for a profile pass successfully. + +### FAIL +- unload/load timeout, +- unload/load command failure, +- module state validation failure, +- profile hook failure, +- smoke validation failure. + +### SKIP +- module not present, +- module built into the kernel, +- required commands not available, +- profile explicitly not reloadable. + +## Notes +- This suite is intended for **profiled, supported modules**, not blind reload of every loaded kernel module. +- The current structure avoids hidden run.sh-to-library globals as much as possible by passing explicit arguments and hook context. +- Profile hooks receive context arguments from the engine, so module-specific logic can store logs in the correct iteration directory without depending on hidden globals. diff --git a/Runner/suites/Kernel/Baseport/Module_Reload_Validation/profiles/fastrpc.profile b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/profiles/fastrpc.profile new file mode 100755 index 00000000..6dbd2ce0 --- /dev/null +++ b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/profiles/fastrpc.profile @@ -0,0 +1,438 @@ +PROFILE_NAME="fastrpc" +PROFILE_DESCRIPTION="FastRPC module reload validation" +MODULE_NAME="fastrpc" + +MODULE_RELOAD_SUPPORTED="yes" +PROFILE_MODE_DEFAULT="daemon_lifecycle" + +PROFILE_REQUIRED_CMDS="modprobe rmmod systemctl ps sed" +PROFILE_SERVICES="adsprpcd.service cdsprpcd.service" +PROFILE_DEVICE_PATTERNS="/dev/fastrpc-* /dev/*dsp*" +PROFILE_SYSFS_PATTERNS="/sys/module/fastrpc /sys/class/remoteproc/*" + +profile_fastrpc_service_known() { + svc="$1" + + if ! command -v systemctl >/dev/null 2>&1; then + return 1 + fi + + systemctl list-unit-files "$svc" >/dev/null 2>&1 +} + +profile_fastrpc_list_pids() { + proc_pat="$1" + + ps -eo pid=,args= 2>/dev/null | while IFS= read -r line; do + line_trim="$(printf '%s\n' "$line" | sed 's/^[[:space:]]*//')" + [ -n "$line_trim" ] || continue + + pid="${line_trim%% *}" + cmd="${line_trim#"$pid"}" + cmd="$(printf '%s\n' "$cmd" | sed 's/^[[:space:]]*//')" + + [ -n "$pid" ] || continue + [ "$pid" = "$$" ] && continue + + case "$cmd" in + *"$proc_pat"*) + printf '%s\n' "$pid" + ;; + esac + done +} + +profile_fastrpc_proc_running() { + proc_pat="$1" + pids="$(profile_fastrpc_list_pids "$proc_pat")" + + if [ -n "$pids" ]; then + return 0 + fi + return 1 +} + +profile_fastrpc_signal_pattern() { + proc_pat="$1" + sig="$2" + pids="$(profile_fastrpc_list_pids "$proc_pat")" + + for pid in $pids; do + if [ -n "$pid" ] && [ "$pid" != "$$" ]; then + log_info "[fastrpc] sending SIG${sig} to pid=$pid pattern=$proc_pat" + kill "-$sig" "$pid" >/dev/null 2>&1 || true + fi + done + + return 0 +} + +profile_fastrpc_kill_remaining_procs() { + sig="$1" + + profile_fastrpc_signal_pattern "/usr/bin/adsprpcd" "$sig" + profile_fastrpc_signal_pattern "/usr/bin/cdsprpcd" "$sig" + + return 0 +} + +profile_fastrpc_wait_no_procs() { + timeout_s="${1:-10}" + elapsed=0 + + while [ "$elapsed" -lt "$timeout_s" ] 2>/dev/null; do + if profile_fastrpc_proc_running "/usr/bin/adsprpcd"; then + : + elif profile_fastrpc_proc_running "/usr/bin/cdsprpcd"; then + : + else + return 0 + fi + + sleep 1 + elapsed=$((elapsed + 1)) + done + + if profile_fastrpc_proc_running "/usr/bin/adsprpcd"; then + return 1 + fi + if profile_fastrpc_proc_running "/usr/bin/cdsprpcd"; then + return 1 + fi + + return 0 +} + +profile_fastrpc_service_state_value() { + svc="$1" + prop="$2" + + val="$(systemctl show -p "$prop" --value "$svc" 2>/dev/null || true)" + if [ -n "$val" ]; then + printf '%s\n' "$val" + else + printf 'unknown\n' + fi +} + +profile_fastrpc_log_service_process_summary() { + if command -v systemctl >/dev/null 2>&1; then + if profile_fastrpc_service_known "adsprpcd.service"; then + adsp_active="$(profile_fastrpc_service_state_value "adsprpcd.service" "ActiveState")" + adsp_sub="$(profile_fastrpc_service_state_value "adsprpcd.service" "SubState")" + adsp_pid="$(profile_fastrpc_service_state_value "adsprpcd.service" "MainPID")" + log_info "[fastrpc] adsprpcd.service state=${adsp_active}/${adsp_sub} mainpid=${adsp_pid}" + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + cdsp_active="$(profile_fastrpc_service_state_value "cdsprpcd.service" "ActiveState")" + cdsp_sub="$(profile_fastrpc_service_state_value "cdsprpcd.service" "SubState")" + cdsp_pid="$(profile_fastrpc_service_state_value "cdsprpcd.service" "MainPID")" + log_info "[fastrpc] cdsprpcd.service state=${cdsp_active}/${cdsp_sub} mainpid=${cdsp_pid}" + fi + fi + + ps -eo pid=,args= 2>/dev/null | while IFS= read -r line; do + line_trim="$(printf '%s\n' "$line" | sed 's/^[[:space:]]*//')" + [ -n "$line_trim" ] || continue + + pid="${line_trim%% *}" + cmd="${line_trim#"$pid"}" + cmd="$(printf '%s\n' "$cmd" | sed 's/^[[:space:]]*//')" + + [ -n "$pid" ] || continue + [ "$pid" = "$$" ] && continue + + case "$cmd" in + *"/usr/bin/adsprpcd"*|*"/usr/bin/cdsprpcd"*) + log_info "[fastrpc] proc pid=$pid cmd=$cmd" + ;; + esac + done +} + +profile_fastrpc_wait_services_active() { + timeout_s="${1:-15}" + elapsed=0 + + while [ "$elapsed" -lt "$timeout_s" ] 2>/dev/null; do + adsp_ok=1 + cdsp_ok=1 + + if profile_fastrpc_service_known "adsprpcd.service"; then + if ! systemctl is-active --quiet adsprpcd.service; then + adsp_ok=0 + fi + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + if ! systemctl is-active --quiet cdsprpcd.service; then + cdsp_ok=0 + fi + fi + + if [ "$adsp_ok" -eq 1 ] && [ "$cdsp_ok" -eq 1 ]; then + return 0 + fi + + sleep 1 + elapsed=$((elapsed + 1)) + done + + adsp_ok=1 + cdsp_ok=1 + + if profile_fastrpc_service_known "adsprpcd.service"; then + if ! systemctl is-active --quiet adsprpcd.service; then + adsp_ok=0 + fi + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + if ! systemctl is-active --quiet cdsprpcd.service; then + cdsp_ok=0 + fi + fi + + if [ "$adsp_ok" -eq 1 ] && [ "$cdsp_ok" -eq 1 ]; then + return 0 + fi + + return 1 +} + +profile_fastrpc_wait_services_inactive() { + timeout_s="${1:-15}" + elapsed=0 + + while [ "$elapsed" -lt "$timeout_s" ] 2>/dev/null; do + adsp_ok=1 + cdsp_ok=1 + + if profile_fastrpc_service_known "adsprpcd.service"; then + if systemctl is-active --quiet adsprpcd.service; then + adsp_ok=0 + fi + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + if systemctl is-active --quiet cdsprpcd.service; then + cdsp_ok=0 + fi + fi + + if [ "$adsp_ok" -eq 1 ] && [ "$cdsp_ok" -eq 1 ]; then + if profile_fastrpc_wait_no_procs 2; then + return 0 + fi + fi + + sleep 1 + elapsed=$((elapsed + 1)) + done + + adsp_ok=1 + cdsp_ok=1 + + if profile_fastrpc_service_known "adsprpcd.service"; then + if systemctl is-active --quiet adsprpcd.service; then + adsp_ok=0 + fi + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + if systemctl is-active --quiet cdsprpcd.service; then + cdsp_ok=0 + fi + fi + + if [ "$adsp_ok" -eq 1 ] && [ "$cdsp_ok" -eq 1 ]; then + if profile_fastrpc_wait_no_procs 1; then + return 0 + fi + fi + + return 1 +} + +profile_prepare() { + return 0 +} + +profile_warmup() { + iter_dir="$1" + : "${iter_dir:=}" + + if ! command -v systemctl >/dev/null 2>&1; then + return 0 + fi + + case "$PROFILE_SELECTED_MODE" in + basic) + return 0 + ;; + daemon_lifecycle|service_rebind) + if profile_fastrpc_service_known "adsprpcd.service"; then + log_info "[fastrpc] warmup: unmasking and starting adsprpcd.service" + systemctl unmask adsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed adsprpcd.service >/dev/null 2>&1 || true + systemctl start adsprpcd.service >/dev/null 2>&1 || true + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + log_info "[fastrpc] warmup: unmasking and starting cdsprpcd.service" + systemctl unmask cdsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed cdsprpcd.service >/dev/null 2>&1 || true + systemctl start cdsprpcd.service >/dev/null 2>&1 || true + fi + + if ! profile_fastrpc_wait_services_active 15; then + profile_fastrpc_log_service_process_summary + log_error "[fastrpc] warmup: services failed to become active" + return 1 + fi + ;; + *) + log_warn "[fastrpc] unknown mode '$PROFILE_SELECTED_MODE', continuing without warmup actions" + ;; + esac + + return 0 +} + +profile_quiesce() { + iter_dir="$1" + : "${iter_dir:=}" + + if ! command -v systemctl >/dev/null 2>&1; then + return 0 + fi + + case "$PROFILE_SELECTED_MODE" in + basic) + return 0 + ;; + daemon_lifecycle|service_rebind) + if profile_fastrpc_service_known "adsprpcd.service"; then + log_info "[fastrpc] quiesce: stopping adsprpcd.service" + systemctl stop adsprpcd.service >/dev/null 2>&1 || true + log_info "[fastrpc] quiesce: masking adsprpcd.service" + systemctl mask adsprpcd.service >/dev/null 2>&1 || true + log_info "[fastrpc] quiesce: killing remaining adsprpcd.service cgroup processes" + systemctl kill --kill-who=all adsprpcd.service >/dev/null 2>&1 || true + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + log_info "[fastrpc] quiesce: stopping cdsprpcd.service" + systemctl stop cdsprpcd.service >/dev/null 2>&1 || true + log_info "[fastrpc] quiesce: masking cdsprpcd.service" + systemctl mask cdsprpcd.service >/dev/null 2>&1 || true + log_info "[fastrpc] quiesce: killing remaining cdsprpcd.service cgroup processes" + systemctl kill --kill-who=all cdsprpcd.service >/dev/null 2>&1 || true + fi + + profile_fastrpc_kill_remaining_procs TERM + sleep 2 + + if ! profile_fastrpc_wait_no_procs 5; then + log_warn "[fastrpc] quiesce: TERM was not enough, sending SIGKILL to remaining rpc daemons" + profile_fastrpc_kill_remaining_procs KILL + sleep 1 + fi + + if ! profile_fastrpc_wait_services_inactive 20; then + profile_fastrpc_log_service_process_summary + log_error "[fastrpc] quiesce: services/processes did not fully stop" + return 1 + fi + ;; + *) + log_warn "[fastrpc] unknown mode '$PROFILE_SELECTED_MODE', continuing without quiesce actions" + ;; + esac + + return 0 +} + +profile_post_unload() { + iter_dir="$1" + : "${iter_dir:=}" + return 0 +} + +profile_post_load() { + iter_dir="$1" + svc_log="$iter_dir/post_load_services.log" + : > "$svc_log" + + if ! command -v systemctl >/dev/null 2>&1; then + return 0 + fi + + case "$PROFILE_SELECTED_MODE" in + basic) + return 0 + ;; + daemon_lifecycle|service_rebind) + if profile_fastrpc_service_known "adsprpcd.service"; then + log_info "[fastrpc] post-load: unmasking and starting adsprpcd.service" + systemctl unmask adsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed adsprpcd.service >/dev/null 2>&1 || true + systemctl start adsprpcd.service >/dev/null 2>&1 || true + printf '===== %s =====\n' "adsprpcd.service" >> "$svc_log" + systemctl status adsprpcd.service --no-pager >> "$svc_log" 2>&1 || true + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + log_info "[fastrpc] post-load: unmasking and starting cdsprpcd.service" + systemctl unmask cdsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed cdsprpcd.service >/dev/null 2>&1 || true + systemctl start cdsprpcd.service >/dev/null 2>&1 || true + printf '===== %s =====\n' "cdsprpcd.service" >> "$svc_log" + systemctl status cdsprpcd.service --no-pager >> "$svc_log" 2>&1 || true + fi + + if ! profile_fastrpc_wait_services_active 15; then + profile_fastrpc_log_service_process_summary + log_error "[fastrpc] post-load: services failed to become active after reload" + return 1 + fi + ;; + *) + log_warn "[fastrpc] unknown mode '$PROFILE_SELECTED_MODE', continuing without post-load actions" + ;; + esac + + return 0 +} + +profile_smoke() { + iter_dir="$1" + : "${iter_dir:=}" + return 0 +} + +profile_finalize() { + profile_root="$1" + : "${profile_root:=}" + + if ! command -v systemctl >/dev/null 2>&1; then + return 0 + fi + + if profile_fastrpc_service_known "adsprpcd.service"; then + log_info "[fastrpc] finalize: restoring adsprpcd.service" + systemctl unmask adsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed adsprpcd.service >/dev/null 2>&1 || true + systemctl start adsprpcd.service >/dev/null 2>&1 || true + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + log_info "[fastrpc] finalize: restoring cdsprpcd.service" + systemctl unmask cdsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed cdsprpcd.service >/dev/null 2>&1 || true + systemctl start cdsprpcd.service >/dev/null 2>&1 || true + fi + + return 0 +} diff --git a/Runner/suites/Kernel/Baseport/Module_Reload_Validation/run.sh b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/run.sh new file mode 100755 index 00000000..8fc4aa2f --- /dev/null +++ b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/run.sh @@ -0,0 +1,229 @@ +#!/bin/sh +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear + +SCRIPT_DIR="$( + cd "$(dirname "$0")" || exit 1 + pwd +)" + +INIT_ENV="" +SEARCH="$SCRIPT_DIR" +while [ "$SEARCH" != "/" ]; do + if [ -f "$SEARCH/init_env" ]; then + INIT_ENV="$SEARCH/init_env" + break + fi + SEARCH=$(dirname "$SEARCH") +done + +if [ -z "$INIT_ENV" ]; then + echo "[ERROR] init_env not found" >&2 + exit 1 +fi + +if [ -z "${__INIT_ENV_LOADED:-}" ]; then + # shellcheck disable=SC1090 + . "$INIT_ENV" + __INIT_ENV_LOADED=1 +fi + +# shellcheck disable=SC1091 +. "$TOOLS/functestlib.sh" +# shellcheck disable=SC1091 +. "$TOOLS/lib_module_reload.sh" + +TESTNAME="Module_Reload_Validation" +PROFILE_DIR_DEFAULT="$SCRIPT_DIR/profiles" +PROFILE_LIST_DEFAULT="$PROFILE_DIR_DEFAULT/enabled.list" + +TARGET_MODULE="" +ITERATIONS="3" +TIMEOUT_UNLOAD="30" +TIMEOUT_LOAD="30" +TIMEOUT_SETTLE="20" +ENABLE_SYSRQ_HANG_DUMP="1" +PROFILE_MODE="" +PROFILE_DIR="$PROFILE_DIR_DEFAULT" +PROFILE_LIST_FILE="$PROFILE_LIST_DEFAULT" +VERBOSE=0 + +usage() { + cat </dev/null; then + set -x +fi + +case "$ITERATIONS" in + ''|*[!0-9]*) log_error "Invalid --iterations: $ITERATIONS"; exit 1 ;; +esac +case "$TIMEOUT_UNLOAD" in + ''|*[!0-9]*) log_error "Invalid --timeout-unload: $TIMEOUT_UNLOAD"; exit 1 ;; +esac +case "$TIMEOUT_LOAD" in + ''|*[!0-9]*) log_error "Invalid --timeout-load: $TIMEOUT_LOAD"; exit 1 ;; +esac +case "$TIMEOUT_SETTLE" in + ''|*[!0-9]*) log_error "Invalid --timeout-settle: $TIMEOUT_SETTLE"; exit 1 ;; +esac + +test_path="$(find_test_case_by_name "$TESTNAME" 2>/dev/null || echo "$SCRIPT_DIR")" +if ! cd "$test_path"; then + log_error "cd failed: $test_path" + exit 1 +fi + +RES_FILE="$SCRIPT_DIR/${TESTNAME}.res" +RESULT_ROOT="$SCRIPT_DIR/results/$TESTNAME" +SUMMARY_FILE="$RESULT_ROOT/summary.txt" +mkdir -p "$RESULT_ROOT" +: > "$SUMMARY_FILE" + +log_info "---------------- Starting $TESTNAME ----------------" +if command -v detect_platform >/dev/null 2>&1; then + detect_platform >/dev/null 2>&1 || true + log_info "Platform Details: machine='${PLATFORM_MACHINE:-unknown}' target='${PLATFORM_TARGET:-unknown}' kernel='${PLATFORM_KERNEL:-}' arch='${PLATFORM_ARCH:-}'" +else + log_info "Platform Details: unknown" +fi + +log_info "Args: module='${TARGET_MODULE:-all-enabled}' iterations=$ITERATIONS unload_timeout=$TIMEOUT_UNLOAD load_timeout=$TIMEOUT_LOAD settle_timeout=$TIMEOUT_SETTLE mode='${PROFILE_MODE:-profile-default}' sysrq_dump=$ENABLE_SYSRQ_HANG_DUMP" + +if [ "$ENABLE_SYSRQ_HANG_DUMP" -eq 1 ] 2>/dev/null; then + log_info "Sysrq hang dump policy: enabled on timeout paths only" +else + log_info "Sysrq hang dump policy: disabled" +fi + +PROFILE_FILES="$(mrv_resolve_profiles "$TARGET_MODULE" "$PROFILE_DIR" "$PROFILE_LIST_FILE")" +resolve_rc=$? +if [ "$resolve_rc" -ne 0 ]; then + echo "$TESTNAME FAIL" > "$RES_FILE" + exit 1 +fi + +if [ -z "$PROFILE_FILES" ]; then + log_skip "$TESTNAME SKIP - no profiles selected" + echo "$TESTNAME SKIP" > "$RES_FILE" + exit 0 +fi + +PROFILE_TMP_LIST="$RESULT_ROOT/.profiles_to_run.list" +printf '%s\n' "$PROFILE_FILES" > "$PROFILE_TMP_LIST" + +pass_count=0 +fail_count=0 +skip_count=0 + +while IFS= read -r profile_path || [ -n "$profile_path" ]; do + [ -n "$profile_path" ] || continue + + mrv_run_one_profile "$profile_path" "$PROFILE_MODE" + rc=$? + base_name="$(basename "$profile_path" .profile)" + + if [ "$rc" -eq 0 ]; then + log_pass "[$base_name] profile PASS" + printf '%s PASS\n' "$base_name" >> "$SUMMARY_FILE" + pass_count=$((pass_count + 1)) + elif [ "$rc" -eq 2 ]; then + log_skip "[$base_name] profile SKIP" + printf '%s SKIP\n' "$base_name" >> "$SUMMARY_FILE" + skip_count=$((skip_count + 1)) + else + log_fail "[$base_name] profile FAIL" + printf '%s FAIL\n' "$base_name" >> "$SUMMARY_FILE" + fail_count=$((fail_count + 1)) + fi +done < "$PROFILE_TMP_LIST" + +log_info "Summary: pass=$pass_count fail=$fail_count skip=$skip_count" + +if [ "$fail_count" -gt 0 ] 2>/dev/null; then + log_fail "$TESTNAME FAIL" + echo "$TESTNAME FAIL" > "$RES_FILE" + exit 1 +fi + +if [ "$pass_count" -gt 0 ] 2>/dev/null; then + log_pass "$TESTNAME PASS" + echo "$TESTNAME PASS" > "$RES_FILE" + exit 0 +fi + +log_skip "$TESTNAME SKIP" +echo "$TESTNAME SKIP" > "$RES_FILE" +exit 0