diff --git a/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation.yaml b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation.yaml new file mode 100755 index 00000000..b98d8170 --- /dev/null +++ b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation.yaml @@ -0,0 +1,28 @@ +metadata: + name: Module_Reload_Validation + format: "Lava-Test Test Definition 1.0" + description: "Generic profiled kernel module unload/reload regression validation" + maintainer: + - srikanth kumar + os: + - linux + scope: + - functional + +params: + PROFILE: "" + ITERATIONS: "3" + MODE: "" + TIMEOUT_UNLOAD: "30" + TIMEOUT_LOAD: "30" + TIMEOUT_SETTLE: "20" + ENABLE_SYSRQ_HANG_DUMP: "1" + +run: + steps: + - REPO_PATH=$PWD + - cd Runner/suites/Kernel/Baseport/Module_Reload_Validation + - SYSRQ_ARG="" + - if [ "${ENABLE_SYSRQ_HANG_DUMP}" = "0" ]; then SYSRQ_ARG="--disable-sysrq-hang-dump"; fi + - ./run.sh --module "${PROFILE}" --iterations "${ITERATIONS}" --mode "${MODE}" --timeout-unload "${TIMEOUT_UNLOAD}" --timeout-load "${TIMEOUT_LOAD}" --timeout-settle "${TIMEOUT_SETTLE}" ${SYSRQ_ARG} || true + - $REPO_PATH/Runner/utils/send-to-lava.sh Module_Reload_Validation.res diff --git a/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation_README.md b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation_README.md new file mode 100644 index 00000000..98d7ab90 --- /dev/null +++ b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/Module_Reload_Validation_README.md @@ -0,0 +1,227 @@ +# Module_Reload_Validation + +## Overview +`Module_Reload_Validation` is a generic, profile-driven kernel module unload/reload regression suite. + +It is intended to catch issues such as: +- module unload hangs, +- failed reloads, +- service/device rebind regressions after reload, +- issues that reproduce on the 1st, 2nd, or later reload iteration. + +The suite uses: +- a **generic engine** in `run.sh`, +- shared helper logic in `Runner/utils/lib_module_reload.sh`, +- **module-specific profiles** under `profiles/`. + +## Folder layout + +```text +Runner/suites/Kernel/Baseport/Module_Reload_Validation/ +├── run.sh +├── Module_Reload_Validation.yaml +├── profiles/ +│ ├── enabled.list +│ └── fastrpc.profile + +Runner/utils/ +└── lib_module_reload.sh +``` + +## Main components + +### `run.sh` +Thin orchestration layer that: +- parses CLI arguments, +- resolves the selected profile(s), +- invokes the generic library engine, +- writes `Module_Reload_Validation.res`. + +### `lib_module_reload.sh` +Shared module reload engine that handles: +- module state checks, +- timeout-controlled unload/load execution, +- per-iteration evidence collection, +- timeout-path hang evidence, +- profile hook dispatch, +- result handling. + +### `profiles/*.profile` +Each profile provides module-specific metadata and optional hook logic. + +Example profile fields: +- `PROFILE_NAME` +- `PROFILE_DESCRIPTION` +- `MODULE_NAME` +- `PROFILE_MODE_DEFAULT` +- `PROFILE_REQUIRED_CMDS` +- `PROFILE_SERVICES` +- `PROFILE_DEVICE_PATTERNS` +- `PROFILE_SYSFS_PATTERNS` + +Optional hooks: +- `profile_prepare` +- `profile_warmup` +- `profile_quiesce` +- `profile_post_unload` +- `profile_post_load` +- `profile_smoke` +- `profile_finalize` + +## Current starter profile + +### `fastrpc.profile` +Current profile covers FastRPC unload/reload validation and supports service lifecycle-based testing. + +Default mode: +- `daemon_lifecycle` + +Relevant services: +- `adsprpcd.service` +- `cdsprpcd.service` + +## Execution flow +For each selected profile: + +1. Validate the profile. +2. Ensure the module is loaded before starting iteration work. +3. Run warmup hook. +4. Capture pre-state logs. +5. Run quiesce hook. +6. Attempt module unload with timeout. +7. Validate module absence. +8. Run post-unload hook. +9. Attempt module reload with timeout. +10. Validate module presence. +11. Run post-load hook. +12. Run smoke hook. +13. Capture post-load state. +14. Repeat for all iterations. +15. Run finalize hook. + +## Hang handling policy +Sysrq dump is **not** triggered on normal passing iterations. + +It is triggered only when an unload/load action actually times out. + +Current behavior: +- normal pass -> no sysrq dump +- quick non-timeout failure -> normal failure evidence only +- timeout / hang -> hang evidence bundle + optional sysrq dump + +Default behavior in current suite: +- sysrq hang dump enabled +- but only used on timeout paths + +## Evidence collected +Per profile / iteration, the suite can capture: +- command logs, +- `lsmod`, +- `modinfo`, +- `ps`, +- `dmesg`, +- service status and recent journal, +- profiled device path presence, +- profiled sysfs path presence, +- `/sys/module//holders`, +- timeout PID `/proc` snapshots, +- optional sysrq task/block dumps. + +Results are stored under: + +```text +results/Module_Reload_Validation//iter_XX/ +``` + +## CLI usage + +### Run one profile +```sh +./run.sh --module fastrpc +``` + +### Run one profile with more iterations +```sh +./run.sh --module fastrpc --iterations 5 +``` + +### Override mode +```sh +./run.sh --module fastrpc --mode daemon_lifecycle +./run.sh --module fastrpc --mode basic +``` + +### Override timeouts +```sh +./run.sh --module fastrpc --timeout-unload 60 --timeout-load 60 --timeout-settle 30 +``` + +### Disable sysrq timeout-path dumps +```sh +./run.sh --module fastrpc --disable-sysrq-hang-dump +``` + +### Run all enabled profiles +```sh +./run.sh +``` + +If `--module` is empty or not given, the suite runs all profiles listed in `profiles/enabled.list`. + +If `--mode` is empty or not given, the profile default mode is used. + +## YAML usage in LAVA +Current YAML is generic and takes profile input from the test plan. + +Important params: +- `PROFILE` +- `ITERATIONS` +- `MODE` +- `TIMEOUT_UNLOAD` +- `TIMEOUT_LOAD` +- `TIMEOUT_SETTLE` +- `ENABLE_SYSRQ_HANG_DUMP` + +Behavior: +- `PROFILE="fastrpc"` -> runs only `fastrpc.profile` +- `PROFILE=""` -> runs all enabled profiles +- `MODE=""` -> uses the profile default mode + +## How to add a new profile +1. Create a new file under `profiles/`, for example: + - `profiles/ath11k_pci.profile` +2. Define the required metadata: + - `PROFILE_NAME` + - `MODULE_NAME` +3. Add hooks only if module-specific lifecycle handling is needed. +4. Add the profile basename to `profiles/enabled.list`. +5. Run locally with: + +```sh +./run.sh --module ath11k_pci +``` + +No YAML duplication is needed for new profiles. + +## Result policy + +### PASS +- all requested iterations for a profile pass successfully. + +### FAIL +- unload/load timeout, +- unload/load command failure, +- module state validation failure, +- profile hook failure, +- smoke validation failure. + +### SKIP +- module not present, +- module built into the kernel, +- required commands not available, +- profile explicitly not reloadable. + +## Notes +- This suite is intended for **profiled, supported modules**, not blind reload of every loaded kernel module. +- The current structure avoids hidden run.sh-to-library globals as much as possible by passing explicit arguments and hook context. +- Profile hooks receive context arguments from the engine, so module-specific logic can store logs in the correct iteration directory without depending on hidden globals. diff --git a/Runner/suites/Kernel/Baseport/Module_Reload_Validation/profiles/fastrpc.profile b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/profiles/fastrpc.profile new file mode 100755 index 00000000..6dbd2ce0 --- /dev/null +++ b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/profiles/fastrpc.profile @@ -0,0 +1,438 @@ +PROFILE_NAME="fastrpc" +PROFILE_DESCRIPTION="FastRPC module reload validation" +MODULE_NAME="fastrpc" + +MODULE_RELOAD_SUPPORTED="yes" +PROFILE_MODE_DEFAULT="daemon_lifecycle" + +PROFILE_REQUIRED_CMDS="modprobe rmmod systemctl ps sed" +PROFILE_SERVICES="adsprpcd.service cdsprpcd.service" +PROFILE_DEVICE_PATTERNS="/dev/fastrpc-* /dev/*dsp*" +PROFILE_SYSFS_PATTERNS="/sys/module/fastrpc /sys/class/remoteproc/*" + +profile_fastrpc_service_known() { + svc="$1" + + if ! command -v systemctl >/dev/null 2>&1; then + return 1 + fi + + systemctl list-unit-files "$svc" >/dev/null 2>&1 +} + +profile_fastrpc_list_pids() { + proc_pat="$1" + + ps -eo pid=,args= 2>/dev/null | while IFS= read -r line; do + line_trim="$(printf '%s\n' "$line" | sed 's/^[[:space:]]*//')" + [ -n "$line_trim" ] || continue + + pid="${line_trim%% *}" + cmd="${line_trim#"$pid"}" + cmd="$(printf '%s\n' "$cmd" | sed 's/^[[:space:]]*//')" + + [ -n "$pid" ] || continue + [ "$pid" = "$$" ] && continue + + case "$cmd" in + *"$proc_pat"*) + printf '%s\n' "$pid" + ;; + esac + done +} + +profile_fastrpc_proc_running() { + proc_pat="$1" + pids="$(profile_fastrpc_list_pids "$proc_pat")" + + if [ -n "$pids" ]; then + return 0 + fi + return 1 +} + +profile_fastrpc_signal_pattern() { + proc_pat="$1" + sig="$2" + pids="$(profile_fastrpc_list_pids "$proc_pat")" + + for pid in $pids; do + if [ -n "$pid" ] && [ "$pid" != "$$" ]; then + log_info "[fastrpc] sending SIG${sig} to pid=$pid pattern=$proc_pat" + kill "-$sig" "$pid" >/dev/null 2>&1 || true + fi + done + + return 0 +} + +profile_fastrpc_kill_remaining_procs() { + sig="$1" + + profile_fastrpc_signal_pattern "/usr/bin/adsprpcd" "$sig" + profile_fastrpc_signal_pattern "/usr/bin/cdsprpcd" "$sig" + + return 0 +} + +profile_fastrpc_wait_no_procs() { + timeout_s="${1:-10}" + elapsed=0 + + while [ "$elapsed" -lt "$timeout_s" ] 2>/dev/null; do + if profile_fastrpc_proc_running "/usr/bin/adsprpcd"; then + : + elif profile_fastrpc_proc_running "/usr/bin/cdsprpcd"; then + : + else + return 0 + fi + + sleep 1 + elapsed=$((elapsed + 1)) + done + + if profile_fastrpc_proc_running "/usr/bin/adsprpcd"; then + return 1 + fi + if profile_fastrpc_proc_running "/usr/bin/cdsprpcd"; then + return 1 + fi + + return 0 +} + +profile_fastrpc_service_state_value() { + svc="$1" + prop="$2" + + val="$(systemctl show -p "$prop" --value "$svc" 2>/dev/null || true)" + if [ -n "$val" ]; then + printf '%s\n' "$val" + else + printf 'unknown\n' + fi +} + +profile_fastrpc_log_service_process_summary() { + if command -v systemctl >/dev/null 2>&1; then + if profile_fastrpc_service_known "adsprpcd.service"; then + adsp_active="$(profile_fastrpc_service_state_value "adsprpcd.service" "ActiveState")" + adsp_sub="$(profile_fastrpc_service_state_value "adsprpcd.service" "SubState")" + adsp_pid="$(profile_fastrpc_service_state_value "adsprpcd.service" "MainPID")" + log_info "[fastrpc] adsprpcd.service state=${adsp_active}/${adsp_sub} mainpid=${adsp_pid}" + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + cdsp_active="$(profile_fastrpc_service_state_value "cdsprpcd.service" "ActiveState")" + cdsp_sub="$(profile_fastrpc_service_state_value "cdsprpcd.service" "SubState")" + cdsp_pid="$(profile_fastrpc_service_state_value "cdsprpcd.service" "MainPID")" + log_info "[fastrpc] cdsprpcd.service state=${cdsp_active}/${cdsp_sub} mainpid=${cdsp_pid}" + fi + fi + + ps -eo pid=,args= 2>/dev/null | while IFS= read -r line; do + line_trim="$(printf '%s\n' "$line" | sed 's/^[[:space:]]*//')" + [ -n "$line_trim" ] || continue + + pid="${line_trim%% *}" + cmd="${line_trim#"$pid"}" + cmd="$(printf '%s\n' "$cmd" | sed 's/^[[:space:]]*//')" + + [ -n "$pid" ] || continue + [ "$pid" = "$$" ] && continue + + case "$cmd" in + *"/usr/bin/adsprpcd"*|*"/usr/bin/cdsprpcd"*) + log_info "[fastrpc] proc pid=$pid cmd=$cmd" + ;; + esac + done +} + +profile_fastrpc_wait_services_active() { + timeout_s="${1:-15}" + elapsed=0 + + while [ "$elapsed" -lt "$timeout_s" ] 2>/dev/null; do + adsp_ok=1 + cdsp_ok=1 + + if profile_fastrpc_service_known "adsprpcd.service"; then + if ! systemctl is-active --quiet adsprpcd.service; then + adsp_ok=0 + fi + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + if ! systemctl is-active --quiet cdsprpcd.service; then + cdsp_ok=0 + fi + fi + + if [ "$adsp_ok" -eq 1 ] && [ "$cdsp_ok" -eq 1 ]; then + return 0 + fi + + sleep 1 + elapsed=$((elapsed + 1)) + done + + adsp_ok=1 + cdsp_ok=1 + + if profile_fastrpc_service_known "adsprpcd.service"; then + if ! systemctl is-active --quiet adsprpcd.service; then + adsp_ok=0 + fi + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + if ! systemctl is-active --quiet cdsprpcd.service; then + cdsp_ok=0 + fi + fi + + if [ "$adsp_ok" -eq 1 ] && [ "$cdsp_ok" -eq 1 ]; then + return 0 + fi + + return 1 +} + +profile_fastrpc_wait_services_inactive() { + timeout_s="${1:-15}" + elapsed=0 + + while [ "$elapsed" -lt "$timeout_s" ] 2>/dev/null; do + adsp_ok=1 + cdsp_ok=1 + + if profile_fastrpc_service_known "adsprpcd.service"; then + if systemctl is-active --quiet adsprpcd.service; then + adsp_ok=0 + fi + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + if systemctl is-active --quiet cdsprpcd.service; then + cdsp_ok=0 + fi + fi + + if [ "$adsp_ok" -eq 1 ] && [ "$cdsp_ok" -eq 1 ]; then + if profile_fastrpc_wait_no_procs 2; then + return 0 + fi + fi + + sleep 1 + elapsed=$((elapsed + 1)) + done + + adsp_ok=1 + cdsp_ok=1 + + if profile_fastrpc_service_known "adsprpcd.service"; then + if systemctl is-active --quiet adsprpcd.service; then + adsp_ok=0 + fi + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + if systemctl is-active --quiet cdsprpcd.service; then + cdsp_ok=0 + fi + fi + + if [ "$adsp_ok" -eq 1 ] && [ "$cdsp_ok" -eq 1 ]; then + if profile_fastrpc_wait_no_procs 1; then + return 0 + fi + fi + + return 1 +} + +profile_prepare() { + return 0 +} + +profile_warmup() { + iter_dir="$1" + : "${iter_dir:=}" + + if ! command -v systemctl >/dev/null 2>&1; then + return 0 + fi + + case "$PROFILE_SELECTED_MODE" in + basic) + return 0 + ;; + daemon_lifecycle|service_rebind) + if profile_fastrpc_service_known "adsprpcd.service"; then + log_info "[fastrpc] warmup: unmasking and starting adsprpcd.service" + systemctl unmask adsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed adsprpcd.service >/dev/null 2>&1 || true + systemctl start adsprpcd.service >/dev/null 2>&1 || true + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + log_info "[fastrpc] warmup: unmasking and starting cdsprpcd.service" + systemctl unmask cdsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed cdsprpcd.service >/dev/null 2>&1 || true + systemctl start cdsprpcd.service >/dev/null 2>&1 || true + fi + + if ! profile_fastrpc_wait_services_active 15; then + profile_fastrpc_log_service_process_summary + log_error "[fastrpc] warmup: services failed to become active" + return 1 + fi + ;; + *) + log_warn "[fastrpc] unknown mode '$PROFILE_SELECTED_MODE', continuing without warmup actions" + ;; + esac + + return 0 +} + +profile_quiesce() { + iter_dir="$1" + : "${iter_dir:=}" + + if ! command -v systemctl >/dev/null 2>&1; then + return 0 + fi + + case "$PROFILE_SELECTED_MODE" in + basic) + return 0 + ;; + daemon_lifecycle|service_rebind) + if profile_fastrpc_service_known "adsprpcd.service"; then + log_info "[fastrpc] quiesce: stopping adsprpcd.service" + systemctl stop adsprpcd.service >/dev/null 2>&1 || true + log_info "[fastrpc] quiesce: masking adsprpcd.service" + systemctl mask adsprpcd.service >/dev/null 2>&1 || true + log_info "[fastrpc] quiesce: killing remaining adsprpcd.service cgroup processes" + systemctl kill --kill-who=all adsprpcd.service >/dev/null 2>&1 || true + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + log_info "[fastrpc] quiesce: stopping cdsprpcd.service" + systemctl stop cdsprpcd.service >/dev/null 2>&1 || true + log_info "[fastrpc] quiesce: masking cdsprpcd.service" + systemctl mask cdsprpcd.service >/dev/null 2>&1 || true + log_info "[fastrpc] quiesce: killing remaining cdsprpcd.service cgroup processes" + systemctl kill --kill-who=all cdsprpcd.service >/dev/null 2>&1 || true + fi + + profile_fastrpc_kill_remaining_procs TERM + sleep 2 + + if ! profile_fastrpc_wait_no_procs 5; then + log_warn "[fastrpc] quiesce: TERM was not enough, sending SIGKILL to remaining rpc daemons" + profile_fastrpc_kill_remaining_procs KILL + sleep 1 + fi + + if ! profile_fastrpc_wait_services_inactive 20; then + profile_fastrpc_log_service_process_summary + log_error "[fastrpc] quiesce: services/processes did not fully stop" + return 1 + fi + ;; + *) + log_warn "[fastrpc] unknown mode '$PROFILE_SELECTED_MODE', continuing without quiesce actions" + ;; + esac + + return 0 +} + +profile_post_unload() { + iter_dir="$1" + : "${iter_dir:=}" + return 0 +} + +profile_post_load() { + iter_dir="$1" + svc_log="$iter_dir/post_load_services.log" + : > "$svc_log" + + if ! command -v systemctl >/dev/null 2>&1; then + return 0 + fi + + case "$PROFILE_SELECTED_MODE" in + basic) + return 0 + ;; + daemon_lifecycle|service_rebind) + if profile_fastrpc_service_known "adsprpcd.service"; then + log_info "[fastrpc] post-load: unmasking and starting adsprpcd.service" + systemctl unmask adsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed adsprpcd.service >/dev/null 2>&1 || true + systemctl start adsprpcd.service >/dev/null 2>&1 || true + printf '===== %s =====\n' "adsprpcd.service" >> "$svc_log" + systemctl status adsprpcd.service --no-pager >> "$svc_log" 2>&1 || true + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + log_info "[fastrpc] post-load: unmasking and starting cdsprpcd.service" + systemctl unmask cdsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed cdsprpcd.service >/dev/null 2>&1 || true + systemctl start cdsprpcd.service >/dev/null 2>&1 || true + printf '===== %s =====\n' "cdsprpcd.service" >> "$svc_log" + systemctl status cdsprpcd.service --no-pager >> "$svc_log" 2>&1 || true + fi + + if ! profile_fastrpc_wait_services_active 15; then + profile_fastrpc_log_service_process_summary + log_error "[fastrpc] post-load: services failed to become active after reload" + return 1 + fi + ;; + *) + log_warn "[fastrpc] unknown mode '$PROFILE_SELECTED_MODE', continuing without post-load actions" + ;; + esac + + return 0 +} + +profile_smoke() { + iter_dir="$1" + : "${iter_dir:=}" + return 0 +} + +profile_finalize() { + profile_root="$1" + : "${profile_root:=}" + + if ! command -v systemctl >/dev/null 2>&1; then + return 0 + fi + + if profile_fastrpc_service_known "adsprpcd.service"; then + log_info "[fastrpc] finalize: restoring adsprpcd.service" + systemctl unmask adsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed adsprpcd.service >/dev/null 2>&1 || true + systemctl start adsprpcd.service >/dev/null 2>&1 || true + fi + + if profile_fastrpc_service_known "cdsprpcd.service"; then + log_info "[fastrpc] finalize: restoring cdsprpcd.service" + systemctl unmask cdsprpcd.service >/dev/null 2>&1 || true + systemctl reset-failed cdsprpcd.service >/dev/null 2>&1 || true + systemctl start cdsprpcd.service >/dev/null 2>&1 || true + fi + + return 0 +} diff --git a/Runner/suites/Kernel/Baseport/Module_Reload_Validation/run.sh b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/run.sh new file mode 100755 index 00000000..8fc4aa2f --- /dev/null +++ b/Runner/suites/Kernel/Baseport/Module_Reload_Validation/run.sh @@ -0,0 +1,229 @@ +#!/bin/sh +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear + +SCRIPT_DIR="$( + cd "$(dirname "$0")" || exit 1 + pwd +)" + +INIT_ENV="" +SEARCH="$SCRIPT_DIR" +while [ "$SEARCH" != "/" ]; do + if [ -f "$SEARCH/init_env" ]; then + INIT_ENV="$SEARCH/init_env" + break + fi + SEARCH=$(dirname "$SEARCH") +done + +if [ -z "$INIT_ENV" ]; then + echo "[ERROR] init_env not found" >&2 + exit 1 +fi + +if [ -z "${__INIT_ENV_LOADED:-}" ]; then + # shellcheck disable=SC1090 + . "$INIT_ENV" + __INIT_ENV_LOADED=1 +fi + +# shellcheck disable=SC1091 +. "$TOOLS/functestlib.sh" +# shellcheck disable=SC1091 +. "$TOOLS/lib_module_reload.sh" + +TESTNAME="Module_Reload_Validation" +PROFILE_DIR_DEFAULT="$SCRIPT_DIR/profiles" +PROFILE_LIST_DEFAULT="$PROFILE_DIR_DEFAULT/enabled.list" + +TARGET_MODULE="" +ITERATIONS="3" +TIMEOUT_UNLOAD="30" +TIMEOUT_LOAD="30" +TIMEOUT_SETTLE="20" +ENABLE_SYSRQ_HANG_DUMP="1" +PROFILE_MODE="" +PROFILE_DIR="$PROFILE_DIR_DEFAULT" +PROFILE_LIST_FILE="$PROFILE_LIST_DEFAULT" +VERBOSE=0 + +usage() { + cat </dev/null; then + set -x +fi + +case "$ITERATIONS" in + ''|*[!0-9]*) log_error "Invalid --iterations: $ITERATIONS"; exit 1 ;; +esac +case "$TIMEOUT_UNLOAD" in + ''|*[!0-9]*) log_error "Invalid --timeout-unload: $TIMEOUT_UNLOAD"; exit 1 ;; +esac +case "$TIMEOUT_LOAD" in + ''|*[!0-9]*) log_error "Invalid --timeout-load: $TIMEOUT_LOAD"; exit 1 ;; +esac +case "$TIMEOUT_SETTLE" in + ''|*[!0-9]*) log_error "Invalid --timeout-settle: $TIMEOUT_SETTLE"; exit 1 ;; +esac + +test_path="$(find_test_case_by_name "$TESTNAME" 2>/dev/null || echo "$SCRIPT_DIR")" +if ! cd "$test_path"; then + log_error "cd failed: $test_path" + exit 1 +fi + +RES_FILE="$SCRIPT_DIR/${TESTNAME}.res" +RESULT_ROOT="$SCRIPT_DIR/results/$TESTNAME" +SUMMARY_FILE="$RESULT_ROOT/summary.txt" +mkdir -p "$RESULT_ROOT" +: > "$SUMMARY_FILE" + +log_info "---------------- Starting $TESTNAME ----------------" +if command -v detect_platform >/dev/null 2>&1; then + detect_platform >/dev/null 2>&1 || true + log_info "Platform Details: machine='${PLATFORM_MACHINE:-unknown}' target='${PLATFORM_TARGET:-unknown}' kernel='${PLATFORM_KERNEL:-}' arch='${PLATFORM_ARCH:-}'" +else + log_info "Platform Details: unknown" +fi + +log_info "Args: module='${TARGET_MODULE:-all-enabled}' iterations=$ITERATIONS unload_timeout=$TIMEOUT_UNLOAD load_timeout=$TIMEOUT_LOAD settle_timeout=$TIMEOUT_SETTLE mode='${PROFILE_MODE:-profile-default}' sysrq_dump=$ENABLE_SYSRQ_HANG_DUMP" + +if [ "$ENABLE_SYSRQ_HANG_DUMP" -eq 1 ] 2>/dev/null; then + log_info "Sysrq hang dump policy: enabled on timeout paths only" +else + log_info "Sysrq hang dump policy: disabled" +fi + +PROFILE_FILES="$(mrv_resolve_profiles "$TARGET_MODULE" "$PROFILE_DIR" "$PROFILE_LIST_FILE")" +resolve_rc=$? +if [ "$resolve_rc" -ne 0 ]; then + echo "$TESTNAME FAIL" > "$RES_FILE" + exit 1 +fi + +if [ -z "$PROFILE_FILES" ]; then + log_skip "$TESTNAME SKIP - no profiles selected" + echo "$TESTNAME SKIP" > "$RES_FILE" + exit 0 +fi + +PROFILE_TMP_LIST="$RESULT_ROOT/.profiles_to_run.list" +printf '%s\n' "$PROFILE_FILES" > "$PROFILE_TMP_LIST" + +pass_count=0 +fail_count=0 +skip_count=0 + +while IFS= read -r profile_path || [ -n "$profile_path" ]; do + [ -n "$profile_path" ] || continue + + mrv_run_one_profile "$profile_path" "$PROFILE_MODE" + rc=$? + base_name="$(basename "$profile_path" .profile)" + + if [ "$rc" -eq 0 ]; then + log_pass "[$base_name] profile PASS" + printf '%s PASS\n' "$base_name" >> "$SUMMARY_FILE" + pass_count=$((pass_count + 1)) + elif [ "$rc" -eq 2 ]; then + log_skip "[$base_name] profile SKIP" + printf '%s SKIP\n' "$base_name" >> "$SUMMARY_FILE" + skip_count=$((skip_count + 1)) + else + log_fail "[$base_name] profile FAIL" + printf '%s FAIL\n' "$base_name" >> "$SUMMARY_FILE" + fail_count=$((fail_count + 1)) + fi +done < "$PROFILE_TMP_LIST" + +log_info "Summary: pass=$pass_count fail=$fail_count skip=$skip_count" + +if [ "$fail_count" -gt 0 ] 2>/dev/null; then + log_fail "$TESTNAME FAIL" + echo "$TESTNAME FAIL" > "$RES_FILE" + exit 1 +fi + +if [ "$pass_count" -gt 0 ] 2>/dev/null; then + log_pass "$TESTNAME PASS" + echo "$TESTNAME PASS" > "$RES_FILE" + exit 0 +fi + +log_skip "$TESTNAME SKIP" +echo "$TESTNAME SKIP" > "$RES_FILE" +exit 0 diff --git a/Runner/utils/lib_module_reload.sh b/Runner/utils/lib_module_reload.sh new file mode 100755 index 00000000..b481441a --- /dev/null +++ b/Runner/utils/lib_module_reload.sh @@ -0,0 +1,597 @@ +#!/bin/sh +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear + +MRV_LAST_CMD_PID="" +MRV_LAST_CMD_ELAPSED="0" +MRV_LAST_TIMEOUT_DIR="" +MRV_PROFILE_REASON="" + +mrv_reset_profile_vars() { + PROFILE_NAME="" + PROFILE_DESCRIPTION="" + MODULE_NAME="" + MODULE_LOAD_CMD="" + MODULE_UNLOAD_CMD="" + MODULE_RELOAD_SUPPORTED="yes" + PROFILE_MODE_DEFAULT="basic" + PROFILE_REQUIRED_CMDS="" + PROFILE_SERVICES="" + PROFILE_DEVICE_PATTERNS="" + PROFILE_SYSFS_PATTERNS="" + PROFILE_PREPARE_HOOK="profile_prepare" + PROFILE_WARMUP_HOOK="profile_warmup" + PROFILE_QUIESCE_HOOK="profile_quiesce" + PROFILE_POST_UNLOAD_HOOK="profile_post_unload" + PROFILE_POST_LOAD_HOOK="profile_post_load" + PROFILE_SMOKE_HOOK="profile_smoke" + PROFILE_FINALIZE_HOOK="profile_finalize" + PROFILE_SELECTED_MODE="" +} + +mrv_module_lsmod_name() { + printf '%s\n' "$1" | tr '-' '_' +} + +mrv_module_loaded() { + name="$(mrv_module_lsmod_name "$1")" + lsmod 2>/dev/null | awk -v mod="$name" 'NR > 1 && $1 == mod { found=1; exit } END { exit !found }' +} + +mrv_module_builtin() { + if [ -f "/lib/modules/$(uname -r)/modules.builtin" ]; then + mod_pat="$(printf '%s\n' "$1" | sed 's/_/[-_]/g')" + grep -Eq "/${mod_pat}(\\.ko(\\.[^.]+)*)?$" "/lib/modules/$(uname -r)/modules.builtin" 2>/dev/null + return $? + fi + return 1 +} + +mrv_wait_module_state() { + want="$1" + name="$2" + timeout_s="$3" + elapsed=0 + + while [ "$elapsed" -lt "$timeout_s" ] 2>/dev/null; do + if [ "$want" = "present" ]; then + if mrv_module_loaded "$name"; then + return 0 + fi + else + if ! mrv_module_loaded "$name"; then + return 0 + fi + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + + if [ "$want" = "present" ]; then + mrv_module_loaded "$name" + return $? + fi + + if mrv_module_loaded "$name"; then + return 1 + fi + return 0 +} + +mrv_capture_patterns() { + patterns="$1" + outfile="$2" + + : > "$outfile" + + if [ -z "$patterns" ]; then + return 0 + fi + + # shellcheck disable=SC2086 + set -- $patterns + for pat in "$@"; do + found=0 + # shellcheck disable=SC2086 + for path in $pat; do + if [ -e "$path" ] || [ -L "$path" ]; then + found=1 + printf 'PATH: %s\n' "$path" >> "$outfile" + ls -ld "$path" >> "$outfile" 2>&1 || true + fi + done + if [ "$found" -eq 0 ] 2>/dev/null; then + printf 'PATH: %s (not present)\n' "$pat" >> "$outfile" + fi + done +} + +mrv_capture_services() { + services="$1" + outfile="$2" + + : > "$outfile" + + if [ -z "$services" ]; then + printf 'No profile services defined\n' >> "$outfile" + return 0 + fi + + if ! command -v systemctl >/dev/null 2>&1; then + printf 'systemctl not available\n' >> "$outfile" + return 0 + fi + + # shellcheck disable=SC2086 + set -- $services + for svc in "$@"; do + printf '===== %s =====\n' "$svc" >> "$outfile" + systemctl status "$svc" --no-pager >> "$outfile" 2>&1 || true + if command -v journalctl >/dev/null 2>&1; then + printf '\n----- journalctl -u %s -----\n' "$svc" >> "$outfile" + journalctl -u "$svc" --no-pager -n 200 >> "$outfile" 2>&1 || true + fi + done +} + +mrv_capture_module_state() { + outdir="$1" + mkdir -p "$outdir" + + { + printf 'profile=%s\n' "${PROFILE_NAME:-unknown}" + printf 'description=%s\n' "${PROFILE_DESCRIPTION:-unknown}" + printf 'module=%s\n' "${MODULE_NAME:-unknown}" + printf 'mode=%s\n' "${PROFILE_SELECTED_MODE:-unknown}" + printf 'kernel=%s\n' "$(uname -r)" + printf 'date=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || echo unknown)" + } > "$outdir/state.txt" + + sys_mod="$(mrv_module_lsmod_name "$MODULE_NAME")" + + { + printf 'Module-focused lsmod view for %s\n' "$sys_mod" + lsmod 2>/dev/null | awk -v mod="$sys_mod" ' + NR == 1 { print; next } + $1 == mod { found=1; print } + END { + if (!found) { + printf "module %s not present in lsmod\n", mod + } + } + ' + } > "$outdir/lsmod.log" 2>&1 || true + + modinfo "$MODULE_NAME" > "$outdir/modinfo.log" 2>&1 || true + ps -ef > "$outdir/ps.log" 2>&1 || true + dmesg > "$outdir/dmesg.log" 2>&1 || true + + if [ -d "/sys/module/$sys_mod" ]; then + find "/sys/module/$sys_mod" -maxdepth 3 -print > "$outdir/sys_module_tree.log" 2>&1 || true + if [ -d "/sys/module/$sys_mod/holders" ]; then + ls -la "/sys/module/$sys_mod/holders" > "$outdir/holders.log" 2>&1 || true + fi + else + printf '/sys/module/%s not present\n' "$sys_mod" > "$outdir/holders.log" + fi + + mrv_capture_services "$PROFILE_SERVICES" "$outdir/services.log" + mrv_capture_patterns "$PROFILE_DEVICE_PATTERNS" "$outdir/devices.log" + mrv_capture_patterns "$PROFILE_SYSFS_PATTERNS" "$outdir/profile_sysfs.log" + + if command -v journalctl >/dev/null 2>&1; then + journalctl -b --no-pager -n 300 > "$outdir/journal.log" 2>&1 || true + fi +} + +mrv_capture_pid_timeout_snapshot() { + pid="$1" + outdir="$2" + + mkdir -p "$outdir" + + if [ -d "/proc/$pid" ]; then + cat "/proc/$pid/status" > "$outdir/proc_status.log" 2>&1 || true + cat "/proc/$pid/wchan" > "$outdir/proc_wchan.log" 2>&1 || true + cat "/proc/$pid/stack" > "$outdir/proc_stack.log" 2>&1 || true + ps -T -p "$pid" > "$outdir/ps_threads.log" 2>&1 || true + fi +} + +mrv_exec_with_timeout() { + timeout_s="$1" + logfile="$2" + shift 2 + cmd="$*" + elapsed=0 + + : > "$logfile" + printf 'CMD: %s\n' "$cmd" >> "$logfile" + + sh -c "$cmd" >> "$logfile" 2>&1 & + cmd_pid=$! + MRV_LAST_CMD_PID="$cmd_pid" + MRV_LAST_CMD_ELAPSED=0 + + while kill -0 "$cmd_pid" 2>/dev/null; do + if [ "$elapsed" -ge "$timeout_s" ] 2>/dev/null; then + MRV_LAST_CMD_ELAPSED="$elapsed" + + printf 'TIMEOUT: command exceeded %ss (pid=%s)\n' "$timeout_s" "$cmd_pid" >> "$logfile" + + if [ -n "$MRV_LAST_TIMEOUT_DIR" ]; then + mrv_capture_pid_timeout_snapshot "$cmd_pid" "$MRV_LAST_TIMEOUT_DIR" + fi + + kill -TERM "$cmd_pid" 2>/dev/null || true + sleep 2 + + if kill -0 "$cmd_pid" 2>/dev/null; then + printf 'TIMEOUT: pid %s ignored SIGTERM, sending SIGKILL\n' "$cmd_pid" >> "$logfile" + kill -KILL "$cmd_pid" 2>/dev/null || true + sleep 1 + fi + + if kill -0 "$cmd_pid" 2>/dev/null; then + printf 'TIMEOUT: pid %s still present after SIGKILL (likely stuck in kernel)\n' "$cmd_pid" >> "$logfile" + fi + + return 124 + fi + + sleep 1 + elapsed=$((elapsed + 1)) + done + + wait "$cmd_pid" + rc=$? + MRV_LAST_CMD_ELAPSED="$elapsed" + return "$rc" +} + +mrv_capture_hang_evidence() { + phase="$1" + outdir="$2" + + mkdir -p "$outdir" + mrv_capture_module_state "$outdir" + + if [ "$ENABLE_SYSRQ_HANG_DUMP" -eq 1 ] 2>/dev/null; then + if [ -w /proc/sysrq-trigger ]; then + printf 'Triggered sysrq-t and sysrq-w\n' > "$outdir/sysrq_actions.log" + echo t > /proc/sysrq-trigger 2>/dev/null || true + echo w > /proc/sysrq-trigger 2>/dev/null || true + dmesg > "$outdir/dmesg_after_sysrq.log" 2>&1 || true + fi + fi + + if command -v fuser >/dev/null 2>&1; then + : > "$outdir/fuser.log" + # shellcheck disable=SC2086 + set -- $PROFILE_DEVICE_PATTERNS + for pat in "$@"; do + # shellcheck disable=SC2086 + for path in $pat; do + if [ -e "$path" ] || [ -L "$path" ]; then + printf '===== %s =====\n' "$path" >> "$outdir/fuser.log" + fuser -vm "$path" >> "$outdir/fuser.log" 2>&1 || true + fi + done + done + fi + + if command -v lsof >/dev/null 2>&1; then + lsof > "$outdir/lsof.log" 2>&1 || true + fi + + { + printf 'phase=%s\n' "$phase" + printf 'pid=%s\n' "$MRV_LAST_CMD_PID" + printf 'elapsed=%s\n' "$MRV_LAST_CMD_ELAPSED" + } > "$outdir/timeout_summary.log" +} + +mrv_run_hook() { + hook_name="$1" + shift + + if [ -n "$hook_name" ] && command -v "$hook_name" >/dev/null 2>&1; then + "$hook_name" "$@" + return $? + fi + + return 0 +} + +mrv_profile_check() { + requested_mode="$1" + + if [ -z "$PROFILE_NAME" ] || [ -z "$MODULE_NAME" ]; then + MRV_PROFILE_REASON="profile missing PROFILE_NAME or MODULE_NAME" + return 2 + fi + + if [ "${MODULE_RELOAD_SUPPORTED:-yes}" != "yes" ]; then + MRV_PROFILE_REASON="profile marks module as non-reloadable" + return 2 + fi + + if mrv_module_builtin "$MODULE_NAME"; then + MRV_PROFILE_REASON="module is built-in and cannot be unloaded" + return 2 + fi + + if ! modinfo "$MODULE_NAME" >/dev/null 2>&1; then + if ! mrv_module_loaded "$MODULE_NAME"; then + MRV_PROFILE_REASON="module not present on image" + return 2 + fi + fi + + if ! command -v modprobe >/dev/null 2>&1; then + MRV_PROFILE_REASON="modprobe not available" + return 2 + fi + + if ! command -v rmmod >/dev/null 2>&1; then + MRV_PROFILE_REASON="rmmod not available" + return 2 + fi + + required="${PROFILE_REQUIRED_CMDS:-}" + if [ -n "$required" ]; then + # shellcheck disable=SC2086 + set -- $required + for req in "$@"; do + if ! command -v "$req" >/dev/null 2>&1; then + MRV_PROFILE_REASON="required command missing: $req" + return 2 + fi + done + fi + + MODULE_LOAD_CMD="${MODULE_LOAD_CMD:-modprobe $MODULE_NAME}" + MODULE_UNLOAD_CMD="${MODULE_UNLOAD_CMD:-rmmod $MODULE_NAME}" + PROFILE_SELECTED_MODE="${requested_mode:-${PROFILE_MODE_DEFAULT:-basic}}" + + return 0 +} + +mrv_post_unload_validate() { + if ! mrv_wait_module_state absent "$MODULE_NAME" "$TIMEOUT_SETTLE"; then + log_fail "[$PROFILE_NAME] module still present after unload settle timeout" + return 1 + fi + return 0 +} + +mrv_post_load_validate() { + if ! mrv_wait_module_state present "$MODULE_NAME" "$TIMEOUT_SETTLE"; then + log_fail "[$PROFILE_NAME] module did not reappear after load settle timeout" + return 1 + fi + return 0 +} + +mrv_run_iteration() { + iter="$1" + iter_dir="$RESULT_ROOT/$PROFILE_NAME/iter_$(printf '%02d' "$iter")" + preload_log="$iter_dir/preload.log" + unload_log="$iter_dir/unload.log" + load_log="$iter_dir/load.log" + unload_elapsed=0 + load_elapsed=0 + + mkdir -p "$iter_dir" + + if ! mrv_module_loaded "$MODULE_NAME"; then + log_info "[$PROFILE_NAME] module not loaded before iteration $iter, creating baseline loaded state" + MRV_LAST_TIMEOUT_DIR="$iter_dir/preload_timeout" + mrv_exec_with_timeout "$TIMEOUT_LOAD" "$preload_log" "$MODULE_LOAD_CMD" + preload_rc=$? + if [ "$preload_rc" -ne 0 ]; then + log_fail "[$PROFILE_NAME] failed to create baseline loaded state before iteration $iter (rc=$preload_rc)" + log_info "[$PROFILE_NAME] preload log: $preload_log" + return 1 + fi + if ! mrv_post_load_validate; then + log_info "[$PROFILE_NAME] baseline post-load validation failed" + return 1 + fi + fi + + if ! mrv_run_hook "$PROFILE_WARMUP_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] warmup hook failed in iteration $iter" + return 1 + fi + + mrv_capture_module_state "$iter_dir/pre_state" + + if ! mrv_run_hook "$PROFILE_QUIESCE_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] quiesce hook failed in iteration $iter" + return 1 + fi + + log_info "[$PROFILE_NAME] iter $iter/$ITERATIONS exec: $MODULE_UNLOAD_CMD" + MRV_LAST_TIMEOUT_DIR="$iter_dir/unload_timeout" + mrv_exec_with_timeout "$TIMEOUT_UNLOAD" "$unload_log" "$MODULE_UNLOAD_CMD" + unload_rc=$? + unload_elapsed="$MRV_LAST_CMD_ELAPSED" + + if [ "$unload_rc" -eq 124 ]; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS unload timed out after ${unload_elapsed}s (pid=$MRV_LAST_CMD_PID)" + log_info "[$PROFILE_NAME] unload timeout log: $unload_log" + log_info "[$PROFILE_NAME] collecting hang evidence in: $iter_dir/hang_evidence" + mrv_capture_hang_evidence unload "$iter_dir/hang_evidence" + log_fail "[$PROFILE_NAME] hang evidence captured; failing iteration $iter/$ITERATIONS" + return 1 + fi + + if [ "$unload_rc" -ne 0 ]; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS unload failed (rc=$unload_rc)" + log_info "[$PROFILE_NAME] unload log: $unload_log" + mrv_capture_module_state "$iter_dir/unload_failure_state" + return 1 + fi + + if ! mrv_post_unload_validate; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS post-unload validation failed" + mrv_capture_module_state "$iter_dir/post_unload_invalid_state" + return 1 + fi + + mrv_capture_module_state "$iter_dir/post_unload_state" + + if ! mrv_run_hook "$PROFILE_POST_UNLOAD_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] post-unload hook failed in iteration $iter" + return 1 + fi + + log_info "[$PROFILE_NAME] iter $iter/$ITERATIONS exec: $MODULE_LOAD_CMD" + MRV_LAST_TIMEOUT_DIR="$iter_dir/load_timeout" + mrv_exec_with_timeout "$TIMEOUT_LOAD" "$load_log" "$MODULE_LOAD_CMD" + load_rc=$? + load_elapsed="$MRV_LAST_CMD_ELAPSED" + + if [ "$load_rc" -eq 124 ]; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS load timed out after ${load_elapsed}s (pid=$MRV_LAST_CMD_PID)" + log_info "[$PROFILE_NAME] load timeout log: $load_log" + log_info "[$PROFILE_NAME] collecting hang evidence in: $iter_dir/load_hang_evidence" + mrv_capture_hang_evidence load "$iter_dir/load_hang_evidence" + log_fail "[$PROFILE_NAME] hang evidence captured, failing iteration $iter/$ITERATIONS" + return 1 + fi + + if [ "$load_rc" -ne 0 ]; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS load failed (rc=$load_rc)" + log_info "[$PROFILE_NAME] load log: $load_log" + mrv_capture_module_state "$iter_dir/load_failure_state" + return 1 + fi + + if ! mrv_post_load_validate; then + log_fail "[$PROFILE_NAME] iter $iter/$ITERATIONS post-load validation failed" + mrv_capture_module_state "$iter_dir/post_load_invalid_state" + return 1 + fi + + if ! mrv_run_hook "$PROFILE_POST_LOAD_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] post-load hook failed in iteration $iter" + mrv_capture_module_state "$iter_dir/post_load_hook_failure_state" + return 1 + fi + + if ! mrv_run_hook "$PROFILE_SMOKE_HOOK" "$iter_dir"; then + log_fail "[$PROFILE_NAME] smoke hook failed in iteration $iter" + mrv_capture_module_state "$iter_dir/smoke_failure_state" + return 1 + fi + + mrv_capture_module_state "$iter_dir/post_load_state" + + if command -v diff >/dev/null 2>&1; then + diff -u "$iter_dir/pre_state/dmesg.log" "$iter_dir/post_load_state/dmesg.log" > "$iter_dir/dmesg.diff" 2>&1 || true + fi + + { + printf 'iter=%s\n' "$iter" + printf 'unload_elapsed=%s\n' "$unload_elapsed" + printf 'load_elapsed=%s\n' "$load_elapsed" + } > "$iter_dir/iteration_metrics.log" + + log_pass "[$PROFILE_NAME] iteration $iter/$ITERATIONS passed" + return 0 +} + +mrv_run_one_profile() ( + profile_file="$1" + requested_mode="$2" + + mrv_reset_profile_vars + + # shellcheck disable=SC1090 + . "$profile_file" + + mrv_profile_check "$requested_mode" + check_rc=$? + if [ "$check_rc" -eq 2 ]; then + log_skip "[$(basename "$profile_file" .profile)] SKIP - $MRV_PROFILE_REASON" + exit 2 + fi + if [ "$check_rc" -ne 0 ]; then + log_fail "[$(basename "$profile_file" .profile)] FAIL - profile validation error" + exit 1 + fi + + profile_root="$RESULT_ROOT/$PROFILE_NAME" + mkdir -p "$profile_root" + + { + printf 'profile=%s\n' "$PROFILE_NAME" + printf 'description=%s\n' "${PROFILE_DESCRIPTION:-unknown}" + printf 'module=%s\n' "$MODULE_NAME" + printf 'mode=%s\n' "$PROFILE_SELECTED_MODE" + } > "$profile_root/profile_info.txt" + + log_info "[$PROFILE_NAME] module=$MODULE_NAME mode=$PROFILE_SELECTED_MODE iterations=$ITERATIONS" + + if ! mrv_run_hook "$PROFILE_PREPARE_HOOK" "$profile_root"; then + log_fail "[$PROFILE_NAME] prepare hook failed" + exit 1 + fi + + iter=1 + while [ "$iter" -le "$ITERATIONS" ] 2>/dev/null; do + if ! mrv_run_iteration "$iter"; then + mrv_run_hook "$PROFILE_FINALIZE_HOOK" "$profile_root" >/dev/null 2>&1 || true + exit 1 + fi + iter=$((iter + 1)) + done + + if ! mrv_run_hook "$PROFILE_FINALIZE_HOOK" "$profile_root"; then + log_fail "[$PROFILE_NAME] finalize hook failed" + exit 1 + fi + + exit 0 +) + +mrv_resolve_profiles() { + target_module="$1" + profile_dir="$2" + profile_list_file="$3" + + if [ -n "$target_module" ]; then + candidate="$profile_dir/$target_module.profile" + if [ ! -f "$candidate" ]; then + log_error "Profile not found: $candidate" + return 1 + fi + printf '%s\n' "$candidate" + return 0 + fi + + if [ -f "$profile_list_file" ]; then + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|'#'*) continue ;; + esac + candidate="$profile_dir/$line.profile" + if [ -f "$candidate" ]; then + printf '%s\n' "$candidate" + else + log_warn "Enabled profile listed but missing: $candidate" + fi + done < "$profile_list_file" + return 0 + fi + + for candidate in "$profile_dir"/*.profile; do + if [ -f "$candidate" ]; then + printf '%s\n' "$candidate" + fi + done + + return 0 +}