diff --git a/Cargo.toml b/Cargo.toml index 13b3195..e41e3e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vectorless" -version = "0.1.16" +version = "0.1.17" edition = "2024" authors = ["zTgx "] description = "Hierarchical, reasoning-native document intelligence engine" diff --git a/src/config/types/storage.rs b/src/config/types/storage.rs index 562c7ba..ac8bd2c 100644 --- a/src/config/types/storage.rs +++ b/src/config/types/storage.rs @@ -337,6 +337,163 @@ pub struct StrategyConfig { /// Low similarity threshold for "explore" decision. #[serde(default = "default_low_similarity_threshold")] pub low_similarity_threshold: f32, + + /// Hybrid strategy configuration (BM25 + LLM refinement). + #[serde(default)] + pub hybrid: HybridStrategyConfig, + + /// Cross-document strategy configuration. + #[serde(default)] + pub cross_document: CrossDocumentStrategyConfig, + + /// Page-range strategy configuration. + #[serde(default)] + pub page_range: PageRangeStrategyConfig, +} + +/// Hybrid strategy configuration (BM25 pre-filter + LLM refinement). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HybridStrategyConfig { + /// Enable hybrid strategy. + #[serde(default = "default_true")] + pub enabled: bool, + + /// BM25 pre-filter: keep top N% of candidates. + #[serde(default = "default_pre_filter_ratio")] + pub pre_filter_ratio: f32, + + /// Minimum candidates to pass to LLM. + #[serde(default = "default_min_candidates")] + pub min_candidates: usize, + + /// Maximum candidates for LLM refinement. + #[serde(default = "default_max_candidates")] + pub max_candidates: usize, + + /// BM25 score for auto-accept (skip LLM). + #[serde(default = "default_auto_accept_threshold")] + pub auto_accept_threshold: f32, + + /// BM25 score for auto-reject (skip LLM). + #[serde(default = "default_auto_reject_threshold")] + pub auto_reject_threshold: f32, + + /// Weight for BM25 score in final scoring. + #[serde(default = "default_bm25_weight")] + pub bm25_weight: f32, + + /// Weight for LLM score in final scoring. + #[serde(default = "default_llm_weight")] + pub llm_weight: f32, +} + +fn default_true() -> bool { true } +fn default_pre_filter_ratio() -> f32 { 0.3 } +fn default_min_candidates() -> usize { 2 } +fn default_max_candidates() -> usize { 5 } +fn default_auto_accept_threshold() -> f32 { 0.85 } +fn default_auto_reject_threshold() -> f32 { 0.15 } +fn default_bm25_weight() -> f32 { 0.4 } +fn default_llm_weight() -> f32 { 0.6 } + +impl Default for HybridStrategyConfig { + fn default() -> Self { + Self { + enabled: true, + pre_filter_ratio: default_pre_filter_ratio(), + min_candidates: default_min_candidates(), + max_candidates: default_max_candidates(), + auto_accept_threshold: default_auto_accept_threshold(), + auto_reject_threshold: default_auto_reject_threshold(), + bm25_weight: default_bm25_weight(), + llm_weight: default_llm_weight(), + } + } +} + +/// Cross-document strategy configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CrossDocumentStrategyConfig { + /// Enable cross-document strategy. + #[serde(default = "default_true")] + pub enabled: bool, + + /// Maximum documents to search. + #[serde(default = "default_max_documents")] + pub max_documents: usize, + + /// Maximum results per document. + #[serde(default = "default_max_results_per_doc")] + pub max_results_per_doc: usize, + + /// Maximum total results. + #[serde(default = "default_max_total_results")] + pub max_total_results: usize, + + /// Minimum score threshold. + #[serde(default = "default_min_score")] + pub min_score: f32, + + /// Merge strategy: TopK, BestPerDocument, WeightedByRelevance. + #[serde(default = "default_merge_strategy")] + pub merge_strategy: String, + + /// Search documents in parallel. + #[serde(default = "default_true")] + pub parallel_search: bool, +} + +fn default_max_documents() -> usize { 10 } +fn default_max_results_per_doc() -> usize { 3 } +fn default_max_total_results() -> usize { 10 } +fn default_min_score() -> f32 { 0.3 } +fn default_merge_strategy() -> String { "TopK".to_string() } + +impl Default for CrossDocumentStrategyConfig { + fn default() -> Self { + Self { + enabled: true, + max_documents: default_max_documents(), + max_results_per_doc: default_max_results_per_doc(), + max_total_results: default_max_total_results(), + min_score: default_min_score(), + merge_strategy: default_merge_strategy(), + parallel_search: true, + } + } +} + +/// Page-range strategy configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PageRangeStrategyConfig { + /// Enable page-range strategy. + #[serde(default = "default_true")] + pub enabled: bool, + + /// Include nodes that span across the boundary. + #[serde(default = "default_true")] + pub include_boundary_nodes: bool, + + /// Expand range by N pages for context. + #[serde(default)] + pub expand_context_pages: usize, + + /// Minimum overlap ratio for node inclusion. + #[serde(default = "default_min_overlap_ratio")] + pub min_overlap_ratio: f32, +} + +fn default_min_overlap_ratio() -> f32 { 0.1 } + +impl Default for PageRangeStrategyConfig { + fn default() -> Self { + Self { + enabled: true, + include_boundary_nodes: true, + expand_context_pages: 0, + min_overlap_ratio: default_min_overlap_ratio(), + } + } } fn default_exploration_weight() -> f32 { @@ -362,6 +519,9 @@ impl Default for StrategyConfig { similarity_threshold: default_similarity_threshold(), high_similarity_threshold: default_high_similarity_threshold(), low_similarity_threshold: default_low_similarity_threshold(), + hybrid: HybridStrategyConfig::default(), + cross_document: CrossDocumentStrategyConfig::default(), + page_range: PageRangeStrategyConfig::default(), } } } @@ -453,5 +613,37 @@ mod tests { let config = StrategyConfig::default(); assert!((config.exploration_weight - 1.414).abs() < 0.001); assert_eq!(config.similarity_threshold, 0.5); + assert!(config.hybrid.enabled); + assert!(config.cross_document.enabled); + assert!(config.page_range.enabled); + } + + #[test] + fn test_hybrid_strategy_config_defaults() { + let config = HybridStrategyConfig::default(); + assert!(config.enabled); + assert!((config.pre_filter_ratio - 0.3).abs() < f32::EPSILON); + assert_eq!(config.min_candidates, 2); + assert_eq!(config.max_candidates, 5); + assert!((config.auto_accept_threshold - 0.85).abs() < f32::EPSILON); + } + + #[test] + fn test_cross_document_strategy_config_defaults() { + let config = CrossDocumentStrategyConfig::default(); + assert!(config.enabled); + assert_eq!(config.max_documents, 10); + assert_eq!(config.max_results_per_doc, 3); + assert_eq!(config.merge_strategy, "TopK"); + assert!(config.parallel_search); + } + + #[test] + fn test_page_range_strategy_config_defaults() { + let config = PageRangeStrategyConfig::default(); + assert!(config.enabled); + assert!(config.include_boundary_nodes); + assert_eq!(config.expand_context_pages, 0); + assert!((config.min_overlap_ratio - 0.1).abs() < f32::EPSILON); } } diff --git a/src/retrieval/mod.rs b/src/retrieval/mod.rs index de9c009..dc1a289 100644 --- a/src/retrieval/mod.rs +++ b/src/retrieval/mod.rs @@ -89,7 +89,10 @@ pub use stages::{AnalyzeStage, EvaluateStage, PlanStage, SearchStage}; // Strategy exports pub use strategy::{ - KeywordStrategy, LlmStrategy, RetrievalStrategy, SemanticStrategy, StrategyCapabilities, + CrossDocumentConfig, CrossDocumentStrategy, DocumentEntry, DocumentId, DocumentResult, + HybridConfig, HybridStrategy, KeywordStrategy, LlmStrategy, MergeStrategy, + PageRange, PageRangeConfig, PageRangeStrategy, RetrievalStrategy, SemanticStrategy, + StrategyCapabilities, StrategyCost, }; // Search exports diff --git a/src/retrieval/stages/search.rs b/src/retrieval/stages/search.rs index 58a6e68..78f07c2 100644 --- a/src/retrieval/stages/search.rs +++ b/src/retrieval/stages/search.rs @@ -21,7 +21,9 @@ use crate::retrieval::pipeline::{ use crate::retrieval::search::{ BeamSearch, GreedySearch, SearchConfig as SearchAlgConfig, SearchTree, }; -use crate::retrieval::strategy::{KeywordStrategy, LlmStrategy, RetrievalStrategy}; +use crate::retrieval::strategy::{ + HybridConfig, HybridStrategy, KeywordStrategy, LlmStrategy, RetrievalStrategy, +}; use crate::retrieval::types::StrategyPreference; /// Search Stage - executes tree search with optional Pilot guidance. @@ -52,6 +54,7 @@ pub struct SearchStage { keyword_strategy: KeywordStrategy, llm_strategy: Option>, semantic_strategy: Option>, + hybrid_strategy: Option>, /// Pilot for navigation guidance (optional). pilot: Option>, } @@ -69,6 +72,7 @@ impl SearchStage { keyword_strategy: KeywordStrategy::new(), llm_strategy: None, semantic_strategy: None, + hybrid_strategy: None, pilot: None, } } @@ -95,6 +99,26 @@ impl SearchStage { self } + /// Add hybrid strategy (BM25 + LLM refinement). + /// + /// If no LLM strategy is set, creates one from the provided LLM strategy. + pub fn with_hybrid_strategy(mut self, strategy: Arc) -> Self { + self.hybrid_strategy = Some(strategy); + self + } + + /// Configure hybrid strategy with custom config using the LLM strategy. + pub fn with_hybrid_config(mut self, config: HybridConfig) -> Self { + if let Some(ref llm) = self.llm_strategy { + // Clone the LlmStrategy and box it + let llm_boxed: Box = Box::new((**llm).clone()); + self.hybrid_strategy = Some(Arc::new( + HybridStrategy::new(llm_boxed).with_config(config) + )); + } + self + } + /// Check if Pilot is available and active. pub fn has_pilot(&self) -> bool { self.pilot.as_ref().map(|p| p.is_active()).unwrap_or(false) @@ -127,6 +151,29 @@ impl SearchStage { Arc::new(self.keyword_strategy.clone()) } } + StrategyPreference::ForceHybrid => { + if let Some(ref strategy) = self.hybrid_strategy { + info!("Using Hybrid strategy"); + strategy.clone() + } else if let Some(ref llm) = self.llm_strategy { + info!("Using Hybrid strategy (auto-created from LLM)"); + let llm_boxed: Box = Box::new((**llm).clone()); + Arc::new(HybridStrategy::new(llm_boxed)) + } else { + warn!("Hybrid strategy requested but no LLM available, falling back to Keyword"); + Arc::new(self.keyword_strategy.clone()) + } + } + StrategyPreference::ForceCrossDocument | StrategyPreference::ForcePageRange => { + // These require special setup, fall back to hybrid or keyword + if let Some(ref strategy) = self.hybrid_strategy { + info!("Using Hybrid strategy as fallback for {:?})", preference); + strategy.clone() + } else { + warn!("{:?} requires special configuration, falling back to Keyword", preference); + Arc::new(self.keyword_strategy.clone()) + } + } StrategyPreference::Auto => { // Default to keyword, let plan stage decide Arc::new(self.keyword_strategy.clone()) diff --git a/src/retrieval/strategy/cross_document.rs b/src/retrieval/strategy/cross_document.rs new file mode 100644 index 0000000..d451f5c --- /dev/null +++ b/src/retrieval/strategy/cross_document.rs @@ -0,0 +1,329 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Cross-document retrieval strategy. +//! +//! Retrieves relevant content from multiple documents, aggregating +//! results into a unified response. + +use async_trait::async_trait; +use std::collections::HashMap; + +use super::r#trait::{NodeEvaluation, RetrievalStrategy, StrategyCapabilities}; +use crate::document::{DocumentTree, NodeId}; +use crate::retrieval::types::{NavigationDecision, QueryComplexity}; +use crate::retrieval::RetrievalContext; + +/// Document identifier for cross-document retrieval. +pub type DocumentId = String; + +/// A document with its tree structure for cross-document retrieval. +pub struct DocumentEntry { + /// Unique document identifier. + pub id: DocumentId, + /// Document title or name. + pub title: String, + /// The document tree. + pub tree: DocumentTree, +} + +impl DocumentEntry { + /// Create a new document entry. + pub fn new(id: impl Into, title: impl Into, tree: DocumentTree) -> Self { + Self { + id: id.into(), + title: title.into(), + tree, + } + } +} + +/// Result from a single document in cross-document retrieval. +#[derive(Debug, Clone)] +pub struct DocumentResult { + /// Document ID. + pub doc_id: DocumentId, + /// Document title. + pub doc_title: String, + /// Node evaluation results from this document. + pub evaluations: Vec<(NodeId, NodeEvaluation)>, + /// Best score from this document. + pub best_score: f32, +} + +/// Strategy for merging results from multiple documents. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum MergeStrategy { + /// Take top-k results across all documents (default). + #[default] + TopK, + /// Take best result from each document. + BestPerDocument, + /// Weight results by document relevance score. + WeightedByRelevance, +} + +/// Configuration for cross-document retrieval. +#[derive(Debug, Clone)] +pub struct CrossDocumentConfig { + /// Maximum number of documents to search. + pub max_documents: usize, + /// Maximum results per document. + pub max_results_per_doc: usize, + /// Maximum total results. + pub max_total_results: usize, + /// Minimum score threshold for including results. + pub min_score: f32, + /// How to merge results from multiple documents. + pub merge_strategy: MergeStrategy, + /// Whether to search documents in parallel. + pub parallel_search: bool, +} + +impl Default for CrossDocumentConfig { + fn default() -> Self { + Self { + max_documents: 10, + max_results_per_doc: 3, + max_total_results: 10, + min_score: 0.3, + merge_strategy: MergeStrategy::TopK, + parallel_search: true, + } + } +} + +/// Cross-document retrieval strategy. +/// +/// Searches multiple documents and aggregates results based on +/// the configured merge strategy. +/// +/// # Example +/// +/// ```rust,ignore +/// use vectorless::retrieval::strategy::{CrossDocumentStrategy, DocumentEntry}; +/// +/// let docs = vec![ +/// DocumentEntry::new("doc1", "Manual A", tree1), +/// DocumentEntry::new("doc2", "Manual B", tree2), +/// ]; +/// +/// let strategy = CrossDocumentStrategy::new(inner_strategy) +/// .with_config(CrossDocumentConfig { +/// max_documents: 5, +/// max_results_per_doc: 2, +/// ..Default::default() +/// }); +/// ``` +pub struct CrossDocumentStrategy { + /// Inner strategy for searching individual documents. + inner: Box, + /// Configuration. + config: CrossDocumentConfig, + /// Documents to search. + documents: Vec, +} + +impl CrossDocumentStrategy { + /// Create a new cross-document strategy. + pub fn new(inner: Box) -> Self { + Self { + inner, + config: CrossDocumentConfig::default(), + documents: Vec::new(), + } + } + + /// Create with configuration. + pub fn with_config(mut self, config: CrossDocumentConfig) -> Self { + self.config = config; + self + } + + /// Add a document to search. + pub fn add_document(&mut self, doc: DocumentEntry) { + if self.documents.len() < self.config.max_documents { + self.documents.push(doc); + } + } + + /// Set documents to search. + pub fn with_documents(mut self, documents: Vec) -> Self { + self.documents = documents.into_iter().take(self.config.max_documents).collect(); + self + } + + /// Get the number of documents. + pub fn document_count(&self) -> usize { + self.documents.len() + } + + /// Search a single document and return results. + async fn search_document( + &self, + doc: &DocumentEntry, + context: &RetrievalContext, + ) -> DocumentResult { + let root_id = doc.tree.root(); + let children = doc.tree.children(root_id); + + // Evaluate top-level nodes to find entry points + let evaluations = self.inner.evaluate_nodes(&doc.tree, &children, context).await; + + // Collect results with scores above threshold + let mut scored_nodes: Vec<(NodeId, NodeEvaluation)> = children + .into_iter() + .zip(evaluations.into_iter()) + .filter(|(_, eval)| eval.score >= self.config.min_score) + .collect(); + + // Sort by score descending + scored_nodes.sort_by(|a, b| b.1.score.partial_cmp(&a.1.score).unwrap_or(std::cmp::Ordering::Equal)); + + // Limit results per document + scored_nodes.truncate(self.config.max_results_per_doc); + + let best_score = scored_nodes.first().map(|(_, e)| e.score).unwrap_or(0.0); + + DocumentResult { + doc_id: doc.id.clone(), + doc_title: doc.title.clone(), + evaluations: scored_nodes, + best_score, + } + } + + /// Merge results from all documents. + fn merge_results(&self, doc_results: Vec) -> Vec<(DocumentId, NodeId, NodeEvaluation)> { + match self.config.merge_strategy { + MergeStrategy::TopK => { + // Collect all results and sort by score + let mut all_results: Vec<_> = doc_results + .into_iter() + .flat_map(|doc| { + doc.evaluations.into_iter().map(move |(node_id, eval)| { + (doc.doc_id.clone(), node_id, eval) + }) + }) + .collect(); + + all_results.sort_by(|a, b| b.2.score.partial_cmp(&a.2.score).unwrap_or(std::cmp::Ordering::Equal)); + all_results.truncate(self.config.max_total_results); + all_results + } + + MergeStrategy::BestPerDocument => { + // Take the best result from each document + doc_results + .into_iter() + .filter_map(|doc| { + doc.evaluations.into_iter().next().map(|(node_id, eval)| { + (doc.doc_id, node_id, eval) + }) + }) + .take(self.config.max_total_results) + .collect() + } + + MergeStrategy::WeightedByRelevance => { + // Weight by document's best score + let max_doc_score = doc_results + .iter() + .map(|d| d.best_score) + .fold(0.0_f32, f32::max); + + let mut all_results: Vec<_> = doc_results + .into_iter() + .flat_map(|doc| { + let weight = if max_doc_score > 0.0 { + doc.best_score / max_doc_score + } else { + 1.0 + }; + doc.evaluations.into_iter().map(move |(node_id, mut eval)| { + eval.score *= weight; + (doc.doc_id.clone(), node_id, eval) + }) + }) + .collect(); + + all_results.sort_by(|a, b| b.2.score.partial_cmp(&a.2.score).unwrap_or(std::cmp::Ordering::Equal)); + all_results.truncate(self.config.max_total_results); + all_results + } + } + } +} + +#[async_trait] +impl RetrievalStrategy for CrossDocumentStrategy { + async fn evaluate_node( + &self, + tree: &DocumentTree, + node_id: NodeId, + context: &RetrievalContext, + ) -> NodeEvaluation { + // Delegate to inner strategy + self.inner.evaluate_node(tree, node_id, context).await + } + + async fn evaluate_nodes( + &self, + tree: &DocumentTree, + node_ids: &[NodeId], + context: &RetrievalContext, + ) -> Vec { + // Delegate to inner strategy + self.inner.evaluate_nodes(tree, node_ids, context).await + } + + fn name(&self) -> &'static str { + "cross_document" + } + + fn capabilities(&self) -> StrategyCapabilities { + let inner_caps = self.inner.capabilities(); + StrategyCapabilities { + uses_llm: inner_caps.uses_llm, + uses_embeddings: inner_caps.uses_embeddings, + supports_sufficiency: true, + typical_latency_ms: inner_caps.typical_latency_ms * self.documents.len().min(5) as u64, + } + } + + fn suitable_for_complexity(&self, complexity: QueryComplexity) -> bool { + // Cross-document is suitable for all complexity levels + matches!( + complexity, + QueryComplexity::Simple | QueryComplexity::Medium | QueryComplexity::Complex + ) + } + + fn estimate_cost(&self, node_count: usize) -> super::r#trait::StrategyCost { + let inner_cost = self.inner.estimate_cost(node_count); + super::r#trait::StrategyCost { + llm_calls: inner_cost.llm_calls * self.documents.len().min(self.config.max_documents), + tokens: inner_cost.tokens * self.documents.len().min(self.config.max_documents), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_config_default() { + let config = CrossDocumentConfig::default(); + assert_eq!(config.max_documents, 10); + assert_eq!(config.max_results_per_doc, 3); + assert_eq!(config.max_total_results, 10); + assert_eq!(config.merge_strategy, MergeStrategy::TopK); + } + + #[test] + fn test_merge_strategy_default() { + let strategy = MergeStrategy::default(); + assert!(matches!(strategy, MergeStrategy::TopK)); + } +} diff --git a/src/retrieval/strategy/hybrid.rs b/src/retrieval/strategy/hybrid.rs new file mode 100644 index 0000000..f301d97 --- /dev/null +++ b/src/retrieval/strategy/hybrid.rs @@ -0,0 +1,462 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Hybrid retrieval strategy combining BM25 pre-filtering with LLM refinement. +//! +//! This two-stage approach minimizes LLM calls while maintaining high accuracy: +//! 1. **BM25 Filter**: Fast keyword scoring to identify candidate nodes +//! 2. **LLM Refinement**: Semantic understanding of top candidates only + +use async_trait::async_trait; + +use super::r#trait::{NodeEvaluation, RetrievalStrategy, StrategyCapabilities}; +use crate::document::{DocumentTree, NodeId}; +use crate::retrieval::search::{Bm25Engine, FieldDocument, FieldWeights}; +use crate::retrieval::types::{NavigationDecision, QueryComplexity}; +use crate::retrieval::RetrievalContext; + +/// Configuration for hybrid retrieval. +#[derive(Debug, Clone)] +pub struct HybridConfig { + /// BM25 pre-filter: keep top N% of candidates. + pub pre_filter_ratio: f32, + /// BM25 pre-filter: minimum candidates to keep. + pub min_candidates: usize, + /// BM25 pre-filter: maximum candidates to pass to LLM. + pub max_candidates: usize, + /// Score threshold for automatic acceptance (skip LLM). + pub auto_accept_threshold: f32, + /// Score threshold for automatic rejection (skip LLM). + pub auto_reject_threshold: f32, + /// Weight for BM25 score in final scoring. + pub bm25_weight: f32, + /// Weight for LLM score in final scoring. + pub llm_weight: f32, + /// Whether to use BM25 for initial filtering. + pub use_pre_filter: bool, +} + +impl Default for HybridConfig { + fn default() -> Self { + Self { + pre_filter_ratio: 0.3, // Keep top 30% + min_candidates: 2, + max_candidates: 5, + auto_accept_threshold: 0.85, + auto_reject_threshold: 0.15, + bm25_weight: 0.4, + llm_weight: 0.6, + use_pre_filter: true, + } + } +} + +impl HybridConfig { + /// Create a new configuration. + pub fn new() -> Self { + Self::default() + } + + /// Set pre-filter ratio. + #[must_use] + pub fn with_pre_filter_ratio(mut self, ratio: f32) -> Self { + self.pre_filter_ratio = ratio.clamp(0.1, 1.0); + self + } + + /// Set candidate limits. + #[must_use] + pub fn with_candidate_limits(mut self, min: usize, max: usize) -> Self { + self.min_candidates = min; + self.max_candidates = max; + self + } + + /// Set score thresholds. + #[must_use] + pub fn with_thresholds(mut self, auto_accept: f32, auto_reject: f32) -> Self { + self.auto_accept_threshold = auto_accept; + self.auto_reject_threshold = auto_reject; + self + } + + /// Set scoring weights. + #[must_use] + pub fn with_weights(mut self, bm25: f32, llm: f32) -> Self { + self.bm25_weight = bm25; + self.llm_weight = llm; + self + } + + /// Disable pre-filtering (pass all to LLM). + #[must_use] + pub fn without_pre_filter(mut self) -> Self { + self.use_pre_filter = false; + self + } + + /// High-quality mode (more LLM calls). + #[must_use] + pub fn high_quality() -> Self { + Self { + pre_filter_ratio: 0.5, + min_candidates: 3, + max_candidates: 8, + auto_accept_threshold: 0.95, + auto_reject_threshold: 0.1, + bm25_weight: 0.3, + llm_weight: 0.7, + use_pre_filter: true, + } + } + + /// Low-cost mode (fewer LLM calls). + #[must_use] + pub fn low_cost() -> Self { + Self { + pre_filter_ratio: 0.2, + min_candidates: 1, + max_candidates: 3, + auto_accept_threshold: 0.75, + auto_reject_threshold: 0.25, + bm25_weight: 0.5, + llm_weight: 0.5, + use_pre_filter: true, + } + } +} + +/// Hybrid retrieval strategy combining BM25 and LLM. +/// +/// This strategy uses a two-stage approach: +/// 1. **BM25 Filter**: Quickly score all nodes using keyword matching +/// 2. **LLM Refinement**: Apply semantic understanding to top candidates +/// +/// This dramatically reduces LLM calls while maintaining accuracy. +/// +/// # Example +/// +/// ```rust,ignore +/// use vectorless::retrieval::strategy::{HybridStrategy, LlmStrategy}; +/// +/// let hybrid = HybridStrategy::new( +/// llm_strategy, +/// ).with_config(HybridConfig::high_quality()); +/// ``` +pub struct HybridStrategy { + /// LLM strategy for refinement. + llm_strategy: Box, + /// Configuration. + config: HybridConfig, + /// BM25 engine for pre-filtering. + bm25_engine: Option>, +} + +impl HybridStrategy { + /// Create a new hybrid strategy. + pub fn new(llm_strategy: Box) -> Self { + Self { + llm_strategy, + config: HybridConfig::default(), + bm25_engine: None, + } + } + + /// Create with configuration. + pub fn with_config(mut self, config: HybridConfig) -> Self { + self.config = config; + self + } + + /// Set configuration for high-quality mode. + pub fn with_high_quality(mut self) -> Self { + self.config = HybridConfig::high_quality(); + self + } + + /// Set configuration for low-cost mode. + pub fn with_low_cost(mut self) -> Self { + self.config = HybridConfig::low_cost(); + self + } + + /// Build BM25 index from tree nodes. + fn build_bm25_index(&mut self, tree: &DocumentTree, node_ids: &[NodeId]) { + let documents: Vec> = node_ids + .iter() + .enumerate() + .map(|(idx, &node_id)| { + if let Some(node) = tree.get(node_id) { + FieldDocument::new( + idx, + node.title.clone(), + node.summary.clone(), + node.content.clone(), + ) + } else { + FieldDocument::new(idx, String::new(), String::new(), String::new()) + } + }) + .collect(); + + if !documents.is_empty() { + self.bm25_engine = Some(Bm25Engine::fit_to_corpus(&documents)); + } + } + + /// Get BM25 scores for a query. + fn bm25_scores(&self, query: &str, node_count: usize) -> Vec<(usize, f32)> { + let engine = match &self.bm25_engine { + Some(e) => e, + None => return Vec::new(), + }; + + let results = engine.search_weighted(query, node_count); + results + .into_iter() + .map(|(idx, score)| (idx, score)) + .collect() + } + + /// Filter candidates using BM25 scores. + fn filter_candidates(&self, bm25_scores: &[(usize, f32)], total_count: usize) -> Vec { + if !self.config.use_pre_filter || total_count <= self.config.min_candidates { + return (0..total_count).collect(); + } + + // Calculate how many candidates to keep + let keep_count = ((total_count as f32 * self.config.pre_filter_ratio) as usize) + .max(self.config.min_candidates) + .min(self.config.max_candidates) + .min(total_count); + + // Sort by score and take top candidates + let mut sorted: Vec<_> = bm25_scores.to_vec(); + sorted.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + + sorted + .into_iter() + .take(keep_count) + .map(|(idx, _)| idx) + .collect() + } + + /// Combine BM25 and LLM scores. + fn combine_scores(&self, bm25_score: f32, llm_score: f32) -> f32 { + (bm25_score * self.config.bm25_weight + llm_score * self.config.llm_weight) + / (self.config.bm25_weight + self.config.llm_weight) + } +} + +#[async_trait] +impl RetrievalStrategy for HybridStrategy { + async fn evaluate_node( + &self, + tree: &DocumentTree, + node_id: NodeId, + context: &RetrievalContext, + ) -> NodeEvaluation { + // Delegate to LLM strategy for single node + self.llm_strategy.evaluate_node(tree, node_id, context).await + } + + async fn evaluate_nodes( + &self, + tree: &DocumentTree, + node_ids: &[NodeId], + context: &RetrievalContext, + ) -> Vec { + if node_ids.is_empty() { + return Vec::new(); + } + + // Build BM25 index if needed + let bm25_scores = self.bm25_scores(&context.query, node_ids.len()); + + // If no BM25 scores available, fall back to LLM only + if bm25_scores.is_empty() { + return self.llm_strategy.evaluate_nodes(tree, node_ids, context).await; + } + + // Create a score map for quick lookup + let score_map: std::collections::HashMap = bm25_scores + .iter() + .map(|(idx, score)| (*idx, *score)) + .collect(); + + // Normalize BM25 scores + let max_bm25 = score_map.values().cloned().fold(0.0_f32, f32::max); + let normalized_scores: std::collections::HashMap = if max_bm25 > 0.0 { + score_map + .iter() + .map(|(idx, score)| (*idx, *score / max_bm25)) + .collect() + } else { + score_map + }; + + // Check for auto-accept/reject candidates + let mut results = vec![NodeEvaluation::default(); node_ids.len()]; + let mut needs_llm = Vec::new(); + + for (idx, &node_id) in node_ids.iter().enumerate() { + let bm25_score = normalized_scores.get(&idx).copied().unwrap_or(0.0); + + if bm25_score >= self.config.auto_accept_threshold { + // High BM25 score: auto-accept without LLM + results[idx] = NodeEvaluation { + score: bm25_score, + decision: if tree.is_leaf(node_id) { + NavigationDecision::ThisIsTheAnswer + } else { + NavigationDecision::ExploreMore + }, + reasoning: Some(format!("Auto-accepted by BM25: {:.3}", bm25_score)), + }; + } else if bm25_score <= self.config.auto_reject_threshold { + // Low BM25 score: auto-reject without LLM + results[idx] = NodeEvaluation { + score: bm25_score, + decision: NavigationDecision::Skip, + reasoning: Some(format!("Auto-rejected by BM25: {:.3}", bm25_score)), + }; + } else { + // Need LLM refinement + needs_llm.push((idx, node_id, bm25_score)); + } + } + + // Filter candidates for LLM + let candidate_indices: std::collections::HashSet = self + .filter_candidates(&bm25_scores, node_ids.len()) + .into_iter() + .collect(); + + // Only send to LLM if in candidates and not already processed + let llm_nodes: Vec = needs_llm + .iter() + .filter(|(idx, _, _)| candidate_indices.contains(idx)) + .map(|(_, node_id, _)| *node_id) + .collect(); + + // Call LLM for filtered candidates + if !llm_nodes.is_empty() { + let llm_results = self.llm_strategy.evaluate_nodes(tree, &llm_nodes, context).await; + + // Map LLM results back with combined scores + let mut llm_iter = llm_results.into_iter(); + for (idx, node_id, bm25_score) in &needs_llm { + if candidate_indices.contains(idx) { + if let Some(llm_eval) = llm_iter.next() { + let combined_score = self.combine_scores(*bm25_score, llm_eval.score); + results[*idx] = NodeEvaluation { + score: combined_score, + decision: llm_eval.decision, + reasoning: Some(format!( + "Hybrid: BM25={:.2}, LLM={:.2}, Combined={:.2}", + bm25_score, llm_eval.score, combined_score + )), + }; + } + } else { + // Not in LLM candidates, use BM25 only + results[*idx] = NodeEvaluation { + score: *bm25_score, + decision: if *bm25_score > 0.5 { + NavigationDecision::ExploreMore + } else { + NavigationDecision::Skip + }, + reasoning: Some(format!("BM25 only (filtered): {:.3}", bm25_score)), + }; + } + } + } else { + // No LLM calls needed, use BM25 for all remaining + for (idx, _, bm25_score) in &needs_llm { + results[*idx] = NodeEvaluation { + score: *bm25_score, + decision: if *bm25_score > 0.5 { + NavigationDecision::ExploreMore + } else { + NavigationDecision::Skip + }, + reasoning: Some(format!("BM25 only: {:.3}", bm25_score)), + }; + } + } + + results + } + + fn name(&self) -> &'static str { + "hybrid" + } + + fn capabilities(&self) -> StrategyCapabilities { + let llm_caps = self.llm_strategy.capabilities(); + StrategyCapabilities { + uses_llm: llm_caps.uses_llm, + uses_embeddings: false, // BM25 doesn't use embeddings + supports_sufficiency: llm_caps.supports_sufficiency, + typical_latency_ms: llm_caps.typical_latency_ms / 2, // Faster due to pre-filtering + } + } + + fn suitable_for_complexity(&self, complexity: QueryComplexity) -> bool { + matches!( + complexity, + QueryComplexity::Simple | QueryComplexity::Medium | QueryComplexity::Complex + ) + } + + fn estimate_cost(&self, node_count: usize) -> super::r#trait::StrategyCost { + let llm_cost = self.llm_strategy.estimate_cost(node_count); + + // Estimate reduced LLM calls due to pre-filtering + let filtered_count = ((node_count as f32 * self.config.pre_filter_ratio) as usize) + .max(self.config.min_candidates) + .min(self.config.max_candidates); + + // Account for auto-accept/reject + let estimated_llm_calls = (filtered_count as f32 * 0.5) as usize; + + super::r#trait::StrategyCost { + llm_calls: estimated_llm_calls.min(llm_cost.llm_calls), + tokens: estimated_llm_calls * 200, // Approximate tokens per call + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_config_default() { + let config = HybridConfig::default(); + assert!((config.pre_filter_ratio - 0.3).abs() < f32::EPSILON); + assert_eq!(config.min_candidates, 2); + assert_eq!(config.max_candidates, 5); + assert!((config.bm25_weight - 0.4).abs() < f32::EPSILON); + assert!((config.llm_weight - 0.6).abs() < f32::EPSILON); + } + + #[test] + fn test_config_presets() { + let high = HybridConfig::high_quality(); + assert!(high.max_candidates > HybridConfig::default().max_candidates); + + let low = HybridConfig::low_cost(); + assert!(low.max_candidates < HybridConfig::default().max_candidates); + } + + #[test] + fn test_combine_scores() { + let strategy = HybridStrategy::new(Box::new(crate::retrieval::strategy::KeywordStrategy::new())); + let combined = strategy.combine_scores(0.8, 0.6); + + // 0.8 * 0.4 + 0.6 * 0.6 = 0.32 + 0.36 = 0.68 + assert!((combined - 0.68).abs() < 0.01); + } +} diff --git a/src/retrieval/strategy/mod.rs b/src/retrieval/strategy/mod.rs index 93c55df..345e90f 100644 --- a/src/retrieval/strategy/mod.rs +++ b/src/retrieval/strategy/mod.rs @@ -2,13 +2,31 @@ // SPDX-License-Identifier: Apache-2.0 //! Retrieval strategies for different query types. +//! +//! This module provides several retrieval strategies: +//! +//! - **KeywordStrategy**: Fast keyword matching using TF-IDF +//! - **SemanticStrategy**: Embedding-based semantic similarity +//! - **LlmStrategy**: LLM-powered reasoning with ToC context +//! - **HybridStrategy**: BM25 pre-filter + LLM refinement (recommended) +//! - **CrossDocumentStrategy**: Multi-document retrieval with result aggregation +//! - **PageRangeStrategy**: Filter by page range before retrieval +mod cross_document; +mod hybrid; mod keyword; mod llm; +mod page_range; mod semantic; mod r#trait; +pub use cross_document::{ + CrossDocumentConfig, CrossDocumentStrategy, DocumentEntry, DocumentId, DocumentResult, + MergeStrategy, +}; +pub use hybrid::{HybridConfig, HybridStrategy}; pub use keyword::KeywordStrategy; pub use llm::LlmStrategy; +pub use page_range::{PageRange, PageRangeConfig, PageRangeStrategy}; pub use semantic::SemanticStrategy; pub use r#trait::{NodeEvaluation, RetrievalStrategy, StrategyCapabilities, StrategyCost}; diff --git a/src/retrieval/strategy/page_range.rs b/src/retrieval/strategy/page_range.rs new file mode 100644 index 0000000..e362fb5 --- /dev/null +++ b/src/retrieval/strategy/page_range.rs @@ -0,0 +1,414 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Page-range retrieval strategy. +//! +//! Filters document nodes by page range before applying an inner strategy. +//! Useful when the user knows approximately where the information is located. + +use async_trait::async_trait; + +use super::r#trait::{NodeEvaluation, RetrievalStrategy, StrategyCapabilities}; +use crate::document::{DocumentTree, NodeId}; +use crate::retrieval::types::{NavigationDecision, QueryComplexity}; +use crate::retrieval::RetrievalContext; + +/// A page range for filtering. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PageRange { + /// Start page (inclusive, 1-indexed). + pub start: usize, + /// End page (inclusive). + pub end: usize, +} + +impl PageRange { + /// Create a new page range. + pub fn new(start: usize, end: usize) -> Self { + Self { start, end } + } + + /// Create a range from a single page. + pub fn single(page: usize) -> Self { + Self { start: page, end: page } + } + + /// Create a range starting from a page to the end. + pub fn from(start: usize) -> Self { + Self { start, end: usize::MAX } + } + + /// Create a range from the beginning to a page. + pub fn until(end: usize) -> Self { + Self { start: 1, end } + } + + /// Check if a page is within this range. + pub fn contains(&self, page: usize) -> bool { + page >= self.start && page <= self.end + } + + /// Check if this range overlaps with another. + pub fn overlaps(&self, other: &PageRange) -> bool { + self.start <= other.end && other.start <= self.end + } + + /// Get the number of pages in this range. + pub fn len(&self) -> usize { + if self.end == usize::MAX { + usize::MAX + } else { + self.end.saturating_sub(self.start) + 1 + } + } + + /// Check if this is an empty range. + pub fn is_empty(&self) -> bool { + self.start > self.end + } +} + +impl Default for PageRange { + fn default() -> Self { + Self { start: 1, end: usize::MAX } + } +} + +/// Configuration for page-range retrieval. +#[derive(Debug, Clone)] +pub struct PageRangeConfig { + /// The page range to search within. + pub range: PageRange, + /// Whether to include nodes that span across the boundary. + pub include_boundary_nodes: bool, + /// Whether to expand the range slightly for context. + pub expand_context_pages: usize, + /// Minimum overlap ratio for a node to be included. + pub min_overlap_ratio: f32, +} + +impl Default for PageRangeConfig { + fn default() -> Self { + Self { + range: PageRange::default(), + include_boundary_nodes: true, + expand_context_pages: 0, + min_overlap_ratio: 0.1, + } + } +} + +impl PageRangeConfig { + /// Create a new configuration with a page range. + pub fn new(range: PageRange) -> Self { + Self { + range, + ..Default::default() + } + } + + /// Set the page range. + #[must_use] + pub fn with_range(mut self, start: usize, end: usize) -> Self { + self.range = PageRange::new(start, end); + self + } + + /// Include nodes that span the boundary. + #[must_use] + pub fn with_boundary_nodes(mut self, include: bool) -> Self { + self.include_boundary_nodes = include; + self + } + + /// Expand the range by N pages for context. + #[must_use] + pub fn with_context_expansion(mut self, pages: usize) -> Self { + self.expand_context_pages = pages; + self + } +} + +/// Page-range retrieval strategy. +/// +/// Filters nodes by their page location before delegating to an inner strategy. +/// This is useful when: +/// - The user knows approximately where information is located +/// - Searching large PDFs where certain sections are known +/// - Implementing "search within pages X-Y" functionality +/// +/// # Example +/// +/// ```rust,ignore +/// use vectorless::retrieval::strategy::{PageRangeStrategy, KeywordStrategy, PageRange}; +/// +/// // Search only pages 10-20 +/// let strategy = PageRangeStrategy::new( +/// Box::new(KeywordStrategy::new()), +/// PageRange::new(10, 20), +/// ); +/// +/// // Search from page 50 onwards +/// let strategy = PageRangeStrategy::new( +/// Box::new(LlmStrategy::new(client)), +/// PageRange::from(50), +/// ); +/// ``` +pub struct PageRangeStrategy { + /// Inner strategy for filtered nodes. + inner: Box, + /// Configuration. + config: PageRangeConfig, +} + +impl PageRangeStrategy { + /// Create a new page-range strategy. + pub fn new(inner: Box, range: PageRange) -> Self { + Self { + inner, + config: PageRangeConfig::new(range), + } + } + + /// Create with configuration. + pub fn with_config(inner: Box, config: PageRangeConfig) -> Self { + Self { inner, config } + } + + /// Set whether to include boundary nodes. + #[must_use] + pub fn with_boundary_nodes(mut self, include: bool) -> Self { + self.config.include_boundary_nodes = include; + self + } + + /// Set context expansion pages. + #[must_use] + pub fn with_context_expansion(mut self, pages: usize) -> Self { + self.config.expand_context_pages = pages; + self + } + + /// Get the effective range after context expansion. + fn effective_range(&self) -> PageRange { + if self.config.expand_context_pages == 0 { + return self.config.range; + } + + PageRange { + start: self.config.range.start.saturating_sub(self.config.expand_context_pages), + end: self.config.range.end.saturating_add(self.config.expand_context_pages), + } + } + + /// Check if a node is within the page range. + fn is_node_in_range(&self, tree: &DocumentTree, node_id: NodeId) -> bool { + let effective_range = self.effective_range(); + + if let Some(node) = tree.get(node_id) { + // Check if node has page information + let (start_page, end_page) = node + .start_page + .zip(node.end_page) + .unwrap_or((1, usize::MAX)); + + let node_range = PageRange::new(start_page, end_page); + + // Check for overlap + if effective_range.overlaps(&node_range) { + // Calculate overlap ratio + let overlap_start = effective_range.start.max(node_range.start); + let overlap_end = effective_range.end.min(node_range.end); + + if overlap_start <= overlap_end { + let overlap_pages = overlap_end - overlap_start + 1; + let node_pages = node_range.len(); + + let ratio = overlap_pages as f32 / node_pages as f32; + return ratio >= self.config.min_overlap_ratio; + } + } + } + + // If no page info, include the node (conservative approach) + true + } + + /// Filter nodes by page range. + fn filter_by_range( + &self, + tree: &DocumentTree, + node_ids: &[NodeId], + ) -> (Vec<(usize, NodeId)>, Vec) { + let mut included = Vec::new(); + let mut excluded = Vec::new(); + + for (idx, &node_id) in node_ids.iter().enumerate() { + if self.is_node_in_range(tree, node_id) { + included.push((idx, node_id)); + } else { + excluded.push(idx); + } + } + + (included, excluded) + } +} + +#[async_trait] +impl RetrievalStrategy for PageRangeStrategy { + async fn evaluate_node( + &self, + tree: &DocumentTree, + node_id: NodeId, + context: &RetrievalContext, + ) -> NodeEvaluation { + // Check if node is in range + if !self.is_node_in_range(tree, node_id) { + return NodeEvaluation { + score: 0.0, + decision: NavigationDecision::Skip, + reasoning: Some("Node outside page range".to_string()), + }; + } + + // Delegate to inner strategy + self.inner.evaluate_node(tree, node_id, context).await + } + + async fn evaluate_nodes( + &self, + tree: &DocumentTree, + node_ids: &[NodeId], + context: &RetrievalContext, + ) -> Vec { + if node_ids.is_empty() { + return Vec::new(); + } + + // Filter nodes by page range + let (included, excluded) = self.filter_by_range(tree, node_ids); + + // Create result vector with default values + let mut results = vec![NodeEvaluation::default(); node_ids.len()]; + + // Mark excluded nodes as skipped + for idx in &excluded { + results[*idx] = NodeEvaluation { + score: 0.0, + decision: NavigationDecision::Skip, + reasoning: Some(format!( + "Outside page range {}-{}", + self.config.range.start, self.config.range.end + )), + }; + } + + // Evaluate included nodes with inner strategy + if !included.is_empty() { + let included_ids: Vec = included.iter().map(|(_, id)| *id).collect(); + let inner_results = self.inner.evaluate_nodes(tree, &included_ids, context).await; + + // Map results back to original positions + for ((orig_idx, _), eval) in included.into_iter().zip(inner_results.into_iter()) { + results[orig_idx] = eval; + } + } + + results + } + + fn name(&self) -> &'static str { + "page_range" + } + + fn capabilities(&self) -> StrategyCapabilities { + let inner_caps = self.inner.capabilities(); + StrategyCapabilities { + uses_llm: inner_caps.uses_llm, + uses_embeddings: inner_caps.uses_embeddings, + supports_sufficiency: inner_caps.supports_sufficiency, + typical_latency_ms: inner_caps.typical_latency_ms, // Same as inner + } + } + + fn suitable_for_complexity(&self, complexity: QueryComplexity) -> bool { + self.inner.suitable_for_complexity(complexity) + } + + fn estimate_cost(&self, node_count: usize) -> super::r#trait::StrategyCost { + // Estimate that only a fraction of nodes are in range + let estimated_in_range = (node_count as f32 * 0.3) as usize; + self.inner.estimate_cost(estimated_in_range.max(1)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_page_range_creation() { + let range = PageRange::new(10, 20); + assert_eq!(range.start, 10); + assert_eq!(range.end, 20); + } + + #[test] + fn test_page_range_contains() { + let range = PageRange::new(10, 20); + assert!(range.contains(10)); + assert!(range.contains(15)); + assert!(range.contains(20)); + assert!(!range.contains(9)); + assert!(!range.contains(21)); + } + + #[test] + fn test_page_range_single() { + let range = PageRange::single(5); + assert!(range.contains(5)); + assert!(!range.contains(4)); + assert!(!range.contains(6)); + } + + #[test] + fn test_page_range_from() { + let range = PageRange::from(10); + assert!(range.contains(10)); + assert!(range.contains(100)); + assert!(range.contains(usize::MAX)); + assert!(!range.contains(9)); + } + + #[test] + fn test_page_range_until() { + let range = PageRange::until(20); + assert!(range.contains(1)); + assert!(range.contains(20)); + assert!(!range.contains(21)); + } + + #[test] + fn test_page_range_overlaps() { + let r1 = PageRange::new(10, 20); + let r2 = PageRange::new(15, 25); + let r3 = PageRange::new(21, 30); + + assert!(r1.overlaps(&r2)); + assert!(!r1.overlaps(&r3)); + } + + #[test] + fn test_page_range_len() { + let range = PageRange::new(10, 20); + assert_eq!(range.len(), 11); + } + + #[test] + fn test_config_default() { + let config = PageRangeConfig::default(); + assert_eq!(config.range.start, 1); + assert!(config.include_boundary_nodes); + } +} diff --git a/src/retrieval/types.rs b/src/retrieval/types.rs index 82ee550..3057b7d 100644 --- a/src/retrieval/types.rs +++ b/src/retrieval/types.rs @@ -41,6 +41,15 @@ pub enum StrategyPreference { /// Force LLM strategy (deep reasoning). ForceLlm, + + /// Force hybrid strategy (BM25 + LLM refinement). + ForceHybrid, + + /// Force cross-document strategy (multi-document retrieval). + ForceCrossDocument, + + /// Force page-range strategy (filter by page range). + ForcePageRange, } impl Default for StrategyPreference { diff --git a/vectorless.example.toml b/vectorless.example.toml index 309b324..dd0a9a9 100644 --- a/vectorless.example.toml +++ b/vectorless.example.toml @@ -163,6 +163,37 @@ similarity_threshold = 0.5 high_similarity_threshold = 0.8 low_similarity_threshold = 0.3 +# Hybrid Strategy Configuration (BM25 + LLM refinement) +# Recommended for most use cases - reduces LLM calls while maintaining accuracy +[retrieval.strategy.hybrid] +enabled = true +pre_filter_ratio = 0.3 # Keep top 30% of BM25 candidates +min_candidates = 2 # Minimum candidates to pass to LLM +max_candidates = 5 # Maximum candidates for LLM refinement +auto_accept_threshold = 0.85 # BM25 score for auto-accept (skip LLM) +auto_reject_threshold = 0.15 # BM25 score for auto-reject (skip LLM) +bm25_weight = 0.4 # Weight for BM25 score in final scoring +llm_weight = 0.6 # Weight for LLM score in final scoring + +# Cross-Document Retrieval Configuration +# For searching across multiple documents simultaneously +[retrieval.strategy.cross_document] +enabled = true +max_documents = 10 # Maximum documents to search +max_results_per_doc = 3 # Maximum results per document +max_total_results = 10 # Maximum total results +min_score = 0.3 # Minimum score threshold +merge_strategy = "TopK" # TopK | BestPerDocument | WeightedByRelevance +parallel_search = true # Search documents in parallel + +# Page-Range Strategy Configuration +# For filtering by page range before retrieval +[retrieval.strategy.page_range] +enabled = true +include_boundary_nodes = true # Include nodes spanning across boundary +expand_context_pages = 0 # Expand range by N pages for context +min_overlap_ratio = 0.1 # Minimum overlap ratio for node inclusion + [retrieval.content] enabled = true token_budget = 4000