diff --git a/crates/tempyr-cli/src/commands/ask.rs b/crates/tempyr-cli/src/commands/ask.rs index 0310d18..3501b17 100644 --- a/crates/tempyr-cli/src/commands/ask.rs +++ b/crates/tempyr-cli/src/commands/ask.rs @@ -1,7 +1,7 @@ +use crate::commands::semantic::SemanticSearchRuntime; use crate::config::ProjectContext; use tempyr_core::graph::Graph; -use tempyr_index::hybrid::{RetrievalConfig, hybrid_retrieve}; -use tempyr_index::indexer::Index; +use tempyr_index::hybrid::RetrievalConfig; pub fn run( ctx: &ProjectContext, @@ -9,15 +9,14 @@ pub fn run( root: Option<&str>, json: bool, ) -> anyhow::Result<()> { - let index_path = ctx.queryable_index_path()?; let graph = Graph::load_from_directory(&ctx.graph_dir, ctx.schema.clone())?; - let index = Index::open(&index_path)?; let config = RetrievalConfig { token_budget: 8000, ..RetrievalConfig::standard() }; - let results = hybrid_retrieve(&index, &graph, question, root, &config, None)?; + let mut semantic_search = SemanticSearchRuntime::new(ctx)?; + let results = semantic_search.hybrid_retrieve(&graph, question, root, config)?; if results.is_empty() { println!("No relevant context found for: {question}"); diff --git a/crates/tempyr-cli/src/commands/context.rs b/crates/tempyr-cli/src/commands/context.rs index 24f504c..bca4d4a 100644 --- a/crates/tempyr-cli/src/commands/context.rs +++ b/crates/tempyr-cli/src/commands/context.rs @@ -1,7 +1,7 @@ +use crate::commands::semantic::SemanticSearchRuntime; use crate::config::ProjectContext; use tempyr_core::graph::Graph; -use tempyr_index::hybrid::{RetrievalConfig, hybrid_retrieve}; -use tempyr_index::indexer::Index; +use tempyr_index::hybrid::RetrievalConfig; pub fn run( ctx: &ProjectContext, @@ -10,15 +10,14 @@ pub fn run( budget: usize, json: bool, ) -> anyhow::Result<()> { - let index_path = ctx.queryable_index_path()?; let graph = Graph::load_from_directory(&ctx.graph_dir, ctx.schema.clone())?; - let index = Index::open(&index_path)?; let config = RetrievalConfig { token_budget: budget, ..RetrievalConfig::standard() }; - let results = hybrid_retrieve(&index, &graph, query, root, &config, None)?; + let mut semantic_search = SemanticSearchRuntime::new(ctx)?; + let results = semantic_search.hybrid_retrieve(&graph, query, root, config)?; if json { let json_results: Vec<_> = results @@ -29,6 +28,7 @@ pub fn run( "combined_score": r.combined_score, "structural_score": r.structural_score, "bm25_score": r.bm25_score, + "vector_score": r.vector_score, }) }) .collect(); @@ -50,8 +50,12 @@ pub fn run( .bm25_score .map(|s| format!(" bm25={s:.2}")) .unwrap_or_default(); + let vector = result + .vector_score + .map(|s| format!(" vec={s:.2}")) + .unwrap_or_default(); println!( - "{} (score={:.3}{structural}{bm25})", + "{} (score={:.3}{structural}{bm25}{vector})", result.node_id, result.combined_score ); } diff --git a/crates/tempyr-cli/src/commands/mod.rs b/crates/tempyr-cli/src/commands/mod.rs index dac8dd4..cac1712 100644 --- a/crates/tempyr-cli/src/commands/mod.rs +++ b/crates/tempyr-cli/src/commands/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod process_utils; pub mod rename; pub mod render_cmd; pub mod search; +pub mod semantic; pub mod status_cmd; pub mod traverse; pub mod update; diff --git a/crates/tempyr-cli/src/commands/render_cmd.rs b/crates/tempyr-cli/src/commands/render_cmd.rs index 53e20fc..78b87a6 100644 --- a/crates/tempyr-cli/src/commands/render_cmd.rs +++ b/crates/tempyr-cli/src/commands/render_cmd.rs @@ -2,6 +2,7 @@ use std::path::Path; use chrono::NaiveDate; +use crate::commands::semantic::SemanticSearchRuntime; use crate::config::ProjectContext; use tempyr_core::graph::Graph; use tempyr_core::temporal::TemporalFilter; @@ -10,6 +11,61 @@ const BUILTIN_PRD: &str = include_str!("../../../../templates/prd.toml"); const BUILTIN_TDD: &str = include_str!("../../../../templates/tdd.toml"); const BUILTIN_TASK_PROMPT: &str = include_str!("../../../../templates/task-prompt.toml"); +struct RenderSemanticSearch<'a> { + ctx: &'a ProjectContext, + graph: &'a Graph, + runtime: Option, +} + +impl<'a> RenderSemanticSearch<'a> { + fn new(ctx: &'a ProjectContext, graph: &'a Graph) -> Self { + Self { + ctx, + graph, + runtime: None, + } + } + + fn runtime(&mut self) -> tempyr_render::Result<&mut SemanticSearchRuntime> { + if self.runtime.is_none() { + let runtime = SemanticSearchRuntime::new(self.ctx).map_err(render_error)?; + self.runtime = Some(runtime); + } + Ok(self.runtime.as_mut().expect("semantic runtime initialized")) + } +} + +impl tempyr_render::SemanticSearchProvider for RenderSemanticSearch<'_> { + fn search( + &mut self, + request: &tempyr_render::SemanticSearchRequest, + ) -> tempyr_render::Result> { + let graph = self.graph; + let results = self + .runtime()? + .vector_search( + graph, + &request.query, + request.max_results, + request.target_type.as_deref(), + request.min_similarity, + ) + .map_err(render_error)?; + + Ok(results + .into_iter() + .map(|result| tempyr_render::SemanticSearchHit { + node_id: result.node_id, + score: result.similarity, + }) + .collect()) + } +} + +fn render_error(err: anyhow::Error) -> tempyr_render::RenderError { + tempyr_render::RenderError::General(err.to_string()) +} + pub fn run( ctx: &ProjectContext, template_name: &str, @@ -35,8 +91,17 @@ pub fn run( .tempyr_dir .join("render") .join(format!("{template_name}.toml")); + let mut semantic_search = RenderSemanticSearch::new(ctx, &graph); let result = if local_path.exists() { - tempyr_render::render(&graph, &local_path, root_id, &temporal_filter)? + tempyr_render::render_with_options( + &graph, + &local_path, + root_id, + &temporal_filter, + tempyr_render::RenderOptions { + semantic_search: Some(&mut semantic_search), + }, + )? } else { // Use built-in template let template_toml = match template_name { @@ -47,7 +112,15 @@ pub fn run( "Unknown template: '{template_name}'. Available: prd, tdd, task-prompt (or place a custom template in .tempyr/render/)" ), }; - tempyr_render::render_from_str(&graph, template_toml, root_id, &temporal_filter)? + tempyr_render::render_from_str_with_options( + &graph, + template_toml, + root_id, + &temporal_filter, + tempyr_render::RenderOptions { + semantic_search: Some(&mut semantic_search), + }, + )? }; if let Some(output_path) = output { diff --git a/crates/tempyr-cli/src/commands/semantic.rs b/crates/tempyr-cli/src/commands/semantic.rs new file mode 100644 index 0000000..3f397a1 --- /dev/null +++ b/crates/tempyr-cli/src/commands/semantic.rs @@ -0,0 +1,62 @@ +use crate::config::ProjectContext; + +use tempyr_core::graph::Graph; +use tempyr_index::embeddings::{self, EmbeddingStore}; +use tempyr_index::hybrid::{HybridResult, RetrievalConfig}; +use tempyr_index::indexer::Index; +use tempyr_index::semantic::SemanticSearchEngine; +use tempyr_index::vector::VectorSearchResult; + +/// Runtime state for commands that need semantic search over the graph. +pub struct SemanticSearchRuntime { + engine: SemanticSearchEngine, + runtime: tokio::runtime::Runtime, +} + +impl SemanticSearchRuntime { + pub fn new(ctx: &ProjectContext) -> anyhow::Result { + let index_path = ctx.queryable_index_path()?; + let index = Index::open(&index_path)?; + let resolved = ctx.resolved_embedding_config()?; + let store_path = ctx.embedding_store_path( + &resolved.provider, + resolved.model.as_deref(), + Some(resolved.dimensions), + ); + let store = EmbeddingStore::open_or_create(&store_path)?; + let provider = embeddings::create_provider_from_resolved(&resolved)?; + let runtime = tokio::runtime::Runtime::new()?; + let engine = SemanticSearchEngine::new(index, store, provider); + + Ok(Self { engine, runtime }) + } + + pub fn vector_search( + &mut self, + graph: &Graph, + query: &str, + max_results: usize, + node_type: Option<&str>, + min_similarity: Option, + ) -> anyhow::Result> { + Ok(self.runtime.block_on(self.engine.vector_search( + graph, + query, + max_results, + node_type, + min_similarity, + ))?) + } + + pub fn hybrid_retrieve( + &mut self, + graph: &Graph, + query: &str, + root: Option<&str>, + config: RetrievalConfig, + ) -> anyhow::Result> { + Ok(self + .runtime + .block_on(self.engine.hybrid_retrieve(graph, query, root, config))?) + } +} diff --git a/crates/tempyr-cli/src/commands/vsearch.rs b/crates/tempyr-cli/src/commands/vsearch.rs index 2f47981..f99c0f1 100644 --- a/crates/tempyr-cli/src/commands/vsearch.rs +++ b/crates/tempyr-cli/src/commands/vsearch.rs @@ -1,13 +1,6 @@ +use crate::commands::semantic::SemanticSearchRuntime; use crate::config::ProjectContext; -use tempyr_index::embeddings::{self, EmbeddingStore, InputType}; -use tempyr_index::indexer::Index; - -fn should_use_legacy_embeddings( - store_embedding_count: usize, - legacy_embedding_count: usize, -) -> bool { - legacy_embedding_count > 0 && legacy_embedding_count > store_embedding_count -} +use tempyr_core::graph::Graph; pub fn run( ctx: &ProjectContext, @@ -16,44 +9,9 @@ pub fn run( node_type: Option<&str>, json: bool, ) -> anyhow::Result<()> { - let index_path = ctx.queryable_index_path()?; - let index = Index::open(&index_path)?; - let resolved = ctx.resolved_embedding_config()?; - let store_path = ctx.embedding_store_path( - &resolved.provider, - resolved.model.as_deref(), - Some(resolved.dimensions), - ); - let store = EmbeddingStore::open_or_create(&store_path)?; - - // Check if embeddings exist - let store_embedding_count = store.count_embeddings_for_index(&index, node_type)?; - let legacy_embedding_count = index.embedding_count_for_node_type(node_type)?; - let use_legacy_index_embeddings = - should_use_legacy_embeddings(store_embedding_count, legacy_embedding_count); - if store_embedding_count == 0 && legacy_embedding_count == 0 { - anyhow::bail!( - "No embeddings found. Run `tempyr index rebuild` with an embedding \ - API key set in Tempyr's shared worktree env, `.env.local`, or \ - your shell environment (VOYAGE_API_KEY or GEMINI_API_KEY)." - ); - } - - // Embed the query - let provider = embeddings::create_provider_from_resolved(&resolved)?; - - let rt = tokio::runtime::Runtime::new()?; - let query_embeddings = rt.block_on(provider.embed(&[query.to_string()], InputType::Query))?; - - if query_embeddings.is_empty() { - anyhow::bail!("Failed to embed query"); - } - - let results = if use_legacy_index_embeddings { - index.vector_search(&query_embeddings[0], max_results, node_type)? - } else { - store.vector_search(&index, &query_embeddings[0], max_results, node_type)? - }; + let graph = Graph::load_from_directory(&ctx.graph_dir, ctx.schema.clone())?; + let mut semantic_search = SemanticSearchRuntime::new(ctx)?; + let results = semantic_search.vector_search(&graph, query, max_results, node_type, None)?; if json { let json_results: Vec<_> = results @@ -78,23 +36,3 @@ pub fn run( Ok(()) } - -#[cfg(test)] -mod tests { - use super::should_use_legacy_embeddings; - - #[test] - fn prefers_legacy_when_shared_store_is_empty() { - assert!(should_use_legacy_embeddings(0, 3)); - } - - #[test] - fn prefers_shared_store_when_coverage_is_equal() { - assert!(!should_use_legacy_embeddings(2, 2)); - } - - #[test] - fn prefers_shared_store_when_it_has_more_coverage() { - assert!(!should_use_legacy_embeddings(3, 2)); - } -} diff --git a/crates/tempyr-index/src/embeddings.rs b/crates/tempyr-index/src/embeddings.rs index 1157320..05ec829 100644 --- a/crates/tempyr-index/src/embeddings.rs +++ b/crates/tempyr-index/src/embeddings.rs @@ -25,6 +25,11 @@ pub trait EmbeddingProvider: Send + Sync { /// Provider name for display. fn name(&self) -> &str; + + /// Stable cache identity for vectors produced by this provider. + fn fingerprint(&self) -> String { + format!("provider={};dimensions={}", self.name(), self.dimensions()) + } } /// Whether the text is a query or a document (some models optimize differently). @@ -136,6 +141,13 @@ impl EmbeddingProvider for VoyageClient { fn name(&self) -> &str { "voyage" } + + fn fingerprint(&self) -> String { + format!( + "provider=voyage;model={};dimensions={}", + self.model, self.dimensions + ) + } } // Google Gemini @@ -320,6 +332,13 @@ impl EmbeddingProvider for GeminiClient { fn name(&self) -> &str { "gemini" } + + fn fingerprint(&self) -> String { + format!( + "provider=gemini;model={};dimensions={}", + self.model, self.dimensions + ) + } } // Local fastembed fallback @@ -373,6 +392,13 @@ impl EmbeddingProvider for FastembedClient { fn name(&self) -> &str { "local (all-MiniLM-L6-v2)" } + + fn fingerprint(&self) -> String { + format!( + "provider=local;model={LOCAL_MODEL};dimensions={}", + self.dimensions() + ) + } } // Provider factory @@ -633,6 +659,7 @@ pub struct EmbeddingStore { impl EmbeddingStore { const SQLITE_BATCH_SIZE: usize = 900; + const PROVIDER_FINGERPRINT_KEY: &str = "provider_fingerprint"; pub fn open_or_create(path: &Path) -> Result { if let Some(parent) = path.parent() { @@ -655,11 +682,54 @@ impl EmbeddingStore { content_hash TEXT PRIMARY KEY, embedding BLOB NOT NULL ); + CREATE TABLE IF NOT EXISTS embedding_store_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); ", )?; Ok(()) } + fn meta_value(&self, key: &str) -> Result> { + let result = self.conn.query_row( + "SELECT value FROM embedding_store_meta WHERE key = ?1", + [key], + |row| row.get(0), + ); + + match result { + Ok(value) => Ok(Some(value)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(e.into()), + } + } + + fn set_meta_value(&self, key: &str, value: &str) -> Result<()> { + self.conn.execute( + "INSERT OR REPLACE INTO embedding_store_meta (key, value) VALUES (?1, ?2)", + rusqlite::params![key, value], + )?; + Ok(()) + } + + fn clear_embeddings(&self) -> Result<()> { + self.conn.execute("DELETE FROM embeddings", [])?; + Ok(()) + } + + pub fn ensure_provider_fingerprint(&self, provider: &dyn EmbeddingProvider) -> Result<()> { + let fingerprint = provider.fingerprint(); + match self.meta_value(Self::PROVIDER_FINGERPRINT_KEY)? { + Some(existing) if existing == fingerprint => Ok(()), + Some(_) => { + self.clear_embeddings()?; + self.set_meta_value(Self::PROVIDER_FINGERPRINT_KEY, &fingerprint) + } + None => self.set_meta_value(Self::PROVIDER_FINGERPRINT_KEY, &fingerprint), + } + } + pub fn has_embedding(&self, content_hash: &str) -> Result { let count: i64 = self.conn.query_row( "SELECT COUNT(*) FROM embeddings WHERE content_hash = ?1", @@ -793,6 +863,8 @@ pub async fn embed_graph( graph: &Graph, provider: &dyn EmbeddingProvider, ) -> Result { + store.ensure_provider_fingerprint(provider)?; + let mut to_embed: Vec<(String, String, String)> = Vec::new(); // (id, hash, text) let mut seen_hashes = HashSet::new(); @@ -914,6 +986,26 @@ reverse = "dependency_of" } } + struct NamedProvider { + name: &'static str, + embeddings: Vec>, + } + + #[async_trait] + impl EmbeddingProvider for NamedProvider { + async fn embed(&self, _texts: &[String], _input_type: InputType) -> Result>> { + Ok(self.embeddings.clone()) + } + + fn dimensions(&self) -> usize { + self.embeddings.first().map_or(0, Vec::len) + } + + fn name(&self) -> &str { + self.name + } + } + #[test] fn embed_graph_rejects_mismatched_provider_output_count() { let tmp = tempfile::tempdir().unwrap(); @@ -1011,6 +1103,64 @@ reverse = "dependency_of" assert_eq!(store.count().unwrap(), 1); } + #[test] + fn embed_graph_replaces_cache_when_provider_fingerprint_changes() { + let tmp = tempfile::tempdir().unwrap(); + let store = EmbeddingStore::open_or_create(&tmp.path().join("embeddings.db")).unwrap(); + let graph = make_graph(); + let content_hash = graph.get_node("feat-replay").unwrap().content_hash.clone(); + let provider_a = NamedProvider { + name: "provider-a", + embeddings: vec![vec![1.0, 0.0]], + }; + let provider_b = NamedProvider { + name: "provider-b", + embeddings: vec![vec![0.0, 1.0]], + }; + + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(embed_graph(&store, &graph, &provider_a)) + .unwrap(); + assert_eq!( + store.get_embedding(&content_hash).unwrap().unwrap(), + vec![1.0, 0.0] + ); + + let stats = rt + .block_on(embed_graph(&store, &graph, &provider_b)) + .unwrap(); + + assert_eq!(stats.embedded, 1); + assert_eq!(store.count().unwrap(), 1); + assert_eq!( + store.get_embedding(&content_hash).unwrap().unwrap(), + vec![0.0, 1.0] + ); + } + + #[test] + fn embed_graph_seeds_legacy_store_fingerprint_without_clearing() { + let tmp = tempfile::tempdir().unwrap(); + let store = EmbeddingStore::open_or_create(&tmp.path().join("embeddings.db")).unwrap(); + let graph = make_graph(); + let content_hash = graph.get_node("feat-replay").unwrap().content_hash.clone(); + store.store_embedding(&content_hash, &[1.0, 0.0]).unwrap(); + let provider = NamedProvider { + name: "legacy-provider", + embeddings: vec![vec![0.0, 1.0]], + }; + + let rt = tokio::runtime::Runtime::new().unwrap(); + let stats = rt.block_on(embed_graph(&store, &graph, &provider)).unwrap(); + + assert_eq!(stats.embedded, 0); + assert_eq!(store.count().unwrap(), 1); + assert_eq!( + store.get_embedding(&content_hash).unwrap().unwrap(), + vec![1.0, 0.0] + ); + } + #[test] fn count_embeddings_for_index_counts_all_nodes_sharing_a_cached_hash() { let tmp = tempfile::tempdir().unwrap(); diff --git a/crates/tempyr-index/src/health.rs b/crates/tempyr-index/src/health.rs index 03dfc7b..d3fa808 100644 --- a/crates/tempyr-index/src/health.rs +++ b/crates/tempyr-index/src/health.rs @@ -157,7 +157,7 @@ pub fn build_report(inputs: &HealthInputs<'_>) -> HealthReport { } if matches!(embedding.store_exists, Some(false)) && embedding.api_key_set != Some(false) { warnings.push(format!( - "Embedding store does not exist at {}. Run `tempyr index rebuild` to populate embeddings.", + "Embedding store does not exist at {}. Run a vector-backed command or `tempyr index rebuild` to populate embeddings.", embedding.store_path.as_ref().map(|p| p.display().to_string()).unwrap_or_default() )); } diff --git a/crates/tempyr-index/src/hybrid.rs b/crates/tempyr-index/src/hybrid.rs index c248cd4..c0587c7 100644 --- a/crates/tempyr-index/src/hybrid.rs +++ b/crates/tempyr-index/src/hybrid.rs @@ -50,7 +50,7 @@ type ScoreMap = std::collections::HashMap; /// Run the full hybrid retrieval pipeline. /// /// Combines structural traversal (if root_id provided), BM25 full-text search, -/// and vector similarity (deferred — always None for now). +/// and vector similarity when a query embedding is provided. pub fn hybrid_retrieve( index: &Index, graph: &Graph, diff --git a/crates/tempyr-index/src/lib.rs b/crates/tempyr-index/src/lib.rs index 77ca1b9..38d1d73 100644 --- a/crates/tempyr-index/src/lib.rs +++ b/crates/tempyr-index/src/lib.rs @@ -5,6 +5,7 @@ pub mod hybrid; pub mod incremental; pub mod indexer; pub mod refresh; +pub mod semantic; pub mod vector; use thiserror::Error; diff --git a/crates/tempyr-index/src/semantic.rs b/crates/tempyr-index/src/semantic.rs new file mode 100644 index 0000000..9d91cea --- /dev/null +++ b/crates/tempyr-index/src/semantic.rs @@ -0,0 +1,287 @@ +use tempyr_core::graph::Graph; + +use crate::embeddings::{self, EmbeddingProvider, EmbeddingStore, InputType}; +use crate::hybrid::{HybridResult, RetrievalConfig, hybrid_retrieve}; +use crate::indexer::Index; +use crate::vector::VectorSearchResult; +use crate::{IndexError, Result}; + +/// Provider-backed semantic retrieval over a graph index. +pub struct SemanticSearchEngine { + index: Index, + store: EmbeddingStore, + provider: Box, +} + +impl SemanticSearchEngine { + pub fn new(index: Index, store: EmbeddingStore, provider: Box) -> Self { + Self { + index, + store, + provider, + } + } + + pub async fn ensure_embeddings(&mut self, graph: &Graph) -> Result<()> { + // embed_graph is content-hash aware and skips cached entries, so keep + // checking the current graph instead of assuming a long-lived engine has + // already seen every future graph mutation. + embeddings::embed_graph(&self.store, graph, self.provider.as_ref()).await?; + Ok(()) + } + + pub async fn vector_search( + &mut self, + graph: &Graph, + query: &str, + max_results: usize, + node_type: Option<&str>, + min_similarity: Option, + ) -> Result> { + self.ensure_embeddings(graph).await?; + let query_embedding = self.embed_query(query).await?; + + let mut results = + self.store + .vector_search(&self.index, &query_embedding, max_results, node_type)?; + if let Some(min_similarity) = min_similarity { + results.retain(|result| result.similarity >= min_similarity); + } + Ok(results) + } + + pub async fn hybrid_retrieve( + &mut self, + graph: &Graph, + query: &str, + root: Option<&str>, + mut config: RetrievalConfig, + ) -> Result> { + self.ensure_embeddings(graph).await?; + config.query_embedding = Some(self.embed_query(query).await?); + hybrid_retrieve(&self.index, graph, query, root, &config, Some(&self.store)) + } + + async fn embed_query(&self, query: &str) -> Result> { + let query_embeddings = self + .provider + .embed(&[query.to_string()], InputType::Query) + .await?; + if query_embeddings.len() != 1 { + return Err(IndexError::General( + "Embedding provider returned wrong number of vectors for the query; expected exactly 1" + .to_string(), + )); + } + Ok(query_embeddings.into_iter().next().unwrap()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::path::PathBuf; + + use async_trait::async_trait; + use tempyr_core::node::parse_node; + use tempyr_core::schema::Schema; + + use super::*; + use crate::embeddings::EmbeddingProvider; + + fn make_schema() -> Schema { + r#" +[meta] +version = "1" +description = "test" + +[node_types.feature] +description = "Feature" +directory = "features" +required_fields = [] +optional_fields = ["status"] +allowed_statuses = ["draft"] +allowed_edges = [] + +[node_types.insight] +description = "Insight" +directory = "insights" +required_fields = [] +optional_fields = [] +allowed_edges = [] + +[edge_types] +"# + .parse() + .unwrap() + } + + fn make_graph() -> Graph { + let mut graph = Graph::new(make_schema()); + graph.add_node( + parse_node( + "---\nid: feat-a\ntype: feature\nstatus: draft\n---\n# Search Topic\n\nFind related insight.\n", + PathBuf::from("graph/features/feat-a.md"), + ) + .unwrap(), + ); + graph.add_node( + parse_node( + "---\nid: insight-a\ntype: insight\n---\n# Related Insight\n\nRelevant context.\n", + PathBuf::from("graph/insights/insight-a.md"), + ) + .unwrap(), + ); + graph + } + + fn make_graph_with_extra_insight() -> Graph { + let mut graph = make_graph(); + graph.add_node( + parse_node( + "---\nid: insight-b\ntype: insight\n---\n# New Insight\n\nFresh context.\n", + PathBuf::from("graph/insights/insight-b.md"), + ) + .unwrap(), + ); + graph + } + + struct FixedProvider; + + #[async_trait] + impl EmbeddingProvider for FixedProvider { + async fn embed(&self, texts: &[String], input_type: InputType) -> Result>> { + Ok(texts + .iter() + .map(|text| match input_type { + InputType::Query => vec![0.0, 1.0], + InputType::Document if text.contains("Related Insight") => vec![0.0, 1.0], + InputType::Document if text.contains("New Insight") => vec![0.0, 1.0], + InputType::Document => vec![1.0, 0.0], + }) + .collect()) + } + + fn dimensions(&self) -> usize { + 2 + } + + fn name(&self) -> &str { + "fixed" + } + } + + struct WrongQueryProvider; + + #[async_trait] + impl EmbeddingProvider for WrongQueryProvider { + async fn embed(&self, texts: &[String], input_type: InputType) -> Result>> { + match input_type { + InputType::Query => Ok(vec![vec![0.0, 1.0], vec![1.0, 0.0]]), + InputType::Document => Ok(texts.iter().map(|_| vec![0.0, 1.0]).collect()), + } + } + + fn dimensions(&self) -> usize { + 2 + } + + fn name(&self) -> &str { + "wrong-query" + } + } + + #[test] + fn vector_search_embeds_missing_graph_nodes_before_searching() { + let tmp = tempfile::tempdir().unwrap(); + let graph = make_graph(); + let index = Index::create_in_memory().unwrap(); + index.rebuild(&graph).unwrap(); + let store = EmbeddingStore::open_or_create(&tmp.path().join("embeddings.db")).unwrap(); + let provider = Box::new(FixedProvider); + let mut engine = SemanticSearchEngine::new(index, store, provider); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let results = runtime + .block_on(engine.vector_search(&graph, "related", 10, Some("insight"), Some(0.7))) + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].node_id, "insight-a"); + } + + #[test] + fn vector_search_rechecks_embeddings_when_graph_changes() { + let tmp = tempfile::tempdir().unwrap(); + let original_graph = make_graph(); + let updated_graph = make_graph_with_extra_insight(); + let index = Index::create_in_memory().unwrap(); + index.rebuild(&updated_graph).unwrap(); + let store = EmbeddingStore::open_or_create(&tmp.path().join("embeddings.db")).unwrap(); + let provider = Box::new(FixedProvider); + let mut engine = SemanticSearchEngine::new(index, store, provider); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime + .block_on(engine.vector_search(&original_graph, "related", 10, Some("insight"), None)) + .unwrap(); + + let results = runtime + .block_on(engine.vector_search(&updated_graph, "new", 10, Some("insight"), Some(0.7))) + .unwrap(); + + assert!(results.iter().any(|result| result.node_id == "insight-b")); + } + + #[test] + fn vector_search_rejects_wrong_query_embedding_count() { + let tmp = tempfile::tempdir().unwrap(); + let graph = make_graph(); + let index = Index::create_in_memory().unwrap(); + index.rebuild(&graph).unwrap(); + let store = EmbeddingStore::open_or_create(&tmp.path().join("embeddings.db")).unwrap(); + let provider = Box::new(WrongQueryProvider); + let mut engine = SemanticSearchEngine::new(index, store, provider); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let err = runtime + .block_on(engine.vector_search(&graph, "related", 10, Some("insight"), None)) + .unwrap_err(); + + assert!( + err.to_string() + .contains("wrong number of vectors for the query; expected exactly 1") + ); + } + + #[test] + fn hybrid_retrieve_includes_vector_scores_after_embedding() { + let tmp = tempfile::tempdir().unwrap(); + let graph = make_graph(); + let index = Index::create_in_memory().unwrap(); + index.rebuild(&graph).unwrap(); + let store = EmbeddingStore::open_or_create(&tmp.path().join("embeddings.db")).unwrap(); + let provider = Box::new(FixedProvider); + let mut engine = SemanticSearchEngine::new(index, store, provider); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let results = runtime + .block_on(engine.hybrid_retrieve(&graph, "related", None, RetrievalConfig::standard())) + .unwrap(); + + assert!( + results + .iter() + .any(|result| { result.node_id == "insight-a" && result.vector_score.is_some() }) + ); + assert_eq!( + results + .iter() + .map(|result| result.node_id.as_str()) + .collect::>() + .len(), + results.len() + ); + } +} diff --git a/crates/tempyr-mcp/src/handler.rs b/crates/tempyr-mcp/src/handler.rs index f2c40b9..e3e173c 100644 --- a/crates/tempyr-mcp/src/handler.rs +++ b/crates/tempyr-mcp/src/handler.rs @@ -20,11 +20,13 @@ use tempyr_core::schema::Schema; use tempyr_core::temporal::TemporalFilter; use tempyr_core::traverse::bfs; use tempyr_core::validate::validate_graph; +use tempyr_index::embeddings::{self, EmbeddingStore}; use tempyr_index::fts::MetadataFilter; use tempyr_index::health::{self, HealthInputs}; -use tempyr_index::hybrid::{RetrievalConfig, hybrid_retrieve}; +use tempyr_index::hybrid::RetrievalConfig; use tempyr_index::indexer::Index; use tempyr_index::refresh::refresh_index_for_graph; +use tempyr_index::semantic::SemanticSearchEngine; use tempyr_interview::gaps::next_questions; use tempyr_interview::proposer; use tempyr_interview::session::{ @@ -501,6 +503,151 @@ fn ensure_index_path( .ok_or_else(|| "Index refresh did not produce a queryable snapshot.".to_string()) } +struct McpSemanticSearch<'a> { + graph_dir: &'a Path, + gf_dir: &'a Path, + schema: &'a Schema, + graph: &'a Graph, + runtime: Option, +} + +struct McpSemanticSearchRuntime { + engine: SemanticSearchEngine, +} + +impl<'a> McpSemanticSearch<'a> { + fn new(graph_dir: &'a Path, gf_dir: &'a Path, schema: &'a Schema, graph: &'a Graph) -> Self { + Self { + graph_dir, + gf_dir, + schema, + graph, + runtime: None, + } + } + + fn runtime(&mut self) -> tempyr_render::Result<&mut McpSemanticSearchRuntime> { + if self.runtime.is_none() { + let runtime = + McpSemanticSearchRuntime::new(self.graph_dir, self.gf_dir, self.schema, self.graph) + .map_err(tempyr_render_error)?; + self.runtime = Some(runtime); + } + Ok(self.runtime.as_mut().expect("semantic runtime initialized")) + } +} + +impl tempyr_render::SemanticSearchProvider for McpSemanticSearch<'_> { + fn search( + &mut self, + request: &tempyr_render::SemanticSearchRequest, + ) -> tempyr_render::Result> { + let graph = self.graph; + let results = self + .runtime()? + .vector_search( + graph, + &request.query, + request.max_results, + request.target_type.as_deref(), + request.min_similarity, + ) + .map_err(tempyr_render_error)?; + + Ok(results + .into_iter() + .map(|result| tempyr_render::SemanticSearchHit { + node_id: result.node_id, + score: result.similarity, + }) + .collect()) + } +} + +impl McpSemanticSearchRuntime { + fn new( + graph_dir: &Path, + gf_dir: &Path, + schema: &Schema, + graph: &Graph, + ) -> Result { + let root = graph_dir + .parent() + .ok_or_else(|| "Failed to resolve project root from graph dir".to_string())?; + let _ = tempyr_core::project::load_project_env_from(root.to_path_buf()); + + let index_path = ensure_index_path(graph_dir, gf_dir, schema, Some(graph))?; + let index = Index::open(&index_path).map_err(|e| format!("Index: {e}"))?; + + let config = embeddings::load_embedding_config_from_file(&gf_dir.join("config.toml")) + .map_err(|e| e.to_string())?; + let resolved = embeddings::resolve_embedding_config(&config).map_err(|e| e.to_string())?; + let cache = tempyr_core::project::cache_layout(root, gf_dir); + let store_path = embeddings::embedding_store_path( + &cache, + &resolved.provider, + resolved.model.as_deref(), + Some(resolved.dimensions), + ); + let store = EmbeddingStore::open_or_create(&store_path).map_err(|e| e.to_string())?; + let provider = + embeddings::create_provider_from_resolved(&resolved).map_err(|e| e.to_string())?; + + Ok(Self { + engine: SemanticSearchEngine::new(index, store, provider), + }) + } + + fn vector_search( + &mut self, + graph: &Graph, + query: &str, + max_results: usize, + node_type: Option<&str>, + min_similarity: Option, + ) -> Result, String> { + block_on_index_future(self.engine.vector_search( + graph, + query, + max_results, + node_type, + min_similarity, + )) + } + + fn hybrid_retrieve( + &mut self, + graph: &Graph, + query: &str, + root: Option<&str>, + config: RetrievalConfig, + ) -> Result, String> { + block_on_index_future(self.engine.hybrid_retrieve(graph, query, root, config)) + } +} + +fn block_on_index_future(future: F) -> Result +where + F: std::future::Future>, +{ + if let Ok(handle) = tokio::runtime::Handle::try_current() { + if handle.runtime_flavor() != tokio::runtime::RuntimeFlavor::MultiThread { + return Err( + "Semantic retrieval requires a multi-threaded Tokio runtime in MCP mode." + .to_string(), + ); + } + tokio::task::block_in_place(|| handle.block_on(future)).map_err(|e| e.to_string()) + } else { + let runtime = tokio::runtime::Runtime::new().map_err(|e| e.to_string())?; + runtime.block_on(future).map_err(|e| e.to_string()) + } +} + +fn tempyr_render_error(err: String) -> tempyr_render::RenderError { + tempyr_render::RenderError::General(err) +} + fn refresh_index_for_current_snapshot( graph_dir: &Path, gf_dir: &Path, @@ -1117,22 +1264,15 @@ impl TempyrServer { .transpose()?; let graph = Graph::load_from_directory(&graph_dir, schema.clone()).map_err(|e| e.to_string())?; - let index_path = ensure_index_path(&graph_dir, &gf_dir, &schema, Some(&graph))?; - let index = Index::open(&index_path).map_err(|e| format!("Index: {e}"))?; let config = RetrievalConfig { token_budget: budget, ..RetrievalConfig::standard() }; - let results = hybrid_retrieve( - &index, - &graph, - &p.query, - resolved_root.as_deref(), - &config, - None, - ) - .map_err(|e| e.to_string())?; + let mut semantic_search = + McpSemanticSearchRuntime::new(&graph_dir, &gf_dir, &schema, &graph)?; + let results = + semantic_search.hybrid_retrieve(&graph, &p.query, resolved_root.as_deref(), config)?; let mut output = String::new(); for r in &results { @@ -1461,7 +1601,8 @@ impl TempyrServer { fn graph_render(&self, Parameters(p): Parameters) -> Result { let (graph_dir, gf_dir, schema) = self.find_project()?; let root_id = ops::resolve_node_id(&graph_dir, &p.root_node).map_err(|e| e.to_string())?; - let graph = Graph::load_from_directory(&graph_dir, schema).map_err(|e| e.to_string())?; + let graph = + Graph::load_from_directory(&graph_dir, schema.clone()).map_err(|e| e.to_string())?; let filter = if p.include_history.unwrap_or(false) { TemporalFilter::with_history() @@ -1469,9 +1610,19 @@ impl TempyrServer { TemporalFilter::current() }; + let mut semantic_search = McpSemanticSearch::new(&graph_dir, &gf_dir, &schema, &graph); let local_path = gf_dir.join("render").join(format!("{}.toml", p.template)); if local_path.exists() { - tempyr_render::render(&graph, &local_path, &root_id, &filter).map_err(|e| e.to_string()) + tempyr_render::render_with_options( + &graph, + &local_path, + &root_id, + &filter, + tempyr_render::RenderOptions { + semantic_search: Some(&mut semantic_search), + }, + ) + .map_err(|e| e.to_string()) } else { let template_toml = match p.template.as_str() { "prd" => include_str!("../../../templates/prd.toml"), @@ -1479,8 +1630,16 @@ impl TempyrServer { "task-prompt" => include_str!("../../../templates/task-prompt.toml"), _ => return Err(format!("Unknown template: '{}'", p.template)), }; - tempyr_render::render_from_str(&graph, template_toml, &root_id, &filter) - .map_err(|e| e.to_string()) + tempyr_render::render_from_str_with_options( + &graph, + template_toml, + &root_id, + &filter, + tempyr_render::RenderOptions { + semantic_search: Some(&mut semantic_search), + }, + ) + .map_err(|e| e.to_string()) } } // Interview tools diff --git a/crates/tempyr-render/src/collector.rs b/crates/tempyr-render/src/collector.rs index 0001758..92991c4 100644 --- a/crates/tempyr-render/src/collector.rs +++ b/crates/tempyr-render/src/collector.rs @@ -3,6 +3,7 @@ use tempyr_core::node::Node; use tempyr_core::temporal::{TemporalFilter, filter_edges, is_node_visible}; use crate::template::SectionDef; +use crate::{RenderError, Result, SemanticSearchProvider, SemanticSearchRequest}; /// Collected data for a single rendered section. #[derive(Debug, Clone)] @@ -24,39 +25,150 @@ pub struct SectionItem { pub internal_edges: Vec<(String, String, String)>, // (from_id, to_id, edge_type) } -/// Collect data for a section by traversing the graph from the root node. -pub fn collect_section( +/// Collect data for a section without a semantic-search provider. +fn collect_section_without_semantic_search( graph: &Graph, root: &Node, section: &SectionDef, filter: &TemporalFilter, -) -> SectionData { +) -> Result { let heading = section.heading.clone(); // Root body sections if section.source.as_deref() == Some("root") { - return collect_root_section(root, section); + return Ok(collect_root_section(root, section)); } - // Semantic search placeholder (not yet implemented) if section.source.as_deref() == Some("semantic_search") { - return SectionData { - heading, - items: Vec::new(), - is_root_section: false, - }; + return Err(RenderError::General(format!( + "Section '{heading}' requires semantic search, but no semantic search provider was configured." + ))); } // Traversal-based sections if let Some(edge_type) = §ion.traverse { - return collect_traverse_section(graph, root, section, edge_type, filter); + return Ok(collect_traverse_section( + graph, root, section, edge_type, filter, + )); } // Fallback: empty section - SectionData { + Ok(SectionData { heading, items: Vec::new(), is_root_section: false, + }) +} + +/// Collect data for a section, using semantic search when requested. +pub(crate) fn collect_section_with_semantic_search( + graph: &Graph, + root: &Node, + section: &SectionDef, + filter: &TemporalFilter, + semantic_search: &mut dyn SemanticSearchProvider, +) -> Result { + if section.source.as_deref() == Some("semantic_search") { + return collect_semantic_section(graph, root, section, filter, semantic_search); + } + + collect_section_without_semantic_search(graph, root, section, filter) +} + +/// Collect data for a section, optionally using semantic search when requested. +pub fn collect_section( + graph: &Graph, + root: &Node, + section: &SectionDef, + filter: &TemporalFilter, + semantic_search: Option<&mut dyn SemanticSearchProvider>, +) -> Result { + if let Some(provider) = semantic_search { + collect_section_with_semantic_search(graph, root, section, filter, provider) + } else { + collect_section_without_semantic_search(graph, root, section, filter) + } +} + +/// Collect a section from semantic vector search. +fn collect_semantic_section( + graph: &Graph, + root: &Node, + section: &SectionDef, + temporal_filter: &TemporalFilter, + semantic_search: &mut dyn SemanticSearchProvider, +) -> Result { + let query = semantic_query(root, section)?; + let request = SemanticSearchRequest { + query, + target_type: section.target_type.clone(), + max_results: section.max_results.unwrap_or(10), + min_similarity: section.min_similarity, + }; + let hits = semantic_search.search(&request)?; + let include_body = section.include_body.unwrap_or(false); + let mut items = Vec::new(); + + for hit in hits { + if let Some(min_similarity) = section.min_similarity + && hit.score < min_similarity + { + continue; + } + + if hit.node_id == root.id() { + continue; + } + + let Some(node) = graph.get_node(&hit.node_id) else { + continue; + }; + + if let Some(target_type) = section.target_type.as_deref() + && node.node_type() != target_type + { + continue; + } + + if !is_node_visible(node, temporal_filter) { + continue; + } + + if !matches_status_filter(node, section) { + continue; + } + + let body = if include_body { + Some(node.body.clone()) + } else { + None + }; + + items.push(SectionItem { + node_id: node.id().to_string(), + title: node.title().to_string(), + node_type: node.node_type().to_string(), + fields: collect_fields(node, section), + body, + sub_items: Vec::new(), + internal_edges: Vec::new(), + }); + } + + Ok(SectionData { + heading: section.heading.clone(), + items, + is_root_section: false, + }) +} + +fn semantic_query(root: &Node, section: &SectionDef) -> Result { + match section.query_from.as_deref().unwrap_or("root") { + "root" => Ok(format!("{}\n\n{}", root.title(), root.body.trim())), + other => Err(RenderError::General(format!( + "Unsupported semantic_search query_from value '{other}' in section '{}'.", + section.heading + ))), } } @@ -124,12 +236,7 @@ fn collect_traverse_section( continue; } - // Apply status filter if specified - if let Some(filter_map) = §ion.filter - && let Some(allowed_statuses) = filter_map.get("status") - && let Some(status) = target_node.status() - && !allowed_statuses.contains(&status.to_string()) - { + if !matches_status_filter(target_node, section) { continue; } @@ -261,6 +368,19 @@ fn collect_fields(node: &Node, section: &SectionDef) -> Vec<(String, String)> { result } +fn matches_status_filter(node: &Node, section: &SectionDef) -> bool { + let Some(filter_map) = §ion.filter else { + return true; + }; + let Some(allowed_statuses) = filter_map.get("status") else { + return true; + }; + let Some(status) = node.status() else { + return true; + }; + allowed_statuses.contains(&status.to_string()) +} + /// Extract a named section (## Heading) from a markdown body. pub fn extract_body_section(body: &str, section_name: &str) -> Option { let heading_prefix = format!("## {section_name}"); @@ -370,7 +490,8 @@ A recording agent captures DOM snapshots. query_from: None, }; - let data = collect_section(&graph, root, §ion, &TemporalFilter::current()); + let data = + collect_section(&graph, root, §ion, &TemporalFilter::current(), None).unwrap(); assert!(data.is_root_section); assert_eq!(data.items.len(), 1); assert!( @@ -410,7 +531,8 @@ A recording agent captures DOM snapshots. query_from: None, }; - let data = collect_section(&graph, root, §ion, &TemporalFilter::current()); + let data = + collect_section(&graph, root, §ion, &TemporalFilter::current(), None).unwrap(); let body = data.items[0].body.as_ref().unwrap(); assert!(body.contains("Engineers need to see")); assert!(!body.contains("## Solution")); // should stop at next heading @@ -438,7 +560,8 @@ A recording agent captures DOM snapshots. query_from: None, }; - let data = collect_section(&graph, root, §ion, &TemporalFilter::current()); + let data = + collect_section(&graph, root, §ion, &TemporalFilter::current(), None).unwrap(); assert_eq!(data.items.len(), 1); assert_eq!(data.items[0].title, "Platform Engineer"); assert!(data.items[0].body.is_some()); @@ -475,7 +598,8 @@ A recording agent captures DOM snapshots. query_from: None, }; - let data = collect_section(&graph, root, §ion, &TemporalFilter::current()); + let data = + collect_section(&graph, root, §ion, &TemporalFilter::current(), None).unwrap(); assert_eq!(data.items.len(), 1); assert_eq!(data.items[0].node_id, "decision-storage"); } @@ -493,8 +617,67 @@ A recording agent captures DOM snapshots. assert!(extract_body_section(body, "Missing").is_none()); } + struct FakeSemanticSearch { + request: Option, + } + + impl SemanticSearchProvider for FakeSemanticSearch { + fn search( + &mut self, + request: &SemanticSearchRequest, + ) -> Result> { + self.request = Some(request.clone()); + Ok(vec![crate::SemanticSearchHit { + node_id: "decision-storage".to_string(), + score: 0.91, + }]) + } + } + #[test] - fn test_semantic_search_placeholder() { + fn test_semantic_search_collects_matching_hits() { + let graph = build_test_graph(); + let root = graph.get_node("feat-replay").unwrap(); + let section = SectionDef { + heading: "Insights".to_string(), + source: Some("semantic_search".to_string()), + body_section: None, + traverse: None, + target_type: Some("decision".to_string()), + include_body: Some(true), + include_fields: None, + filter: None, + sub_traverse: None, + sub_target_type: None, + show_internal_edges: None, + internal_edge_types: None, + max_results: Some(5), + min_similarity: Some(0.7), + query_from: Some("root".to_string()), + }; + let mut provider = FakeSemanticSearch { request: None }; + + let data = collect_section( + &graph, + root, + §ion, + &TemporalFilter::current(), + Some(&mut provider), + ) + .unwrap(); + + let request = provider.request.unwrap(); + assert!(request.query.contains("Session Replay")); + assert_eq!(request.target_type.as_deref(), Some("decision")); + assert_eq!(request.max_results, 5); + assert_eq!(request.min_similarity, Some(0.7)); + assert_eq!(data.items.len(), 1); + assert_eq!(data.items[0].node_id, "decision-storage"); + assert!(data.items[0].body.as_ref().unwrap().contains("ClickHouse")); + } + + #[test] + fn test_semantic_search_requires_provider() { let graph = build_test_graph(); let root = graph.get_node("feat-replay").unwrap(); let section = SectionDef { @@ -515,7 +698,12 @@ A recording agent captures DOM snapshots. query_from: Some("root".to_string()), }; - let data = collect_section(&graph, root, §ion, &TemporalFilter::current()); - assert!(data.items.is_empty()); // placeholder returns empty + let err = + collect_section(&graph, root, §ion, &TemporalFilter::current(), None).unwrap_err(); + + assert!( + err.to_string() + .contains("requires semantic search, but no semantic search provider") + ); } } diff --git a/crates/tempyr-render/src/lib.rs b/crates/tempyr-render/src/lib.rs index 7683afc..5fadf90 100644 --- a/crates/tempyr-render/src/lib.rs +++ b/crates/tempyr-render/src/lib.rs @@ -23,12 +23,55 @@ pub enum RenderError { pub type Result = std::result::Result; +/// Request issued by a semantic-search render section. +#[derive(Debug, Clone, PartialEq)] +pub struct SemanticSearchRequest { + pub query: String, + pub target_type: Option, + pub max_results: usize, + pub min_similarity: Option, +} + +/// One semantic-search result returned to the renderer. +#[derive(Debug, Clone, PartialEq)] +pub struct SemanticSearchHit { + pub node_id: String, + pub score: f64, +} + +/// Semantic search implementation supplied by callers that have index access. +pub trait SemanticSearchProvider { + fn search(&mut self, request: &SemanticSearchRequest) -> Result>; +} + +#[derive(Default)] +pub struct RenderOptions<'a> { + pub semantic_search: Option<&'a mut dyn SemanticSearchProvider>, +} + /// Render a document from a graph using a template. pub fn render( graph: &Graph, template_path: &Path, root_id: &str, temporal_filter: &TemporalFilter, +) -> Result { + render_with_options( + graph, + template_path, + root_id, + temporal_filter, + RenderOptions::default(), + ) +} + +/// Render a document from a graph using a template and caller-provided options. +pub fn render_with_options( + graph: &Graph, + template_path: &Path, + root_id: &str, + temporal_filter: &TemporalFilter, + options: RenderOptions<'_>, ) -> Result { let tmpl = template::RenderTemplate::load(template_path)?; @@ -48,11 +91,13 @@ pub fn render( } // Collect all sections - let sections: Vec<_> = tmpl - .sections - .iter() - .map(|section_def| collector::collect_section(graph, root, section_def, temporal_filter)) - .collect(); + let sections = collect_sections( + graph, + root, + &tmpl.sections, + temporal_filter, + options.semantic_search, + )?; // Format to markdown Ok(formatter::render_to_markdown(&tmpl, root, sections)) @@ -64,6 +109,23 @@ pub fn render_from_str( template_toml: &str, root_id: &str, temporal_filter: &TemporalFilter, +) -> Result { + render_from_str_with_options( + graph, + template_toml, + root_id, + temporal_filter, + RenderOptions::default(), + ) +} + +/// Render using a template string and caller-provided options. +pub fn render_from_str_with_options( + graph: &Graph, + template_toml: &str, + root_id: &str, + temporal_filter: &TemporalFilter, + options: RenderOptions<'_>, ) -> Result { let tmpl: template::RenderTemplate = template_toml.parse()?; @@ -81,15 +143,49 @@ pub fn render_from_str( ))); } - let sections: Vec<_> = tmpl - .sections - .iter() - .map(|section_def| collector::collect_section(graph, root, section_def, temporal_filter)) - .collect(); + let sections = collect_sections( + graph, + root, + &tmpl.sections, + temporal_filter, + options.semantic_search, + )?; Ok(formatter::render_to_markdown(&tmpl, root, sections)) } +fn collect_sections( + graph: &Graph, + root: &tempyr_core::node::Node, + section_defs: &[template::SectionDef], + temporal_filter: &TemporalFilter, + semantic_search: Option<&mut dyn SemanticSearchProvider>, +) -> Result> { + let mut sections = Vec::with_capacity(section_defs.len()); + if let Some(provider) = semantic_search { + for section_def in section_defs { + sections.push(collector::collect_section_with_semantic_search( + graph, + root, + section_def, + temporal_filter, + provider, + )?); + } + } else { + for section_def in section_defs { + sections.push(collector::collect_section( + graph, + root, + section_def, + temporal_filter, + None, + )?); + } + } + Ok(sections) +} + #[cfg(test)] mod tests { use super::*; @@ -146,6 +242,19 @@ A recording agent captures DOM snapshots. graph } + struct FakeSemanticSearch; + + impl SemanticSearchProvider for FakeSemanticSearch { + fn search(&mut self, request: &SemanticSearchRequest) -> Result> { + assert!(request.query.contains("Session Replay")); + assert_eq!(request.target_type.as_deref(), Some("decision")); + Ok(vec![SemanticSearchHit { + node_id: "decision-storage".to_string(), + score: 0.95, + }]) + } + } + #[test] fn test_render_prd_integration() { let graph = build_full_graph(); @@ -216,4 +325,40 @@ source = "root" assert!(result.contains("Simple Doc: Session Replay")); assert!(result.contains("## Overview")); } + + #[test] + fn test_render_from_str_with_semantic_provider() { + let graph = build_full_graph(); + let template_toml = r#" +[meta] +name = "Semantic Doc" +root_types = ["feature"] +output_format = "markdown" + +[[sections]] +heading = "Related Decisions" +source = "semantic_search" +query_from = "root" +target_type = "decision" +max_results = 3 +min_similarity = 0.7 +include_body = true +"#; + let mut provider = FakeSemanticSearch; + + let result = render_from_str_with_options( + &graph, + template_toml, + "feat-replay", + &TemporalFilter::current(), + RenderOptions { + semantic_search: Some(&mut provider), + }, + ) + .unwrap(); + + assert!(result.contains("## Related Decisions")); + assert!(result.contains("### Storage Backend")); + assert!(result.contains("Use ClickHouse.")); + } } diff --git a/crates/tempyr-render/src/template.rs b/crates/tempyr-render/src/template.rs index a5c9126..cd3beb6 100644 --- a/crates/tempyr-render/src/template.rs +++ b/crates/tempyr-render/src/template.rs @@ -159,5 +159,6 @@ mod tests { assert_eq!(insights.source.as_deref(), Some("semantic_search")); assert_eq!(insights.query_from.as_deref(), Some("root")); assert_eq!(insights.max_results, Some(5)); + assert_eq!(insights.include_body, Some(true)); } } diff --git a/docs/graphspec.md b/docs/graphspec.md index f501800..4036e05 100644 --- a/docs/graphspec.md +++ b/docs/graphspec.md @@ -586,11 +586,14 @@ Step 6: Output | Command | Behavior | |---------|----------| | `tempyr search ` | BM25 only (fast, keyword-exact) | -| `tempyr vsearch ` | Vector only (semantic similarity) | -| `tempyr context [--root ]` | Full hybrid pipeline | +| `tempyr vsearch ` | Vector-only semantic similarity; populates missing embeddings first | +| `tempyr context [--root ]` | Full hybrid pipeline; populates missing embeddings first | | `tempyr traverse [--depth N]` | Structural only, no ranking | | `tempyr ask ` | Full hybrid → feed to LLM → answer | +Vector-backed commands use the configured embedding provider to fill missing +entries in the shared embedding store before retrieval. + ### 3.7 Interview Engine The interview engine is the core product differentiator. It manages a structured conversation that produces graph nodes. @@ -963,6 +966,7 @@ query_from = "root" # use root node body as search query target_type = "insight" max_results = 5 min_similarity = 0.7 +include_body = true ``` #### 3.8.2 Rendering with Temporal Filters diff --git a/templates/tdd.toml b/templates/tdd.toml index 8d773ad..3aab742 100644 --- a/templates/tdd.toml +++ b/templates/tdd.toml @@ -58,3 +58,4 @@ query_from = "root" target_type = "insight" max_results = 5 min_similarity = 0.7 +include_body = true