diff --git a/doc/todo.md b/doc/todo.md index 873350b..984d9c9 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -1,9 +1,7 @@ 需求输入和展示,多行,复杂需求,界面不够优化,输入框不够大。 -Agent runtime 重构:统一 ProjectState + 单写者模型、context 压缩、plan-centric 反馈处理(详见 doc/context.md) +❯ 在前端,计划和日志,每一个条目,都应有一个小小的comment按钮,按一下,直接快速引用,然后输入docus到反馈输入那里,表示要评论的是这 + 个地方。这样llm也知道用户具体在指啥。 同时,允许多处引用,再点一个其他的comment按钮,就引用两处,等等。按钮做的不要太眨眼间,比 + 如用hover显示或者就是小一点不占地方,但要ui意图清晰易用。 -template - ---- 时间观察app ---- diff --git a/src/agent.rs b/src/agent.rs index 3d7398a..c9d52ce 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -33,6 +33,7 @@ pub enum WsMessage { RequirementUpdate { workflow_id: String, requirement: String }, ReportReady { workflow_id: String }, ProjectUpdate { project_id: String, name: String }, + LlmCallLog { workflow_id: String, entry: crate::db::LlmCallLogEntry }, Error { message: String }, } @@ -45,7 +46,7 @@ pub struct PlanStepInfo { pub status: Option, } -fn plan_infos_from_state(state: &AgentState) -> Vec { +pub fn plan_infos_from_state(state: &AgentState) -> Vec { state.steps.iter().map(|s| { let status = match s.status { StepStatus::Pending => "pending", @@ -832,6 +833,61 @@ async fn log_execution( }); } +/// Log an LLM call to llm_call_log and broadcast to frontend. +#[allow(clippy::too_many_arguments)] +async fn log_llm_call( + pool: &SqlitePool, + broadcast_tx: &broadcast::Sender, + workflow_id: &str, + step_order: i32, + phase: &str, + messages_count: i32, + tools_count: i32, + tool_calls_json: &str, + text_response: &str, + prompt_tokens: Option, + completion_tokens: Option, + latency_ms: i64, +) { + let id = uuid::Uuid::new_v4().to_string(); + let _ = sqlx::query( + "INSERT INTO llm_call_log (id, workflow_id, step_order, phase, messages_count, tools_count, tool_calls, text_response, prompt_tokens, completion_tokens, latency_ms, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))" + ) + .bind(&id) + .bind(workflow_id) + .bind(step_order) + .bind(phase) + .bind(messages_count) + .bind(tools_count) + .bind(tool_calls_json) + .bind(text_response) + .bind(prompt_tokens.map(|v| v as i32)) + .bind(completion_tokens.map(|v| v as i32)) + .bind(latency_ms as i32) + .execute(pool) + .await; + + let entry = crate::db::LlmCallLogEntry { + id: id.clone(), + workflow_id: workflow_id.to_string(), + step_order, + phase: phase.to_string(), + messages_count, + tools_count, + tool_calls: tool_calls_json.to_string(), + text_response: text_response.to_string(), + prompt_tokens: prompt_tokens.map(|v| v as i32), + completion_tokens: completion_tokens.map(|v| v as i32), + latency_ms: latency_ms as i32, + created_at: String::new(), + }; + + let _ = broadcast_tx.send(WsMessage::LlmCallLog { + workflow_id: workflow_id.to_string(), + entry, + }); +} + /// Process user feedback: call LLM to decide whether to revise the plan. /// Returns the (possibly modified) AgentState ready for resumed execution. async fn process_feedback( @@ -952,8 +1008,16 @@ async fn run_agent_loop( AgentPhase::Completed => break, }; let messages = state.build_messages(&system_prompt, requirement); + let msg_count = messages.len() as i32; + let tool_count = tools.len() as i32; + let phase_label = match &state.phase { + AgentPhase::Planning => "planning".to_string(), + AgentPhase::Executing { step } => format!("executing({})", step), + AgentPhase::Completed => "completed".to_string(), + }; tracing::info!("[workflow {}] LLM call #{} phase={:?} msgs={}", workflow_id, iteration + 1, state.phase, messages.len()); + let call_start = std::time::Instant::now(); let response = match llm.chat_with_tools(messages, tools).await { Ok(r) => r, Err(e) => { @@ -961,6 +1025,11 @@ async fn run_agent_loop( return Err(e); } }; + let latency_ms = call_start.elapsed().as_millis() as i64; + + let (prompt_tokens, completion_tokens) = response.usage.as_ref() + .map(|u| (Some(u.prompt_tokens), Some(u.completion_tokens))) + .unwrap_or((None, None)); let choice = response.choices.into_iter().next() .ok_or_else(|| anyhow::anyhow!("No response from LLM"))?; @@ -968,6 +1037,9 @@ async fn run_agent_loop( // Add assistant message to chat history state.current_step_chat_history.push(choice.message.clone()); + // Collect text_response for logging + let llm_text_response = choice.message.content.clone().unwrap_or_default(); + if let Some(tool_calls) = &choice.message.tool_calls { tracing::info!("[workflow {}] Tool calls: {}", workflow_id, tool_calls.iter().map(|tc| tc.function.name.as_str()).collect::>().join(", ")); @@ -1184,6 +1256,22 @@ async fn run_agent_loop( } } + // Build tool_calls JSON for LLM call log + let tc_json: Vec = tool_calls.iter().map(|tc| { + serde_json::json!({ + "name": tc.function.name, + "arguments_preview": truncate_str(&tc.function.arguments, 200), + }) + }).collect(); + let tc_json_str = serde_json::to_string(&tc_json).unwrap_or_else(|_| "[]".to_string()); + + log_llm_call( + pool, broadcast_tx, workflow_id, state.current_step(), + &phase_label, msg_count, tool_count, + &tc_json_str, &llm_text_response, + prompt_tokens, completion_tokens, latency_ms, + ).await; + if phase_transition { continue; } @@ -1195,6 +1283,13 @@ async fn run_agent_loop( // Log text response to execution_log for frontend display log_execution(pool, broadcast_tx, workflow_id, state.current_step(), "text_response", "", content, "done").await; + log_llm_call( + pool, broadcast_tx, workflow_id, state.current_step(), + &phase_label, msg_count, tool_count, + "[]", content, + prompt_tokens, completion_tokens, latency_ms, + ).await; + // Text response does NOT end the workflow. Only advance_step progresses. // In Planning phase, LLM may be thinking before calling update_plan — just continue. } diff --git a/src/api/workflows.rs b/src/api/workflows.rs index 13ca917..13fe1a0 100644 --- a/src/api/workflows.rs +++ b/src/api/workflows.rs @@ -8,8 +8,9 @@ use axum::{ }; use serde::Deserialize; use crate::AppState; -use crate::agent::AgentEvent; -use crate::db::{Workflow, ExecutionLogEntry, Comment}; +use crate::agent::{AgentEvent, PlanStepInfo}; +use crate::db::{Workflow, ExecutionLogEntry, Comment, LlmCallLogEntry}; +use crate::state::AgentState; use super::{ApiResult, db_err}; #[derive(serde::Serialize)] @@ -33,6 +34,8 @@ pub fn router(state: Arc) -> Router { .route("/workflows/{id}/steps", get(list_steps)) .route("/workflows/{id}/comments", get(list_comments).post(create_comment)) .route("/workflows/{id}/report", get(get_report)) + .route("/workflows/{id}/plan", get(get_plan)) + .route("/workflows/{id}/llm-calls", get(list_llm_calls)) .with_state(state) } @@ -153,3 +156,38 @@ async fn get_report( None => Err((StatusCode::NOT_FOUND, "Workflow not found").into_response()), } } + +async fn get_plan( + State(state): State>, + Path(workflow_id): Path, +) -> ApiResult> { + let snapshot_json: Option = sqlx::query_scalar( + "SELECT state_json FROM agent_state_snapshots WHERE workflow_id = ? ORDER BY created_at DESC LIMIT 1" + ) + .bind(&workflow_id) + .fetch_optional(&state.db.pool) + .await + .map_err(db_err)?; + + if let Some(json) = snapshot_json { + if let Ok(agent_state) = serde_json::from_str::(&json) { + return Ok(Json(crate::agent::plan_infos_from_state(&agent_state))); + } + } + + Ok(Json(vec![])) +} + +async fn list_llm_calls( + State(state): State>, + Path(workflow_id): Path, +) -> ApiResult> { + sqlx::query_as::<_, LlmCallLogEntry>( + "SELECT * FROM llm_call_log WHERE workflow_id = ? ORDER BY created_at" + ) + .bind(&workflow_id) + .fetch_all(&state.db.pool) + .await + .map(Json) + .map_err(db_err) +} diff --git a/src/db.rs b/src/db.rs index 1bb91b1..db8686d 100644 --- a/src/db.rs +++ b/src/db.rs @@ -158,6 +158,32 @@ impl Database { .execute(&self.pool) .await?; + sqlx::query( + "CREATE TABLE IF NOT EXISTS llm_call_log ( + id TEXT PRIMARY KEY, + workflow_id TEXT NOT NULL REFERENCES workflows(id), + step_order INTEGER NOT NULL, + phase TEXT NOT NULL, + messages_count INTEGER NOT NULL, + tools_count INTEGER NOT NULL, + tool_calls TEXT NOT NULL DEFAULT '[]', + text_response TEXT NOT NULL DEFAULT '', + prompt_tokens INTEGER, + completion_tokens INTEGER, + latency_ms INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + )" + ) + .execute(&self.pool) + .await?; + + // Migration: add text_response column to llm_call_log + let _ = sqlx::query( + "ALTER TABLE llm_call_log ADD COLUMN text_response TEXT NOT NULL DEFAULT ''" + ) + .execute(&self.pool) + .await; + sqlx::query( "CREATE TABLE IF NOT EXISTS timers ( id TEXT PRIMARY KEY, @@ -237,3 +263,19 @@ pub struct Timer { pub last_run_at: String, pub created_at: String, } + +#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow)] +pub struct LlmCallLogEntry { + pub id: String, + pub workflow_id: String, + pub step_order: i32, + pub phase: String, + pub messages_count: i32, + pub tools_count: i32, + pub tool_calls: String, + pub text_response: String, + pub prompt_tokens: Option, + pub completion_tokens: Option, + pub latency_ms: i32, + pub created_at: String, +} diff --git a/src/llm.rs b/src/llm.rs index e99ce08..a52094f 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -67,9 +67,20 @@ pub struct ToolCallFunction { pub arguments: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Usage { + #[serde(default)] + pub prompt_tokens: u32, + #[serde(default)] + pub completion_tokens: u32, + #[serde(default)] + pub total_tokens: u32, +} + #[derive(Debug, Deserialize)] pub struct ChatResponse { pub choices: Vec, + pub usage: Option, } #[derive(Debug, Deserialize)] diff --git a/src/main.rs b/src/main.rs index c5bb03a..dc31981 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ mod db; mod kb; mod llm; mod exec; -mod state; +pub mod state; mod timer; mod ws; diff --git a/web/src/api.ts b/web/src/api.ts index 6f9e11c..02c620b 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -1,4 +1,4 @@ -import type { Project, Workflow, ExecutionLogEntry, Comment, Timer, KbArticle, KbArticleSummary } from './types' +import type { Project, Workflow, ExecutionLogEntry, Comment, Timer, KbArticle, KbArticleSummary, PlanStepInfo, LlmCallLogEntry } from './types' const BASE = '/api' @@ -58,6 +58,12 @@ export const api = { getReport: (workflowId: string) => request<{ report: string }>(`/workflows/${workflowId}/report`), + listPlanSteps: (workflowId: string) => + request(`/workflows/${workflowId}/plan`), + + listLlmCalls: (workflowId: string) => + request(`/workflows/${workflowId}/llm-calls`), + listTimers: (projectId: string) => request(`/projects/${projectId}/timers`), diff --git a/web/src/components/CommentSection.vue b/web/src/components/CommentSection.vue index 3052200..086c7aa 100644 --- a/web/src/components/CommentSection.vue +++ b/web/src/components/CommentSection.vue @@ -1,21 +1,35 @@