refactor(agent): 迁移AI会话引擎至pi-agent-core库

将原有基于Anthropic/OpenAI SDK的直播聊天代理重构为使用`@earendil-works/pi-agent-core`和`@earendil-works/pi-ai`库的统一API。

新增pi-bridge、pi-model、pi-persist、pi-tools四个模块,封装Agent路由、模型配置、消息持久化和工具适配逻辑。移除`chat.ts`中大量死代码,简化WebSocket处理流程。

BREAKING CHANGE: 移除`VideoAgent`类的`getAnthropicClient`、`getOpenAIClient`、`executeTool`等方法,外部调用需迁移至新pi-bridge API。`PROJECT_ROOT`路径计算方式变更,从`../../..`变为`../../`。
This commit is contained in:
2026-05-08 01:43:33 +08:00
parent 2ab5396461
commit a6f2973f21
10 changed files with 2642 additions and 522 deletions

2277
web/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -12,6 +12,8 @@
}, },
"dependencies": { "dependencies": {
"@anthropic-ai/sdk": "^0.95.0", "@anthropic-ai/sdk": "^0.95.0",
"@earendil-works/pi-agent-core": "^0.74.0",
"@earendil-works/pi-ai": "^0.74.0",
"@types/multer": "^2.1.0", "@types/multer": "^2.1.0",
"better-sqlite3": "^11.6.0", "better-sqlite3": "^11.6.0",
"cors": "^2.8.5", "cors": "^2.8.5",

View File

@@ -1,102 +1,12 @@
import Anthropic from '@anthropic-ai/sdk';
import OpenAI from 'openai';
import { tools, ToolDefinition } from './tools/index';
import { getDb } from '../db';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { fileURLToPath } from 'url'; import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url); const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename); const __dirname = path.dirname(__filename);
const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..', '..'); const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..');
export type Protocol = 'anthropic' | 'openai';
interface ApiConfig {
protocol: Protocol;
apiKey: string;
baseURL: string | undefined;
model: string;
}
function getApiConfig(): ApiConfig {
const configRow = getDb().prepare('SELECT value FROM configs WHERE key = ?').get('api_keys') as { value: string } | undefined;
let apiKey = process.env.ANTHROPIC_API_KEY || '';
let baseURL: string | undefined;
let model = process.env.ANTHROPIC_MODEL || 'claude-sonnet-4-6';
let protocol: Protocol = 'anthropic';
if (configRow) {
try {
const cfg = JSON.parse(configRow.value);
if (cfg.ANTHROPIC_AUTH_TOKEN) apiKey = cfg.ANTHROPIC_AUTH_TOKEN;
if (cfg.ANTHROPIC_BASE_URL) baseURL = cfg.ANTHROPIC_BASE_URL;
if (cfg.ANTHROPIC_MODEL) model = cfg.ANTHROPIC_MODEL;
if (cfg.PROTOCOL === 'openai') protocol = 'openai';
} catch {}
}
return { protocol, apiKey, baseURL, model };
}
function getAnthropicClient(): Anthropic {
const { apiKey, baseURL } = getApiConfig();
return new Anthropic({ apiKey, baseURL });
}
function getOpenAIClient(): OpenAI {
const { apiKey, baseURL } = getApiConfig();
return new OpenAI({ apiKey, baseURL: baseURL || 'https://api.openai.com/v1' });
}
export class VideoAgent { export class VideoAgent {
private tools: ToolDefinition[];
constructor() {
this.tools = tools;
}
getProtocol(): Protocol {
return getApiConfig().protocol;
}
getModel(): string {
return getApiConfig().model;
}
getAnthropicClient(): Anthropic {
return getAnthropicClient();
}
getOpenAIClient(): OpenAI {
return getOpenAIClient();
}
getAnthropicTools(): Anthropic.Tool[] {
return this.tools.map((t) => ({
name: t.name,
description: t.description,
input_schema: t.input_schema,
}));
}
getOpenAITools(): OpenAI.ChatCompletionTool[] {
return this.tools.map((t) => ({
type: 'function' as const,
function: {
name: t.name,
description: t.description,
parameters: t.input_schema,
},
}));
}
async executeTool(name: string, params: Record<string, unknown>): Promise<string> {
const tool = this.tools.find((t) => t.name === name);
if (!tool) throw new Error(`Unknown tool: ${name}`);
return tool.execute(params);
}
getSystemPrompt(): string { getSystemPrompt(): string {
const accountsDir = path.join(PROJECT_ROOT, 'accounts'); const accountsDir = path.join(PROJECT_ROOT, 'accounts');

View File

@@ -0,0 +1,117 @@
import { Agent } from '@earendil-works/pi-agent-core';
import type { AgentEvent } from '@earendil-works/pi-agent-core';
import { streamSimple } from '@earendil-works/pi-ai';
import type { AssistantMessage } from '@earendil-works/pi-ai';
import { WebSocket } from 'ws';
import { createPiModel } from './pi-model';
import { createPiTools } from './pi-tools';
import { tools } from './tools/index';
import { videoAgent } from './index';
import { dbToPiMessages, saveUserMessage, saveAssistantMessage, saveToolResult, type DbMessage } from './pi-persist';
import { getDb } from '../db';
interface RunContext {
currentAssistantMsgId: string | null;
}
export async function runAgentChat(ws: WebSocket, convId: string, userContent: string) {
const userMsgId = saveUserMessage(convId, userContent);
ws.send(JSON.stringify({ type: 'message', data: { id: userMsgId, role: 'user', content: userContent } }));
const msgCount = getDb().prepare('SELECT COUNT(*) as count FROM messages WHERE conversation_id = ?').get(convId) as { count: number };
if (msgCount.count <= 1) {
const title = userContent.slice(0, 30) + (userContent.length > 30 ? '...' : '');
getDb().prepare("UPDATE conversations SET title = ?, updated_at = datetime('now') WHERE id = ?").run(title, convId);
}
getDb().prepare("UPDATE conversations SET updated_at = datetime('now') WHERE id = ?").run(convId);
const history = getDb().prepare(
'SELECT * FROM messages WHERE conversation_id = ? AND id != ? ORDER BY created_at'
).all(convId, userMsgId) as DbMessage[];
const piMessages = dbToPiMessages(history);
const { model, apiKey } = createPiModel();
const piTools = createPiTools(tools);
const agent = new Agent({
initialState: {
systemPrompt: videoAgent.getSystemPrompt(),
model,
thinkingLevel: 'off',
tools: piTools,
messages: piMessages,
},
streamFn: streamSimple,
getApiKey: () => apiKey,
});
const ctx: RunContext = { currentAssistantMsgId: null };
agent.subscribe((event: AgentEvent) => {
handleAgentEvent(ws, convId, event, ctx);
});
ws.send(JSON.stringify({ type: 'status', data: { status: 'thinking' } }));
try {
await agent.prompt(userContent);
} catch (err) {
const errMsg = (err as Error).message;
console.error('[pi-bridge] Agent error:', errMsg);
ws.send(JSON.stringify({
type: 'message',
data: { id: '', role: 'assistant', content: `抱歉,出错了:${errMsg}` },
}));
}
}
function handleAgentEvent(ws: WebSocket, convId: string, event: AgentEvent, ctx: RunContext) {
switch (event.type) {
case 'message_start': {
if (event.message.role === 'assistant') {
const id = crypto.randomUUID();
ctx.currentAssistantMsgId = id;
ws.send(JSON.stringify({ type: 'message_start', data: { id } }));
}
break;
}
case 'message_update': {
const piEvent = event.assistantMessageEvent;
const id = ctx.currentAssistantMsgId || '';
if (piEvent.type === 'text_delta') {
ws.send(JSON.stringify({ type: 'text_delta', data: { id, text: piEvent.delta } }));
} else if (piEvent.type === 'thinking_delta') {
ws.send(JSON.stringify({ type: 'reasoning_delta', data: { id, text: piEvent.delta } }));
}
break;
}
case 'message_end': {
if (event.message.role === 'assistant') {
const id = ctx.currentAssistantMsgId || '';
ws.send(JSON.stringify({ type: 'message_end', data: { id } }));
saveAssistantMessage(convId, event.message as AssistantMessage);
ctx.currentAssistantMsgId = null;
}
break;
}
case 'tool_execution_start': {
ws.send(JSON.stringify({ type: 'tool_start', data: { tool: event.toolName, input: event.args } }));
break;
}
case 'tool_execution_end': {
const resultText = event.result?.content?.map((c: any) => c.text || '').join('') || '';
if (event.isError) {
ws.send(JSON.stringify({ type: 'tool_error', data: { tool: event.toolName, error: resultText } }));
} else {
ws.send(JSON.stringify({ type: 'tool_result', data: { tool: event.toolName, result: resultText.slice(0, 1000) } }));
}
saveToolResult(convId, event.toolCallId, event.toolName, resultText, event.isError);
break;
}
}
}

View File

@@ -0,0 +1,57 @@
import { registerBuiltInApiProviders } from '@earendil-works/pi-ai';
import type { Model } from '@earendil-works/pi-ai';
import { getDb } from '../db';
registerBuiltInApiProviders();
export interface PiModelConfig {
model: Model<any>;
apiKey: string;
}
export function createPiModel(): PiModelConfig {
const row = getDb().prepare('SELECT value FROM configs WHERE key = ?').get('api_keys') as { value: string } | undefined;
let apiKey = process.env.ANTHROPIC_API_KEY || '';
let baseURL: string | undefined;
let modelId = process.env.ANTHROPIC_MODEL || 'claude-sonnet-4-6';
let protocol: 'anthropic' | 'openai' = 'anthropic';
if (row) {
try {
const cfg = JSON.parse(row.value);
if (cfg.ANTHROPIC_AUTH_TOKEN) apiKey = cfg.ANTHROPIC_AUTH_TOKEN;
if (cfg.ANTHROPIC_BASE_URL) baseURL = cfg.ANTHROPIC_BASE_URL;
if (cfg.ANTHROPIC_MODEL) modelId = cfg.ANTHROPIC_MODEL;
if (cfg.PROTOCOL === 'openai') protocol = 'openai';
} catch {}
}
const model: Model<any> = protocol === 'openai'
? {
id: modelId,
name: modelId,
api: 'openai-completions',
provider: 'openai',
baseUrl: baseURL || 'https://api.openai.com/v1',
reasoning: false,
input: ['text'] as ('text' | 'image')[],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 8192,
}
: {
id: modelId,
name: modelId,
api: 'anthropic-messages',
provider: 'anthropic',
baseUrl: baseURL || 'https://api.anthropic.com',
reasoning: true,
input: ['text', 'image'] as ('text' | 'image')[],
cost: { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 },
contextWindow: 200000,
maxTokens: 8192,
};
return { model, apiKey };
}

View File

@@ -0,0 +1,164 @@
import { randomUUID } from 'crypto';
import { getDb } from '../db';
import type { Message, AssistantMessage, TextContent, ToolCall } from '@earendil-works/pi-ai';
export interface DbMessage {
id: string;
conversation_id: string;
role: string;
content: string;
tool_calls: string | null;
created_at: string;
}
export function dbToPiMessages(dbMessages: DbMessage[]): Message[] {
const sanitized = sanitizeHistory(dbMessages);
return sanitized.map(dbToPiMessage);
}
function dbToPiMessage(msg: DbMessage): Message {
if (msg.role === 'user') {
return { role: 'user', content: msg.content, timestamp: Date.parse(msg.created_at) || Date.now() };
}
if (msg.role === 'assistant') {
const content: (TextContent | ToolCall)[] = [];
const textContent = msg.content && msg.content !== '(调用工具)' ? msg.content : '';
if (textContent) {
content.push({ type: 'text', text: textContent });
}
if (msg.tool_calls) {
try {
const parsed = JSON.parse(msg.tool_calls);
const blocks = Array.isArray(parsed) ? parsed : parsed.content_blocks;
if (Array.isArray(blocks)) {
for (const b of blocks) {
if (b.type === 'tool_use') {
content.push({
type: 'toolCall',
id: b.id,
name: b.name,
arguments: b.input || b.arguments || {},
});
}
}
}
} catch {}
}
return {
role: 'assistant',
content,
api: 'unknown' as any,
provider: 'unknown',
model: 'unknown',
usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } },
stopReason: 'stop',
timestamp: Date.parse(msg.created_at) || Date.now(),
};
}
if (msg.role === 'tool') {
try {
const { tool_use_id, content: resultContent } = JSON.parse(msg.content);
return {
role: 'toolResult',
toolCallId: tool_use_id,
toolName: '',
content: [{ type: 'text', text: resultContent }],
isError: false,
timestamp: Date.parse(msg.created_at) || Date.now(),
};
} catch {
return {
role: 'toolResult',
toolCallId: 'unknown',
toolName: '',
content: [{ type: 'text', text: msg.content }],
isError: true,
timestamp: Date.parse(msg.created_at) || Date.now(),
};
}
}
return { role: 'user', content: msg.content, timestamp: Date.now() };
}
function sanitizeHistory(messages: DbMessage[]): DbMessage[] {
const cleaned = messages.filter((m) => {
if (m.role === 'assistant' && m.content.startsWith('抱歉,出错了:')) return false;
return true;
});
const result: DbMessage[] = [];
for (const msg of cleaned) {
if (msg.role === 'tool') {
let hasPrecedingToolCall = false;
for (let j = result.length - 1; j >= 0; j--) {
const prev = result[j];
if (prev.role === 'assistant') {
if (prev.tool_calls) {
try {
const parsed = JSON.parse(prev.tool_calls);
const blocks = Array.isArray(parsed) ? parsed : parsed.content_blocks;
if (blocks?.some((b: any) => b.type === 'tool_use')) {
hasPrecedingToolCall = true;
}
} catch {}
}
break;
}
if (prev.role === 'tool') continue;
break;
}
if (!hasPrecedingToolCall) continue;
}
result.push(msg);
}
return result;
}
export function saveUserMessage(convId: string, content: string): string {
const id = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(id, convId, 'user', content);
return id;
}
export function saveAssistantMessage(convId: string, msg: AssistantMessage): string {
const id = randomUUID();
const textParts = msg.content.filter((c): c is TextContent => c.type === 'text');
const text = textParts.map((c) => c.text).join('');
const toolCalls = msg.content.filter((c): c is ToolCall => c.type === 'toolCall');
if (toolCalls.length > 0) {
const dbToolCalls = toolCalls.map((tc) => ({
type: 'tool_use',
id: tc.id,
name: tc.name,
input: tc.arguments,
}));
const contentBlocks = text ? [{ type: 'text', text }, ...dbToolCalls] : dbToolCalls;
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content, tool_calls) VALUES (?, ?, ?, ?, ?)'
).run(id, convId, 'assistant', text || '(调用工具)', JSON.stringify(contentBlocks));
} else {
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(id, convId, 'assistant', text);
}
return id;
}
export function saveToolResult(convId: string, toolCallId: string, toolName: string, result: string, isError: boolean): string {
const id = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(id, convId, 'tool', JSON.stringify({ tool_use_id: toolCallId, tool_name: toolName, content: result }));
return id;
}

View File

@@ -0,0 +1,18 @@
import type { AgentTool, AgentToolResult } from '@earendil-works/pi-agent-core';
import type { ToolDefinition } from './tools/types';
export function createPiTools(tools: ToolDefinition[]): AgentTool[] {
return tools.map((t): AgentTool => ({
name: t.name,
description: t.description,
parameters: t.input_schema as any,
label: t.name,
execute: async (_toolCallId: string, params: any): Promise<AgentToolResult<any>> => {
const result = await t.execute(params);
return {
content: [{ type: 'text' as const, text: result }],
details: null,
};
},
}));
}

View File

@@ -2,8 +2,11 @@ import { Router } from 'express';
import fs from 'fs/promises'; import fs from 'fs/promises';
import fss from 'fs'; import fss from 'fs';
import path from 'path'; import path from 'path';
import { fileURLToPath } from 'url';
import multer from 'multer'; import multer from 'multer';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const ACCOUNTS_DIR = path.resolve(__dirname, '..', '..', '..', 'accounts'); const ACCOUNTS_DIR = path.resolve(__dirname, '..', '..', '..', 'accounts');
export const accountsRouter = Router(); export const accountsRouter = Router();

View File

@@ -1,7 +1,10 @@
import { Router } from 'express'; import { Router } from 'express';
import fs from 'fs/promises'; import fs from 'fs/promises';
import path from 'path'; import path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..'); const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..');
export const promptsRouter = Router(); export const promptsRouter = Router();

View File

@@ -1,54 +1,7 @@
import { WebSocket } from 'ws'; import { WebSocket } from 'ws';
import { randomUUID } from 'crypto'; import { randomUUID } from 'crypto';
import { getDb } from '../db'; import { getDb } from '../db';
import { videoAgent } from '../agent'; import { runAgentChat } from '../agent/pi-bridge';
import type Anthropic from '@anthropic-ai/sdk';
import type OpenAI from 'openai';
type MessageParam = Anthropic.MessageParam;
type ContentBlock = Anthropic.ContentBlock;
interface DbMessage {
id: string;
conversation_id: string;
role: string;
content: string;
tool_calls: string | null;
created_at: string;
}
function filterContent(blocks: ContentBlock[]): ContentBlock[] {
return blocks.filter((b) => b.type === 'text' || b.type === 'tool_use');
}
function dbToAnthropic(msg: DbMessage): MessageParam {
if (msg.role === 'user') {
return { role: 'user', content: msg.content };
}
if (msg.role === 'assistant') {
if (msg.tool_calls) {
try {
const parsed = JSON.parse(msg.tool_calls) as ContentBlock[];
return { role: 'assistant', content: filterContent(parsed) };
} catch {
return { role: 'assistant', content: msg.content };
}
}
return { role: 'assistant', content: msg.content };
}
if (msg.role === 'tool') {
try {
const { tool_use_id, content } = JSON.parse(msg.content);
return {
role: 'user',
content: [{ type: 'tool_result' as const, tool_use_id, content }],
};
} catch {
return { role: 'user', content: msg.content };
}
}
return { role: 'user', content: msg.content };
}
export function handleChat(ws: WebSocket) { export function handleChat(ws: WebSocket) {
let conversationId: string | null = null; let conversationId: string | null = null;
@@ -61,7 +14,7 @@ export function handleChat(ws: WebSocket) {
conversationId = msg.conversationId || randomUUID(); conversationId = msg.conversationId || randomUUID();
const history = getDb().prepare( const history = getDb().prepare(
'SELECT * FROM messages WHERE conversation_id = ? ORDER BY created_at' 'SELECT * FROM messages WHERE conversation_id = ? ORDER BY created_at'
).all(conversationId) as DbMessage[]; ).all(conversationId);
ws.send(JSON.stringify({ type: 'history', data: { conversationId, messages: history } })); ws.send(JSON.stringify({ type: 'history', data: { conversationId, messages: history } }));
return; return;
} }
@@ -81,7 +34,7 @@ export function handleChat(ws: WebSocket) {
ws.send(JSON.stringify({ type: 'error', data: { message: '没有活跃对话,请先创建或选择一个对话' } })); ws.send(JSON.stringify({ type: 'error', data: { message: '没有活跃对话,请先创建或选择一个对话' } }));
return; return;
} }
await handleChatMessage(ws, conversationId, msg.content); await runAgentChat(ws, conversationId, msg.content);
} }
} catch (e) { } catch (e) {
console.error('WebSocket error:', e); console.error('WebSocket error:', e);
@@ -91,381 +44,3 @@ export function handleChat(ws: WebSocket) {
ws.on('close', () => {}); ws.on('close', () => {});
} }
// Helper: convert DB messages to OpenAI format
function extractToolCalls(blocks: ContentBlock[]): OpenAI.ChatCompletionMessageToolCall[] {
return blocks
.filter((b) => b.type === 'tool_use')
.map((b) => ({
id: (b as { id: string }).id,
type: 'function' as const,
function: {
name: (b as { name: string }).name,
arguments: JSON.stringify((b as { input: unknown }).input),
},
}));
}
function dbToOpenAI(msg: DbMessage): OpenAI.ChatCompletionMessageParam {
if (msg.role === 'user') {
return { role: 'user', content: msg.content };
}
if (msg.role === 'assistant') {
const result: Record<string, unknown> = { role: 'assistant', content: msg.content || null };
if (!msg.tool_calls) return result as unknown as OpenAI.ChatCompletionMessageParam;
let parsed: unknown;
try { parsed = JSON.parse(msg.tool_calls); } catch { return result as unknown as OpenAI.ChatCompletionMessageParam; }
// Legacy format: parsed is ContentBlock[] (array)
if (Array.isArray(parsed)) {
const toolCalls = extractToolCalls(parsed);
if (toolCalls.length > 0) {
result.tool_calls = toolCalls;
const textBlocks = parsed.filter((b) => b.type === 'text');
result.content = textBlocks.map((b) => (b as { text: string }).text).join('') || null;
}
return result as unknown as OpenAI.ChatCompletionMessageParam;
}
// New format: parsed is { reasoning_content?, content_blocks? }
const meta = parsed as { reasoning_content?: string; content_blocks?: ContentBlock[] };
if (meta.reasoning_content) {
result.reasoning_content = meta.reasoning_content;
}
if (meta.content_blocks) {
const toolCalls = extractToolCalls(meta.content_blocks);
if (toolCalls.length > 0) {
result.tool_calls = toolCalls;
const textBlocks = meta.content_blocks.filter((b) => b.type === 'text');
result.content = textBlocks.map((b) => (b as { text: string }).text).join('') || null;
}
}
return result as unknown as OpenAI.ChatCompletionMessageParam;
}
if (msg.role === 'tool') {
try {
const { tool_use_id, content } = JSON.parse(msg.content);
return { role: 'tool', tool_call_id: tool_use_id, content };
} catch {
return { role: 'tool', tool_call_id: 'unknown', content: msg.content };
}
}
return { role: 'user', content: msg.content };
}
// --- Anthropic protocol streaming ---
async function streamAnthropic(
ws: WebSocket,
convId: string,
messages: MessageParam[],
): Promise<void> {
const client = videoAgent.getAnthropicClient();
const model = videoAgent.getModel();
const systemPrompt = videoAgent.getSystemPrompt();
let currentMessages = messages;
let maxLoops = 10;
while (maxLoops-- > 0) {
console.log(`[chat:anthropic] Loop ${9 - maxLoops}, messages: ${currentMessages.length}`);
const stream = client.messages.stream({
model,
max_tokens: 4096,
system: systemPrompt,
tools: videoAgent.getAnthropicTools(),
messages: currentMessages,
});
const assistantMsgId = randomUUID();
ws.send(JSON.stringify({ type: 'message_start', data: { id: assistantMsgId } }));
for await (const event of stream) {
if (event.type === 'content_block_delta') {
if (event.delta.type === 'text_delta') {
ws.send(JSON.stringify({
type: 'text_delta',
data: { id: assistantMsgId, text: event.delta.text },
}));
}
}
}
const finalMsg = await stream.finalMessage();
ws.send(JSON.stringify({ type: 'message_end', data: { id: assistantMsgId } }));
const toolUses = finalMsg.content.filter((b): b is Anthropic.ToolUseBlock => b.type === 'tool_use');
const textBlocks = finalMsg.content.filter((b): b is Anthropic.TextBlock => b.type === 'text');
const finalText = textBlocks.map((b) => b.text).join('');
if (toolUses.length === 0) {
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', finalText);
return;
}
// Save assistant with tool calls
const cleanContent = filterContent(finalMsg.content as ContentBlock[]);
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content, tool_calls) VALUES (?, ?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', finalText || '(调用工具)', JSON.stringify(cleanContent));
currentMessages.push({ role: 'assistant', content: cleanContent });
// Execute tools
const toolResults: Anthropic.ToolResultBlockParam[] = [];
for (const tool of toolUses) {
ws.send(JSON.stringify({ type: 'tool_start', data: { tool: tool.name, input: tool.input } }));
console.log(`[chat:anthropic] Executing tool: ${tool.name}`);
try {
const result = await videoAgent.executeTool(tool.name, tool.input as Record<string, unknown>);
toolResults.push({ type: 'tool_result', tool_use_id: tool.id, content: result });
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tool.id, content: result }));
ws.send(JSON.stringify({ type: 'tool_result', data: { tool: tool.name, result: result.slice(0, 1000) } }));
} catch (err) {
const errMsg = (err as Error).message;
toolResults.push({ type: 'tool_result', tool_use_id: tool.id, content: `Error: ${errMsg}` });
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tool.id, content: `Error: ${errMsg}` }));
ws.send(JSON.stringify({ type: 'tool_error', data: { tool: tool.name, error: errMsg } }));
}
}
currentMessages.push({ role: 'user', content: toolResults });
}
}
// --- OpenAI protocol streaming ---
async function streamOpenAI(
ws: WebSocket,
convId: string,
dbMessages: DbMessage[],
): Promise<void> {
const client = videoAgent.getOpenAIClient();
const model = videoAgent.getModel();
const systemPrompt = videoAgent.getSystemPrompt();
const openaiTools = videoAgent.getOpenAITools();
let currentDbMessages = [...dbMessages];
let maxLoops = 10;
while (maxLoops-- > 0) {
const openaiMessages: OpenAI.ChatCompletionMessageParam[] = [
{ role: 'system', content: systemPrompt },
...currentDbMessages.map(dbToOpenAI),
];
console.log(`[chat:openai] Loop ${9 - maxLoops}, messages: ${openaiMessages.length}`);
const assistantMsgId = randomUUID();
ws.send(JSON.stringify({ type: 'message_start', data: { id: assistantMsgId } }));
let fullText = '';
let reasoningContent = '';
let toolCallsAcc: Array<{ id: string; name: string; arguments: string }> = [];
const stream = await client.chat.completions.create({
model,
messages: openaiMessages,
tools: openaiTools.length > 0 ? openaiTools : undefined,
stream: true,
});
for await (const chunk of stream) {
const delta = chunk.choices[0]?.delta;
if (delta?.content) {
fullText += delta.content;
ws.send(JSON.stringify({
type: 'text_delta',
data: { id: assistantMsgId, text: delta.content },
}));
}
// DeepSeek thinking mode: capture reasoning_content
if ((delta as Record<string, unknown>)?.reasoning_content) {
const chunk = (delta as Record<string, unknown>).reasoning_content as string;
reasoningContent += chunk;
ws.send(JSON.stringify({
type: 'reasoning_delta',
data: { id: assistantMsgId, text: chunk },
}));
}
if (delta?.tool_calls) {
for (const tc of delta.tool_calls) {
if (tc.index !== undefined) {
while (toolCallsAcc.length <= tc.index) {
toolCallsAcc.push({ id: '', name: '', arguments: '' });
}
if (tc.id) toolCallsAcc[tc.index].id = tc.id;
if (tc.function?.name) toolCallsAcc[tc.index].name = tc.function.name;
if (tc.function?.arguments) toolCallsAcc[tc.index].arguments += tc.function.arguments;
}
}
}
}
ws.send(JSON.stringify({ type: 'message_end', data: { id: assistantMsgId } }));
// Store extra metadata (reasoning_content, tool_calls) in tool_calls column as JSON
const extraMeta: Record<string, unknown> = {};
if (reasoningContent) extraMeta.reasoning_content = reasoningContent;
// No tool calls — save and done
if (toolCallsAcc.length === 0) {
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content, tool_calls) VALUES (?, ?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', fullText, Object.keys(extraMeta).length > 0 ? JSON.stringify(extraMeta) : null);
return;
}
// Save assistant with tool calls in Anthropic-compatible format for DB
const dbToolCalls = toolCallsAcc.map((tc) => ({
type: 'tool_use',
id: tc.id,
name: tc.name,
input: JSON.parse(tc.arguments || '{}'),
}));
const cleanContent = fullText
? [{ type: 'text', text: fullText }, ...dbToolCalls]
: dbToolCalls;
extraMeta.content_blocks = cleanContent;
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content, tool_calls) VALUES (?, ?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', fullText || '(调用工具)', JSON.stringify(extraMeta));
// Execute tools and collect results
for (const tc of toolCallsAcc) {
ws.send(JSON.stringify({ type: 'tool_start', data: { tool: tc.name, input: JSON.parse(tc.arguments || '{}') } }));
console.log(`[chat:openai] Executing tool: ${tc.name}`);
try {
const params = JSON.parse(tc.arguments || '{}');
const result = await videoAgent.executeTool(tc.name, params);
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tc.id, content: result }));
ws.send(JSON.stringify({ type: 'tool_result', data: { tool: tc.name, result: result.slice(0, 1000) } }));
} catch (err) {
const errMsg = (err as Error).message;
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tc.id, content: `Error: ${errMsg}` }));
ws.send(JSON.stringify({ type: 'tool_error', data: { tool: tc.name, error: errMsg } }));
}
}
// Reload all messages for next loop
currentDbMessages = getDb().prepare(
'SELECT * FROM messages WHERE conversation_id = ? ORDER BY created_at'
).all(convId) as DbMessage[];
}
}
// Clean and validate message history before sending to API
function sanitizeHistory(messages: DbMessage[]): DbMessage[] {
// 1. Filter out error messages stored in DB
const cleaned = messages.filter((m) => {
if (m.role === 'assistant' && m.content.startsWith('抱歉,出错了:')) return false;
return true;
});
// 2. Validate sequence: tool messages must follow assistant with tool_calls
const result: DbMessage[] = [];
for (let i = 0; i < cleaned.length; i++) {
const msg = cleaned[i];
// Tool message: check if preceding assistant has tool_calls
if (msg.role === 'tool') {
let hasPrecedingToolCall = false;
for (let j = result.length - 1; j >= 0; j--) {
const prev = result[j];
if (prev.role === 'assistant') {
if (prev.tool_calls) {
try {
const parsed = JSON.parse(prev.tool_calls);
const blocks = Array.isArray(parsed) ? parsed : parsed.content_blocks;
if (blocks?.some((b: ContentBlock) => b.type === 'tool_use')) {
hasPrecedingToolCall = true;
}
} catch {}
}
break; // stop at nearest assistant
}
if (prev.role === 'tool') continue; // skip consecutive tool messages
break;
}
if (!hasPrecedingToolCall) continue; // skip orphan tool message
}
result.push(msg);
}
return result;
}
async function handleChatMessage(ws: WebSocket, convId: string, content: string) {
const userMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(userMsgId, convId, 'user', content);
ws.send(JSON.stringify({ type: 'message', data: { id: userMsgId, role: 'user', content } }));
const msgCount = getDb().prepare(
'SELECT COUNT(*) as count FROM messages WHERE conversation_id = ?'
).get(convId) as { count: number };
if (msgCount.count <= 1) {
const title = content.slice(0, 30) + (content.length > 30 ? '...' : '');
getDb().prepare('UPDATE conversations SET title = ?, updated_at = datetime(\'now\') WHERE id = ?')
.run(title, convId);
}
getDb().prepare('UPDATE conversations SET updated_at = datetime(\'now\') WHERE id = ?').run(convId);
const rawHistory = getDb().prepare(
'SELECT * FROM messages WHERE conversation_id = ? AND id != ? ORDER BY created_at'
).all(convId, userMsgId) as DbMessage[];
const history = sanitizeHistory(rawHistory);
ws.send(JSON.stringify({ type: 'status', data: { status: 'thinking' } }));
try {
const protocol = videoAgent.getProtocol();
if (protocol === 'openai') {
// OpenAI protocol
await streamOpenAI(ws, convId, history);
} else {
// Anthropic protocol (default)
const messages: MessageParam[] = history.map(dbToAnthropic);
await streamAnthropic(ws, convId, messages);
}
} catch (err) {
const errMsg = (err as Error).message;
console.error('[chat] LLM error:', errMsg);
const errId = randomUUID();
// Don't store error in DB to avoid polluting history
ws.send(JSON.stringify({
type: 'message',
data: { id: errId, role: 'assistant', content: `抱歉,出错了:${errMsg}` },
}));
}
}