feat(web): 重构前端UI并支持OpenAI协议

- 添加账号管理详情页(基本信息、提示词、CapCut、参考图标签页)
- 重构资产页面,按项目组分开展示图片/视频
- 聊天界面支持深度思考内容折叠展示、复制、删除消息
- 设置页面支持Agent配置(Anthropic/OpenAI协议)和工具配置
- 后端支持OpenAI兼容协议流式输出和DeepSeek思考模式
- 添加对话置顶/删除功能、数据库迁移、资产清单API
- 添加账号参考图上传/删除、技能配置持久化、连接测试API
This commit is contained in:
2026-05-07 23:48:26 +08:00
parent 01963aac96
commit 088bdb9a8e
40 changed files with 2594 additions and 678 deletions

View File

@@ -1,4 +1,5 @@
import Anthropic from '@anthropic-ai/sdk';
import OpenAI from 'openai';
import { tools, ToolDefinition } from './tools';
import { getDb } from '../db';
import fs from 'fs';
@@ -9,34 +10,44 @@ const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..', '..');
function getAnthropicClient(): Anthropic {
export type Protocol = 'anthropic' | 'openai';
interface ApiConfig {
protocol: Protocol;
apiKey: string;
baseURL: string | undefined;
model: string;
}
function getApiConfig(): ApiConfig {
const configRow = getDb().prepare('SELECT value FROM configs WHERE key = ?').get('api_keys') as { value: string } | undefined;
let apiKey = process.env.ANTHROPIC_API_KEY || '';
let baseURL: string | undefined;
let model = process.env.ANTHROPIC_MODEL || 'claude-sonnet-4-6';
let protocol: Protocol = 'anthropic';
if (configRow) {
try {
const cfg = JSON.parse(configRow.value);
if (cfg.ANTHROPIC_AUTH_TOKEN) apiKey = cfg.ANTHROPIC_AUTH_TOKEN;
if (cfg.ANTHROPIC_BASE_URL) baseURL = cfg.ANTHROPIC_BASE_URL;
if (cfg.ANTHROPIC_MODEL) model = cfg.ANTHROPIC_MODEL;
if (cfg.PROTOCOL === 'openai') protocol = 'openai';
} catch {}
}
return new Anthropic({
apiKey,
baseURL,
});
return { protocol, apiKey, baseURL, model };
}
function getModel(): string {
const configRow = getDb().prepare('SELECT value FROM configs WHERE key = ?').get('api_keys') as { value: string } | undefined;
if (configRow) {
try {
const cfg = JSON.parse(configRow.value);
if (cfg.ANTHROPIC_MODEL) return cfg.ANTHROPIC_MODEL;
} catch {}
}
return process.env.ANTHROPIC_MODEL || 'claude-sonnet-4-6';
function getAnthropicClient(): Anthropic {
const { apiKey, baseURL } = getApiConfig();
return new Anthropic({ apiKey, baseURL });
}
function getOpenAIClient(): OpenAI {
const { apiKey, baseURL } = getApiConfig();
return new OpenAI({ apiKey, baseURL: baseURL || 'https://api.openai.com/v1' });
}
export class VideoAgent {
@@ -46,6 +57,22 @@ export class VideoAgent {
this.tools = tools;
}
getProtocol(): Protocol {
return getApiConfig().protocol;
}
getModel(): string {
return getApiConfig().model;
}
getAnthropicClient(): Anthropic {
return getAnthropicClient();
}
getOpenAIClient(): OpenAI {
return getOpenAIClient();
}
getAnthropicTools(): Anthropic.Tool[] {
return this.tools.map((t) => ({
name: t.name,
@@ -54,6 +81,17 @@ export class VideoAgent {
}));
}
getOpenAITools(): OpenAI.ChatCompletionTool[] {
return this.tools.map((t) => ({
type: 'function' as const,
function: {
name: t.name,
description: t.description,
parameters: t.input_schema,
},
}));
}
async executeTool(name: string, params: Record<string, unknown>): Promise<string> {
const tool = this.tools.find((t) => t.name === name);
if (!tool) throw new Error(`Unknown tool: ${name}`);
@@ -61,7 +99,6 @@ export class VideoAgent {
}
getSystemPrompt(): string {
// Dynamically list accounts
const accountsDir = path.join(PROJECT_ROOT, 'accounts');
let accountList = '暂无账号';
if (fs.existsSync(accountsDir)) {
@@ -107,14 +144,6 @@ ${accountList}
- 如果用户只是闲聊,就闲聊。如果用户想做视频,引导完成流程
- 不要编造账号或文件路径,使用工具获取真实信息`;
}
getClient(): Anthropic {
return getAnthropicClient();
}
getModel(): string {
return getModel();
}
}
export const videoAgent = new VideoAgent();

View File

@@ -6,7 +6,6 @@ import { SCHEMA_SQL } from './schema';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const DB_PATH = path.resolve(__dirname, '..', '..', 'data', 'meitu-agent.db');
let db: Database.Database;
@@ -22,7 +21,18 @@ export function getDb(): Database.Database {
return db;
}
// Ensure a column exists on a table, add it with DEFAULT if missing
function ensureColumn(d: Database.Database, table: string, column: string, definition: string) {
const cols = d.prepare(`PRAGMA table_info('${table}')`).all() as { name: string }[];
if (!cols.some((c) => c.name === column)) {
d.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
}
}
export function initDb(): void {
const d = getDb();
d.exec(SCHEMA_SQL);
// Migrations for existing databases
ensureColumn(d, 'conversations', 'pinned', 'INTEGER NOT NULL DEFAULT 0');
}

View File

@@ -3,6 +3,7 @@ export const SCHEMA_SQL = `
id TEXT PRIMARY KEY,
title TEXT NOT NULL,
account_id TEXT,
pinned INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);

View File

@@ -1,49 +1,60 @@
import { Router } from 'express';
import fs from 'fs';
import fs from 'fs/promises';
import path from 'path';
import multer from 'multer';
const ACCOUNTS_DIR = path.resolve(__dirname, '..', '..', '..', 'accounts');
export const accountsRouter = Router();
function readAccountJson(id: string): Record<string, unknown> | null {
async function readAccountJson(id: string): Promise<Record<string, unknown> | null> {
const p = path.join(ACCOUNTS_DIR, id, 'account.json');
if (!fs.existsSync(p)) return null;
return JSON.parse(fs.readFileSync(p, 'utf-8'));
try {
const data = await fs.readFile(p, 'utf-8');
return JSON.parse(data);
} catch {
return null;
}
}
function writeAccountJson(id: string, data: Record<string, unknown>): void {
async function writeAccountJson(id: string, data: Record<string, unknown>): Promise<void> {
const dir = path.join(ACCOUNTS_DIR, id);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'account.json'), JSON.stringify(data, null, 2), 'utf-8');
await fs.mkdir(dir, { recursive: true });
await fs.writeFile(path.join(dir, 'account.json'), JSON.stringify(data, null, 2), 'utf-8');
}
// List
accountsRouter.get('/', (_req, res) => {
const dirs = fs.readdirSync(ACCOUNTS_DIR, { withFileTypes: true })
.filter((d) => d.isDirectory() && !d.name.startsWith('_') && !d.name.startsWith('.'))
.map((d) => d.name);
accountsRouter.get('/', async (_req, res) => {
const dirs = await fs.readdir(ACCOUNTS_DIR, { withFileTypes: true });
const accountDirs = dirs.filter((d) => d.isDirectory() && !d.name.startsWith('_') && !d.name.startsWith('.'));
const accounts = dirs
.map((id) => readAccountJson(id))
.filter(Boolean);
const accounts: Record<string, unknown>[] = [];
for (const d of accountDirs) {
const data = await readAccountJson(d.name);
if (data) accounts.push(data);
}
res.json(accounts);
});
// Get
accountsRouter.get('/:id', (req, res) => {
const data = readAccountJson(req.params.id);
accountsRouter.get('/:id', async (req, res) => {
const data = await readAccountJson(req.params.id);
if (!data) return res.status(404).json({ error: 'Account not found' });
res.json(data);
});
// Create
accountsRouter.post('/', (req, res) => {
accountsRouter.post('/', async (req, res) => {
const { id, ...rest } = req.body;
if (!id) return res.status(400).json({ error: 'id is required' });
const dir = path.join(ACCOUNTS_DIR, id);
if (fs.existsSync(dir)) return res.status(409).json({ error: 'Account already exists' });
try {
await fs.access(dir);
return res.status(409).json({ error: 'Account already exists' });
} catch {
// Directory doesn't exist, proceed
}
const data = {
id,
@@ -62,24 +73,75 @@ accountsRouter.post('/', (req, res) => {
capcut: rest.capcut || {},
};
writeAccountJson(id, data);
await writeAccountJson(id, data);
res.status(201).json(data);
});
// Update
accountsRouter.put('/:id', (req, res) => {
const existing = readAccountJson(req.params.id);
accountsRouter.put('/:id', async (req, res) => {
const existing = await readAccountJson(req.params.id);
if (!existing) return res.status(404).json({ error: 'Account not found' });
const merged = { ...existing, ...req.body, id: req.params.id };
writeAccountJson(req.params.id, merged);
await writeAccountJson(req.params.id, merged);
res.json(merged);
});
// Delete
accountsRouter.delete('/:id', (req, res) => {
accountsRouter.delete('/:id', async (req, res) => {
const dir = path.join(ACCOUNTS_DIR, req.params.id);
if (!fs.existsSync(dir)) return res.status(404).json({ error: 'Account not found' });
fs.rmSync(dir, { recursive: true });
try {
await fs.access(dir);
} catch {
return res.status(404).json({ error: 'Account not found' });
}
await fs.rm(dir, { recursive: true });
res.status(204).send();
});
// Reference image upload
const upload = multer({ dest: path.join(ACCOUNTS_DIR, '_uploads') });
accountsRouter.post('/:id/references/upload', upload.single('file'), async (req, res) => {
const id = req.params.id as string;
const file = req.file as Express.Multer.File | undefined;
if (!file) return res.status(400).json({ error: 'No file uploaded' });
const account = await readAccountJson(id);
if (!account) return res.status(404).json({ error: 'Account not found' });
const refsDir = path.join(ACCOUNTS_DIR, id, 'references');
await fs.mkdir(refsDir, { recursive: true });
const destPath = path.join(refsDir, file.originalname);
await fs.rename(file.path, destPath);
const refEntry = { file: `references/${file.originalname}` };
const refs = (account.references as unknown[]) || [];
refs.push(refEntry);
account.references = refs;
await writeAccountJson(id, account);
res.json({ ok: true, reference: refEntry });
});
// Reference image delete
accountsRouter.delete('/:id/references/:index', async (req, res) => {
const id = req.params.id as string;
const idx = parseInt(req.params.index as string);
const account = await readAccountJson(id);
if (!account) return res.status(404).json({ error: 'Account not found' });
const refs = (account.references as unknown[]) || [];
if (idx < 0 || idx >= refs.length) return res.status(400).json({ error: 'Invalid index' });
const removed = refs.splice(idx, 1)[0] as { file?: string };
if (removed?.file) {
const filePath = path.join(ACCOUNTS_DIR, id, removed.file);
try { await fs.unlink(filePath); } catch { /* ignore */ }
}
account.references = refs;
await writeAccountJson(id, account);
res.json({ ok: true });
});

View File

@@ -1,7 +1,8 @@
import { Router } from 'express';
import { getDb } from '../db';
import { randomUUID } from 'crypto';
import fs from 'fs';
import fs from 'fs/promises';
import fss from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
@@ -25,8 +26,23 @@ assetsRouter.get('/', (req, res) => {
res.json(rows);
});
// List manifests with metadata
assetsRouter.get('/manifests', async (_req, res) => {
const rows = getDb().prepare(`
SELECT manifest_path, account_id,
COUNT(*) as asset_count,
SUM(CASE WHEN type = 'image' THEN 1 ELSE 0 END) as image_count,
SUM(CASE WHEN type = 'video' THEN 1 ELSE 0 END) as video_count,
MAX(created_at) as latest_at
FROM assets
GROUP BY manifest_path
ORDER BY latest_at DESC
`).all();
res.json(rows);
});
// Delete asset and its file
assetsRouter.delete('/:id', (req, res) => {
assetsRouter.delete('/:id', async (req, res) => {
const asset = getDb().prepare('SELECT * FROM assets WHERE id = ?').get(req.params.id) as {
file_path: string;
} | undefined;
@@ -34,36 +50,38 @@ assetsRouter.delete('/:id', (req, res) => {
if (!asset) return res.status(404).json({ error: 'Asset not found' });
const filePath = path.join(PROJECT_ROOT, asset.file_path);
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
try { await fs.unlink(filePath); } catch { /* ignore */ }
getDb().prepare('DELETE FROM assets WHERE id = ?').run(req.params.id);
res.status(204).send();
});
// Scan output directories and index assets
assetsRouter.post('/scan', (_req, res) => {
assetsRouter.post('/scan', async (_req, res) => {
const outputDir = path.join(PROJECT_ROOT, 'output');
if (!fs.existsSync(outputDir)) return res.json({ indexed: 0 });
try {
await fs.access(outputDir);
} catch {
return res.json({ indexed: 0 });
}
const dirs = fs.readdirSync(outputDir, { withFileTypes: true })
.filter((d) => d.isDirectory());
const dirs = await fs.readdir(outputDir, { withFileTypes: true });
const dirEntries = dirs.filter((d) => d.isDirectory());
let indexed = 0;
for (const dir of dirs) {
for (const dir of dirEntries) {
const manifestPath = path.join(outputDir, dir.name, 'manifest.json');
if (!fs.existsSync(manifestPath)) continue;
let manifest;
try {
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
manifest = JSON.parse(await fs.readFile(manifestPath, 'utf-8'));
} catch { continue; }
const accountId = manifest.account?.id || '';
// Scan images
const imagesDir = path.join(outputDir, dir.name, 'images');
if (fs.existsSync(imagesDir)) {
const files = fs.readdirSync(imagesDir);
try {
const files = await fs.readdir(imagesDir);
for (const file of files) {
if (!/\.(jpe?g|png|webp)$/i.test(file)) continue;
const id = randomUUID();
@@ -79,12 +97,12 @@ assetsRouter.post('/scan', (_req, res) => {
).run(id, accountId, path.relative(PROJECT_ROOT, manifestPath), 'image', relPath, shotIndex);
indexed++;
}
}
} catch { /* images dir may not exist */ }
// Scan videos
const videosDir = path.join(outputDir, dir.name, 'videos');
if (fs.existsSync(videosDir)) {
const files = fs.readdirSync(videosDir);
try {
const files = await fs.readdir(videosDir);
for (const file of files) {
if (!/\.mp4$/i.test(file)) continue;
const id = randomUUID();
@@ -100,7 +118,7 @@ assetsRouter.post('/scan', (_req, res) => {
).run(id, accountId, path.relative(PROJECT_ROOT, manifestPath), 'video', relPath, shotIndex);
indexed++;
}
}
} catch { /* videos dir may not exist */ }
}
res.json({ indexed });
@@ -112,6 +130,6 @@ assetsRouter.get('/file', (req, res) => {
if (!filePath) return res.status(400).send('Missing path');
const fullPath = path.resolve(PROJECT_ROOT, filePath);
if (!fullPath.startsWith(PROJECT_ROOT)) return res.status(403).send('Forbidden');
if (!fs.existsSync(fullPath)) return res.status(404).send('Not found');
if (!fss.existsSync(fullPath)) return res.status(404).send('Not found');
res.sendFile(fullPath);
});

View File

@@ -1,7 +1,17 @@
import { Router } from 'express';
import { randomUUID } from 'crypto';
import Anthropic from '@anthropic-ai/sdk';
import OpenAI from 'openai';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { getDb } from '../db';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..');
const SKILLS_CONFIG_PATH = path.join(PROJECT_ROOT, '.claude', 'skills', 'config.json');
export const configsRouter = Router();
configsRouter.get('/', (_req, res) => {
@@ -24,3 +34,77 @@ configsRouter.put('/:key', (req, res) => {
`).run(randomUUID(), req.params.key, JSON.stringify(value));
res.json({ key: req.params.key, ok: true });
});
// Skills config.json (pipeline tools configuration)
configsRouter.get('/skills/file', (_req, res) => {
try {
const data = JSON.parse(fs.readFileSync(SKILLS_CONFIG_PATH, 'utf-8'));
res.json(data);
} catch {
res.status(404).json({ error: 'Skills config not found' });
}
});
configsRouter.put('/skills/file', (req, res) => {
try {
fs.writeFileSync(SKILLS_CONFIG_PATH, JSON.stringify(req.body, null, 2), 'utf-8');
res.json({ ok: true });
} catch (e) {
res.status(500).json({ error: (e as Error).message });
}
});
// Test API connection
configsRouter.post('/test-connection', async (_req, res) => {
const configRow = getDb().prepare('SELECT value FROM configs WHERE key = ?').get('api_keys') as { value: string } | undefined;
if (!configRow) return res.json({ ok: false, error: '未配置 API Key' });
let apiKey = '';
let baseURL: string | undefined;
let protocol: string = 'anthropic';
let model: string = 'claude-sonnet-4-6';
try {
const cfg = JSON.parse(configRow.value);
apiKey = cfg.ANTHROPIC_AUTH_TOKEN || '';
baseURL = cfg.ANTHROPIC_BASE_URL;
protocol = cfg.PROTOCOL || 'anthropic';
model = cfg.ANTHROPIC_MODEL || model;
} catch {
return res.json({ ok: false, error: '配置解析失败' });
}
if (!apiKey) return res.json({ ok: false, error: 'API Key 为空' });
try {
if (protocol === 'openai') {
const client = new OpenAI({ apiKey, baseURL: baseURL || 'https://api.openai.com/v1' });
const models = await client.models.list();
const modelIds: string[] = [];
for await (const m of models) { modelIds.push(m.id); if (modelIds.length >= 5) break; }
res.json({ ok: true, models: modelIds });
} else {
const client = new Anthropic({ apiKey, baseURL });
// Anthropic SDK doesn't have a models.list() in all versions, try a minimal request
try {
await client.messages.create({
model,
max_tokens: 1,
messages: [{ role: 'user', content: 'hi' }],
});
res.json({ ok: true, models: [model] });
} catch (e: unknown) {
const err = e as { status?: number; message?: string };
if (err.status === 401) {
res.json({ ok: false, error: `认证失败: ${err.message}` });
} else if (err.status === 400 || err.status === undefined) {
res.json({ ok: true, models: [model] });
} else {
res.json({ ok: false, error: `错误 (${err.status}): ${err.message}` });
}
}
}
} catch (e) {
res.json({ ok: false, error: (e as Error).message });
}
});

View File

@@ -19,7 +19,7 @@ pipelineRouter.get('/conversations', (req, res) => {
sql += ' WHERE title LIKE ?';
params.push(`%${search}%`);
}
sql += ' ORDER BY updated_at DESC LIMIT 100';
sql += ' ORDER BY pinned DESC, updated_at DESC LIMIT 100';
const rows = getDb().prepare(sql).all(...params);
res.json(rows);
});
@@ -36,13 +36,19 @@ pipelineRouter.delete('/conversations/:id', (req, res) => {
res.status(204).send();
});
// Rename conversation
// Update conversation (rename / toggle pin)
pipelineRouter.patch('/conversations/:id', (req, res) => {
const { title } = req.body;
if (!title) return res.status(400).json({ error: 'title required' });
getDb().prepare(
'UPDATE conversations SET title = ?, updated_at = datetime(\'now\') WHERE id = ?'
).run(title, req.params.id);
const { title, pinned } = req.body;
if (title !== undefined) {
getDb().prepare(
'UPDATE conversations SET title = ?, updated_at = datetime(\'now\') WHERE id = ?'
).run(title, req.params.id);
}
if (pinned !== undefined) {
getDb().prepare(
'UPDATE conversations SET pinned = ?, updated_at = datetime(\'now\') WHERE id = ?'
).run(pinned ? 1 : 0, req.params.id);
}
res.json({ ok: true });
});

View File

@@ -1,5 +1,5 @@
import { Router } from 'express';
import fs from 'fs';
import fs from 'fs/promises';
import path from 'path';
const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..');
@@ -12,18 +12,21 @@ const PROMPT_FILES: Record<string, string> = {
video: 'prompts/视频提示词.md',
};
promptsRouter.get('/:accountId/:type', (req, res) => {
promptsRouter.get('/:accountId/:type', async (req, res) => {
const { accountId, type } = req.params;
const relPath = PROMPT_FILES[type];
if (!relPath) return res.status(400).json({ error: 'Unknown type: ' + type });
const fullPath = path.join(PROJECT_ROOT, 'accounts', accountId, relPath);
if (!fs.existsSync(fullPath)) return res.status(404).json({ error: 'File not found' });
res.json({ path: relPath, content: fs.readFileSync(fullPath, 'utf-8') });
try {
const content = await fs.readFile(fullPath, 'utf-8');
res.json({ path: relPath, content });
} catch {
res.status(404).json({ error: 'File not found' });
}
});
promptsRouter.put('/:accountId/:type', (req, res) => {
promptsRouter.put('/:accountId/:type', async (req, res) => {
const { accountId, type } = req.params;
const { content } = req.body;
const relPath = PROMPT_FILES[type];
@@ -31,9 +34,8 @@ promptsRouter.put('/:accountId/:type', (req, res) => {
const fullPath = path.join(PROJECT_ROOT, 'accounts', accountId, relPath);
const dir = path.dirname(fullPath);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(fullPath, content, 'utf-8');
await fs.mkdir(dir, { recursive: true });
await fs.writeFile(fullPath, content, 'utf-8');
res.json({ ok: true });
});

View File

@@ -3,6 +3,7 @@ import { randomUUID } from 'crypto';
import { getDb } from '../db';
import { videoAgent } from '../agent';
import type Anthropic from '@anthropic-ai/sdk';
import type OpenAI from 'openai';
type MessageParam = Anthropic.MessageParam;
type ContentBlock = Anthropic.ContentBlock;
@@ -91,6 +92,295 @@ export function handleChat(ws: WebSocket) {
ws.on('close', () => {});
}
// Helper: convert DB messages to OpenAI format
function extractToolCalls(blocks: ContentBlock[]): OpenAI.ChatCompletionMessageToolCall[] {
return blocks
.filter((b) => b.type === 'tool_use')
.map((b) => ({
id: (b as { id: string }).id,
type: 'function' as const,
function: {
name: (b as { name: string }).name,
arguments: JSON.stringify((b as { input: unknown }).input),
},
}));
}
function dbToOpenAI(msg: DbMessage): OpenAI.ChatCompletionMessageParam {
if (msg.role === 'user') {
return { role: 'user', content: msg.content };
}
if (msg.role === 'assistant') {
const result: Record<string, unknown> = { role: 'assistant', content: msg.content || null };
if (!msg.tool_calls) return result as unknown as OpenAI.ChatCompletionMessageParam;
let parsed: unknown;
try { parsed = JSON.parse(msg.tool_calls); } catch { return result as unknown as OpenAI.ChatCompletionMessageParam; }
// Legacy format: parsed is ContentBlock[] (array)
if (Array.isArray(parsed)) {
const toolCalls = extractToolCalls(parsed);
if (toolCalls.length > 0) {
result.tool_calls = toolCalls;
const textBlocks = parsed.filter((b) => b.type === 'text');
result.content = textBlocks.map((b) => (b as { text: string }).text).join('') || null;
}
return result as unknown as OpenAI.ChatCompletionMessageParam;
}
// New format: parsed is { reasoning_content?, content_blocks? }
const meta = parsed as { reasoning_content?: string; content_blocks?: ContentBlock[] };
if (meta.reasoning_content) {
result.reasoning_content = meta.reasoning_content;
}
if (meta.content_blocks) {
const toolCalls = extractToolCalls(meta.content_blocks);
if (toolCalls.length > 0) {
result.tool_calls = toolCalls;
const textBlocks = meta.content_blocks.filter((b) => b.type === 'text');
result.content = textBlocks.map((b) => (b as { text: string }).text).join('') || null;
}
}
return result as unknown as OpenAI.ChatCompletionMessageParam;
}
if (msg.role === 'tool') {
try {
const { tool_use_id, content } = JSON.parse(msg.content);
return { role: 'tool', tool_call_id: tool_use_id, content };
} catch {
return { role: 'tool', tool_call_id: 'unknown', content: msg.content };
}
}
return { role: 'user', content: msg.content };
}
// --- Anthropic protocol streaming ---
async function streamAnthropic(
ws: WebSocket,
convId: string,
messages: MessageParam[],
): Promise<void> {
const client = videoAgent.getAnthropicClient();
const model = videoAgent.getModel();
const systemPrompt = videoAgent.getSystemPrompt();
let currentMessages = messages;
let maxLoops = 10;
while (maxLoops-- > 0) {
console.log(`[chat:anthropic] Loop ${9 - maxLoops}, messages: ${currentMessages.length}`);
const stream = client.messages.stream({
model,
max_tokens: 4096,
system: systemPrompt,
tools: videoAgent.getAnthropicTools(),
messages: currentMessages,
});
const assistantMsgId = randomUUID();
ws.send(JSON.stringify({ type: 'message_start', data: { id: assistantMsgId } }));
for await (const event of stream) {
if (event.type === 'content_block_delta') {
if (event.delta.type === 'text_delta') {
ws.send(JSON.stringify({
type: 'text_delta',
data: { id: assistantMsgId, text: event.delta.text },
}));
}
}
}
const finalMsg = await stream.finalMessage();
ws.send(JSON.stringify({ type: 'message_end', data: { id: assistantMsgId } }));
const toolUses = finalMsg.content.filter((b): b is Anthropic.ToolUseBlock => b.type === 'tool_use');
const textBlocks = finalMsg.content.filter((b): b is Anthropic.TextBlock => b.type === 'text');
const finalText = textBlocks.map((b) => b.text).join('');
if (toolUses.length === 0) {
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', finalText);
return;
}
// Save assistant with tool calls
const cleanContent = filterContent(finalMsg.content as ContentBlock[]);
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content, tool_calls) VALUES (?, ?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', finalText || '(调用工具)', JSON.stringify(cleanContent));
currentMessages.push({ role: 'assistant', content: cleanContent });
// Execute tools
const toolResults: Anthropic.ToolResultBlockParam[] = [];
for (const tool of toolUses) {
ws.send(JSON.stringify({ type: 'tool_start', data: { tool: tool.name, input: tool.input } }));
console.log(`[chat:anthropic] Executing tool: ${tool.name}`);
try {
const result = await videoAgent.executeTool(tool.name, tool.input as Record<string, unknown>);
toolResults.push({ type: 'tool_result', tool_use_id: tool.id, content: result });
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tool.id, content: result }));
ws.send(JSON.stringify({ type: 'tool_result', data: { tool: tool.name, result: result.slice(0, 1000) } }));
} catch (err) {
const errMsg = (err as Error).message;
toolResults.push({ type: 'tool_result', tool_use_id: tool.id, content: `Error: ${errMsg}` });
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tool.id, content: `Error: ${errMsg}` }));
ws.send(JSON.stringify({ type: 'tool_error', data: { tool: tool.name, error: errMsg } }));
}
}
currentMessages.push({ role: 'user', content: toolResults });
}
}
// --- OpenAI protocol streaming ---
async function streamOpenAI(
ws: WebSocket,
convId: string,
dbMessages: DbMessage[],
): Promise<void> {
const client = videoAgent.getOpenAIClient();
const model = videoAgent.getModel();
const systemPrompt = videoAgent.getSystemPrompt();
const openaiTools = videoAgent.getOpenAITools();
let currentDbMessages = [...dbMessages];
let maxLoops = 10;
while (maxLoops-- > 0) {
const openaiMessages: OpenAI.ChatCompletionMessageParam[] = [
{ role: 'system', content: systemPrompt },
...currentDbMessages.map(dbToOpenAI),
];
console.log(`[chat:openai] Loop ${9 - maxLoops}, messages: ${openaiMessages.length}`);
const assistantMsgId = randomUUID();
ws.send(JSON.stringify({ type: 'message_start', data: { id: assistantMsgId } }));
let fullText = '';
let reasoningContent = '';
let toolCallsAcc: Array<{ id: string; name: string; arguments: string }> = [];
const stream = await client.chat.completions.create({
model,
messages: openaiMessages,
tools: openaiTools.length > 0 ? openaiTools : undefined,
stream: true,
});
for await (const chunk of stream) {
const delta = chunk.choices[0]?.delta;
if (delta?.content) {
fullText += delta.content;
ws.send(JSON.stringify({
type: 'text_delta',
data: { id: assistantMsgId, text: delta.content },
}));
}
// DeepSeek thinking mode: capture reasoning_content
if ((delta as Record<string, unknown>)?.reasoning_content) {
const chunk = (delta as Record<string, unknown>).reasoning_content as string;
reasoningContent += chunk;
ws.send(JSON.stringify({
type: 'reasoning_delta',
data: { id: assistantMsgId, text: chunk },
}));
}
if (delta?.tool_calls) {
for (const tc of delta.tool_calls) {
if (tc.index !== undefined) {
while (toolCallsAcc.length <= tc.index) {
toolCallsAcc.push({ id: '', name: '', arguments: '' });
}
if (tc.id) toolCallsAcc[tc.index].id = tc.id;
if (tc.function?.name) toolCallsAcc[tc.index].name = tc.function.name;
if (tc.function?.arguments) toolCallsAcc[tc.index].arguments += tc.function.arguments;
}
}
}
}
ws.send(JSON.stringify({ type: 'message_end', data: { id: assistantMsgId } }));
// Store extra metadata (reasoning_content, tool_calls) in tool_calls column as JSON
const extraMeta: Record<string, unknown> = {};
if (reasoningContent) extraMeta.reasoning_content = reasoningContent;
// No tool calls — save and done
if (toolCallsAcc.length === 0) {
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content, tool_calls) VALUES (?, ?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', fullText, Object.keys(extraMeta).length > 0 ? JSON.stringify(extraMeta) : null);
return;
}
// Save assistant with tool calls in Anthropic-compatible format for DB
const dbToolCalls = toolCallsAcc.map((tc) => ({
type: 'tool_use',
id: tc.id,
name: tc.name,
input: JSON.parse(tc.arguments || '{}'),
}));
const cleanContent = fullText
? [{ type: 'text', text: fullText }, ...dbToolCalls]
: dbToolCalls;
extraMeta.content_blocks = cleanContent;
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content, tool_calls) VALUES (?, ?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', fullText || '(调用工具)', JSON.stringify(extraMeta));
// Execute tools and collect results
for (const tc of toolCallsAcc) {
ws.send(JSON.stringify({ type: 'tool_start', data: { tool: tc.name, input: JSON.parse(tc.arguments || '{}') } }));
console.log(`[chat:openai] Executing tool: ${tc.name}`);
try {
const params = JSON.parse(tc.arguments || '{}');
const result = await videoAgent.executeTool(tc.name, params);
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tc.id, content: result }));
ws.send(JSON.stringify({ type: 'tool_result', data: { tool: tc.name, result: result.slice(0, 1000) } }));
} catch (err) {
const errMsg = (err as Error).message;
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tc.id, content: `Error: ${errMsg}` }));
ws.send(JSON.stringify({ type: 'tool_error', data: { tool: tc.name, error: errMsg } }));
}
}
// Reload all messages for next loop
currentDbMessages = getDb().prepare(
'SELECT * FROM messages WHERE conversation_id = ? ORDER BY created_at'
).all(convId) as DbMessage[];
}
}
async function handleChatMessage(ws: WebSocket, convId: string, content: string) {
const userMsgId = randomUUID();
getDb().prepare(
@@ -112,106 +402,18 @@ async function handleChatMessage(ws: WebSocket, convId: string, content: string)
'SELECT * FROM messages WHERE conversation_id = ? AND id != ? ORDER BY created_at'
).all(convId, userMsgId) as DbMessage[];
const messages: MessageParam[] = history.map(dbToAnthropic);
const client = videoAgent.getClient();
const model = videoAgent.getModel();
const systemPrompt = videoAgent.getSystemPrompt();
ws.send(JSON.stringify({ type: 'status', data: { status: 'thinking' } }));
try {
let currentMessages = messages;
let maxLoops = 10;
const protocol = videoAgent.getProtocol();
while (maxLoops-- > 0) {
console.log(`[chat] Calling LLM, loop ${9 - maxLoops}, messages: ${currentMessages.length}`);
const stream = client.messages.stream({
model,
max_tokens: 4096,
system: systemPrompt,
tools: videoAgent.getAnthropicTools(),
messages: currentMessages,
});
const assistantMsgId = randomUUID();
ws.send(JSON.stringify({ type: 'message_start', data: { id: assistantMsgId } }));
for await (const event of stream) {
if (event.type === 'content_block_delta') {
if (event.delta.type === 'text_delta') {
ws.send(JSON.stringify({
type: 'text_delta',
data: { id: assistantMsgId, text: event.delta.text },
}));
}
}
}
const finalMsg = await stream.finalMessage();
ws.send(JSON.stringify({ type: 'message_end', data: { id: assistantMsgId } }));
const toolUses = finalMsg.content.filter((b): b is Anthropic.ToolUseBlock => b.type === 'tool_use');
const textBlocks = finalMsg.content.filter((b): b is Anthropic.TextBlock => b.type === 'text');
const finalText = textBlocks.map((b) => b.text).join('');
// No tool calls — save and done
if (toolUses.length === 0) {
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', finalText);
console.log(`[chat] Done, response: ${finalText.slice(0, 80)}`);
return;
}
// Save assistant message with filtered content (no thinking blocks)
const cleanContent = filterContent(finalMsg.content as ContentBlock[]);
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content, tool_calls) VALUES (?, ?, ?, ?, ?)'
).run(assistantMsgId, convId, 'assistant', finalText || '(调用工具)', JSON.stringify(cleanContent));
currentMessages.push({ role: 'assistant', content: cleanContent });
// Execute tools
const toolResults: Anthropic.ToolResultBlockParam[] = [];
for (const tool of toolUses) {
ws.send(JSON.stringify({
type: 'tool_start',
data: { tool: tool.name, input: tool.input },
}));
console.log(`[chat] Executing tool: ${tool.name}`);
try {
const result = await videoAgent.executeTool(tool.name, tool.input as Record<string, unknown>);
toolResults.push({ type: 'tool_result', tool_use_id: tool.id, content: result });
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tool.id, content: result }));
ws.send(JSON.stringify({
type: 'tool_result',
data: { tool: tool.name, result: result.slice(0, 1000) },
}));
} catch (err) {
const errMsg = (err as Error).message;
toolResults.push({ type: 'tool_result', tool_use_id: tool.id, content: `Error: ${errMsg}` });
const toolMsgId = randomUUID();
getDb().prepare(
'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)'
).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tool.id, content: `Error: ${errMsg}` }));
ws.send(JSON.stringify({
type: 'tool_error',
data: { tool: tool.name, error: errMsg },
}));
}
}
currentMessages.push({ role: 'user', content: toolResults });
if (protocol === 'openai') {
// OpenAI protocol
await streamOpenAI(ws, convId, history);
} else {
// Anthropic protocol (default)
const messages: MessageParam[] = history.map(dbToAnthropic);
await streamAnthropic(ws, convId, messages);
}
} catch (err) {
const errMsg = (err as Error).message;