feat(agent): 支持工具执行结果中的图片和视频资产预览

在聊天消息组件中添加资产灯箱预览功能,支持展示工具调用返回的图片和视频资源。新增 `AssetLightbox` 组件用于全屏浏览资产,并扩展消息类型以包含资产元数据。同时引入 `@radix-ui/react-select` 依赖并为服务端添加资产 URL 转换工具函数。
This commit is contained in:
2026-05-08 02:14:28 +08:00
parent e16305840b
commit 803b55605f
9 changed files with 894 additions and 14 deletions

View File

@@ -12,6 +12,7 @@ import { tools } from './tools/index';
import { videoAgent } from './index';
import { dbToPiMessages, saveUserMessage, saveAssistantMessage, saveToolResult, type DbMessage } from './pi-persist';
import { getDb } from '../db';
import { manifestRelToUrl } from './tools/shared';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
@@ -134,13 +135,43 @@ function handleAgentEvent(ws: WebSocket, convId: string, event: AgentEvent, ctx:
case 'tool_execution_end': {
const resultText = event.result?.content?.map((c: any) => c.text || '').join('') || '';
const assets = extractAssets(event.toolName, resultText);
if (event.isError) {
ws.send(JSON.stringify({ type: 'tool_error', data: { tool: event.toolName, error: resultText } }));
} else {
ws.send(JSON.stringify({ type: 'tool_result', data: { tool: event.toolName, result: resultText.slice(0, 1000) } }));
ws.send(JSON.stringify({ type: 'tool_result', data: { tool: event.toolName, result: resultText.slice(0, 1000), assets } }));
}
saveToolResult(convId, event.toolCallId, event.toolName, resultText, event.isError);
break;
}
}
}
function extractAssets(toolName: string, resultText: string): Array<{ type: 'image' | 'video'; url: string; name: string }> {
const assets: Array<{ type: 'image' | 'video'; url: string; name: string }> = [];
try {
const data = JSON.parse(resultText);
if (toolName === 'generate_images' && data.manifestPath && Array.isArray(data.images)) {
for (const img of data.images) {
if (img.file) {
assets.push({ type: 'image', url: manifestRelToUrl(data.manifestPath, img.file), name: img.file.split('/').pop() || '' });
}
if (Array.isArray(img.candidates)) {
for (const c of img.candidates) {
assets.push({ type: 'image', url: manifestRelToUrl(data.manifestPath, c), name: c.split('/').pop() || '' });
}
}
}
}
if (toolName === 'generate_videos' && data.manifestPath && Array.isArray(data.videos)) {
for (const vid of data.videos) {
if (vid.videoUrl) {
assets.push({ type: 'video', url: vid.videoUrl, name: vid.video?.split('/').pop() || 'video' });
} else if (vid.video) {
assets.push({ type: 'video', url: manifestRelToUrl(data.manifestPath, vid.video), name: vid.video.split('/').pop() || '' });
}
}
}
} catch {}
return assets;
}

View File

@@ -16,6 +16,15 @@ export function loadJSON(filePath: string): unknown {
return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
}
export function manifestRelToUrl(manifestPath: string, relPath: string): string {
if (!relPath) return '';
if (relPath.startsWith('http://') || relPath.startsWith('https://')) return relPath;
const manifestDir = path.dirname(path.resolve(manifestPath));
const absPath = path.resolve(manifestDir, relPath);
const relToRoot = path.relative(PROJECT_ROOT, absPath).replace(/\\/g, '/');
return `/api/assets/file?path=${encodeURIComponent(relToRoot)}`;
}
export function runInit(params: {
account: string;
mode: string;