feat(agent): 支持工具执行结果中的图片和视频资产预览
在聊天消息组件中添加资产灯箱预览功能,支持展示工具调用返回的图片和视频资源。新增 `AssetLightbox` 组件用于全屏浏览资产,并扩展消息类型以包含资产元数据。同时引入 `@radix-ui/react-select` 依赖并为服务端添加资产 URL 转换工具函数。
This commit is contained in:
@@ -12,6 +12,7 @@ import { tools } from './tools/index';
|
||||
import { videoAgent } from './index';
|
||||
import { dbToPiMessages, saveUserMessage, saveAssistantMessage, saveToolResult, type DbMessage } from './pi-persist';
|
||||
import { getDb } from '../db';
|
||||
import { manifestRelToUrl } from './tools/shared';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
@@ -134,13 +135,43 @@ function handleAgentEvent(ws: WebSocket, convId: string, event: AgentEvent, ctx:
|
||||
|
||||
case 'tool_execution_end': {
|
||||
const resultText = event.result?.content?.map((c: any) => c.text || '').join('') || '';
|
||||
const assets = extractAssets(event.toolName, resultText);
|
||||
if (event.isError) {
|
||||
ws.send(JSON.stringify({ type: 'tool_error', data: { tool: event.toolName, error: resultText } }));
|
||||
} else {
|
||||
ws.send(JSON.stringify({ type: 'tool_result', data: { tool: event.toolName, result: resultText.slice(0, 1000) } }));
|
||||
ws.send(JSON.stringify({ type: 'tool_result', data: { tool: event.toolName, result: resultText.slice(0, 1000), assets } }));
|
||||
}
|
||||
saveToolResult(convId, event.toolCallId, event.toolName, resultText, event.isError);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function extractAssets(toolName: string, resultText: string): Array<{ type: 'image' | 'video'; url: string; name: string }> {
|
||||
const assets: Array<{ type: 'image' | 'video'; url: string; name: string }> = [];
|
||||
try {
|
||||
const data = JSON.parse(resultText);
|
||||
if (toolName === 'generate_images' && data.manifestPath && Array.isArray(data.images)) {
|
||||
for (const img of data.images) {
|
||||
if (img.file) {
|
||||
assets.push({ type: 'image', url: manifestRelToUrl(data.manifestPath, img.file), name: img.file.split('/').pop() || '' });
|
||||
}
|
||||
if (Array.isArray(img.candidates)) {
|
||||
for (const c of img.candidates) {
|
||||
assets.push({ type: 'image', url: manifestRelToUrl(data.manifestPath, c), name: c.split('/').pop() || '' });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (toolName === 'generate_videos' && data.manifestPath && Array.isArray(data.videos)) {
|
||||
for (const vid of data.videos) {
|
||||
if (vid.videoUrl) {
|
||||
assets.push({ type: 'video', url: vid.videoUrl, name: vid.video?.split('/').pop() || 'video' });
|
||||
} else if (vid.video) {
|
||||
assets.push({ type: 'video', url: manifestRelToUrl(data.manifestPath, vid.video), name: vid.video.split('/').pop() || '' });
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
return assets;
|
||||
}
|
||||
|
||||
@@ -16,6 +16,15 @@ export function loadJSON(filePath: string): unknown {
|
||||
return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
||||
}
|
||||
|
||||
export function manifestRelToUrl(manifestPath: string, relPath: string): string {
|
||||
if (!relPath) return '';
|
||||
if (relPath.startsWith('http://') || relPath.startsWith('https://')) return relPath;
|
||||
const manifestDir = path.dirname(path.resolve(manifestPath));
|
||||
const absPath = path.resolve(manifestDir, relPath);
|
||||
const relToRoot = path.relative(PROJECT_ROOT, absPath).replace(/\\/g, '/');
|
||||
return `/api/assets/file?path=${encodeURIComponent(relToRoot)}`;
|
||||
}
|
||||
|
||||
export function runInit(params: {
|
||||
account: string;
|
||||
mode: string;
|
||||
|
||||
Reference in New Issue
Block a user