From 001dbde9f4b43c525d186d8caef0c458c9a33703 Mon Sep 17 00:00:00 2001 From: sion123 <450702724@qq.com> Date: Thu, 7 May 2026 03:22:15 +0800 Subject: [PATCH] feat(web): integrate Claude LLM streaming with markdown rendering - Add Anthropic SDK with DeepSeek-compatible API config - Streaming tool-use loop in WebSocket chat handler - GitHub-style markdown rendering with markdown-it - Tool status indicators and thinking states in chat UI - Fix Tailwind content path and CSS border utility Co-Authored-By: Claude Opus 4.7 --- web/client/src/components/chat/ChatInput.tsx | 7 +- .../src/components/chat/ChatMessage.tsx | 19 +- web/client/src/components/chat/ChatView.tsx | 19 +- web/client/src/hooks/useChat.ts | 55 +++- web/client/src/index.css | 79 ++++- web/client/src/lib/markdown.ts | 12 + web/client/tailwind.config.ts | 2 +- web/data/meitu-agent.db | Bin 0 -> 4096 bytes web/data/meitu-agent.db-shm | Bin 0 -> 32768 bytes web/data/meitu-agent.db-wal | Bin 0 -> 103032 bytes web/package-lock.json | 164 +++++++++++ web/package.json | 3 + web/server/agent/index.ts | 103 ++++++- web/server/agent/tools.ts | 120 +++++--- web/server/ws/chat.ts | 271 ++++++++++++++++-- 15 files changed, 759 insertions(+), 95 deletions(-) create mode 100644 web/client/src/lib/markdown.ts create mode 100644 web/data/meitu-agent.db create mode 100644 web/data/meitu-agent.db-shm create mode 100644 web/data/meitu-agent.db-wal diff --git a/web/client/src/components/chat/ChatInput.tsx b/web/client/src/components/chat/ChatInput.tsx index 0c64acf..42a3bb0 100644 --- a/web/client/src/components/chat/ChatInput.tsx +++ b/web/client/src/components/chat/ChatInput.tsx @@ -2,7 +2,7 @@ import { useState, useRef } from 'react'; import { Send } from 'lucide-react'; import { Button } from '@/components/ui/button'; -export function ChatInput({ onSend }: { onSend: (content: string) => void }) { +export function ChatInput({ onSend, disabled }: { onSend: (content: string) => void; disabled?: boolean }) { const [input, setInput] = useState(''); const ref = useRef(null); @@ -29,10 +29,11 @@ export function ChatInput({ onSend }: { onSend: (content: string) => void }) { onChange={(e) => setInput(e.target.value)} onKeyDown={handleKeyDown} rows={1} - placeholder="输入指令..." + placeholder={disabled ? '等待回复中...' : '输入指令...'} className="flex-1 bg-transparent text-sm resize-none outline-none placeholder:text-zinc-600" + disabled={disabled} /> - diff --git a/web/client/src/components/chat/ChatMessage.tsx b/web/client/src/components/chat/ChatMessage.tsx index 9230033..95643a2 100644 --- a/web/client/src/components/chat/ChatMessage.tsx +++ b/web/client/src/components/chat/ChatMessage.tsx @@ -1,18 +1,31 @@ import { cn } from '@/lib/utils'; +import { renderMarkdown } from '@/lib/markdown'; import type { Message } from '@/types'; export function ChatMessage({ message }: { message: Message }) { const isUser = message.role === 'user'; + const isEmpty = !message.content; return (
- {message.content} + {isEmpty ? ( + ... + ) : isUser ? ( +

{message.content}

+ ) : ( +
+ )}
); diff --git a/web/client/src/components/chat/ChatView.tsx b/web/client/src/components/chat/ChatView.tsx index e46c010..e5bd01f 100644 --- a/web/client/src/components/chat/ChatView.tsx +++ b/web/client/src/components/chat/ChatView.tsx @@ -3,13 +3,14 @@ import { useAppStore } from '@/store'; import { useChat } from '@/hooks/useChat'; import { ChatMessage } from './ChatMessage'; import { ChatInput } from './ChatInput'; +import { PipelineProgress } from './PipelineProgress'; import { ScrollArea } from '@/components/ui/scroll-area'; import { Button } from '@/components/ui/button'; -import { RefreshCw } from 'lucide-react'; +import { RefreshCw, Loader2 } from 'lucide-react'; export function ChatView() { - const { activeConversationId, conversations, setConversations, selectedAccountId } = useAppStore(); - const { messages, connected, send, createConversation } = useChat(activeConversationId); + const { activeConversationId, setConversations, selectedAccountId } = useAppStore(); + const { messages, connected, thinking, toolStatus, send, createConversation } = useChat(activeConversationId); const [manifestPath, setManifestPath] = useState(null); useEffect(() => { @@ -19,7 +20,6 @@ export function ChatView() { .catch(() => {}); }, [messages]); - // Check for associated manifest in messages useEffect(() => { const toolMsgs = messages.filter((m) => m.role === 'tool'); if (toolMsgs.length > 0) { @@ -80,12 +80,21 @@ export function ChatView() { )}
+ {messages.map((msg) => ( ))} + + {thinking && ( +
+ + {toolStatus || '思考中...'} +
+ )} - + + ); } diff --git a/web/client/src/hooks/useChat.ts b/web/client/src/hooks/useChat.ts index 4b63bc2..1bc3218 100644 --- a/web/client/src/hooks/useChat.ts +++ b/web/client/src/hooks/useChat.ts @@ -2,22 +2,75 @@ import { useState, useEffect, useCallback, useRef } from 'react'; import { chatSocket } from '@/lib/websocket'; import type { Message } from '@/types'; +interface StreamingMessage { + id: string; + role: 'assistant'; + content: string; + created_at: string; + streaming: boolean; +} + export function useChat(conversationId: string | null) { const [messages, setMessages] = useState([]); const [connected, setConnected] = useState(false); + const [thinking, setThinking] = useState(false); + const [toolStatus, setToolStatus] = useState(null); const pendingRef = useRef(false); useEffect(() => { chatSocket.connect(); chatSocket.on('connected', () => setConnected(true)); + chatSocket.on('history', (data) => { setMessages((data.messages as Message[]) || []); }); + chatSocket.on('message', (data) => { setMessages((prev) => [...prev, data as unknown as Message]); }); + // Streaming handlers + chatSocket.on('status', (data) => { + if (data.status === 'thinking') setThinking(true); + }); + + chatSocket.on('message_start', (data) => { + setThinking(false); + // Add placeholder for streaming + setMessages((prev) => [...prev, { + id: data.id as string, + role: 'assistant', + content: '', + created_at: new Date().toISOString(), + conversation_id: '', + } as Message]); + }); + + chatSocket.on('text_delta', (data) => { + setMessages((prev) => prev.map((m) => + m.id === data.id ? { ...m, content: m.content + (data.text as string) } : m + )); + }); + + chatSocket.on('message_end', () => { + setThinking(false); + }); + + chatSocket.on('tool_start', (data) => { + setToolStatus(`正在执行: ${data.tool}...`); + }); + + chatSocket.on('tool_result', () => { + setToolStatus(null); + setThinking(true); + }); + + chatSocket.on('tool_error', (data) => { + setToolStatus(`工具执行失败: ${data.tool}`); + setTimeout(() => setToolStatus(null), 3000); + }); + return () => { chatSocket.disconnect(); }; @@ -42,5 +95,5 @@ export function useChat(conversationId: string | null) { chatSocket.send('create_conversation', { title, accountId }); }, []); - return { messages, connected, send, createConversation }; + return { messages, connected, thinking, toolStatus, send, createConversation }; } diff --git a/web/client/src/index.css b/web/client/src/index.css index 8457857..5f708d9 100644 --- a/web/client/src/index.css +++ b/web/client/src/index.css @@ -4,9 +4,86 @@ @layer base { * { - @apply border-border; + @apply border-zinc-800; } body { @apply bg-zinc-950 text-zinc-50; } } + +/* GitHub-style Markdown */ +@layer components { + .markdown-body { + color-scheme: dark; + } + .markdown-body h1 { font-size: 1.5em; font-weight: 600; margin: 0.67em 0; padding-bottom: 0.3em; border-bottom: 1px solid #30363d; } + .markdown-body h2 { font-size: 1.25em; font-weight: 600; margin: 0.83em 0; padding-bottom: 0.3em; border-bottom: 1px solid #30363d; } + .markdown-body h3 { font-size: 1.1em; font-weight: 600; margin: 1em 0 0.25em; } + .markdown-body h4 { font-size: 1em; font-weight: 600; margin: 1em 0 0.25em; } + .markdown-body p { margin: 0 0 0.75em; } + .markdown-body p:last-child { margin-bottom: 0; } + .markdown-body ul, .markdown-body ol { padding-left: 1.5em; margin: 0 0 0.75em; } + .markdown-body li { margin: 0.15em 0; } + .markdown-body li > p { margin: 0; } + .markdown-body ul { list-style-type: disc; } + .markdown-body ol { list-style-type: decimal; } + .markdown-body ul ul { list-style-type: circle; } + .markdown-body ul ul ul { list-style-type: square; } + .markdown-body blockquote { + margin: 0 0 0.75em; + padding: 0 1em; + color: #8b949e; + border-left: 3px solid #30363d; + } + .markdown-body code { + background: rgba(110,118,129,0.2); + padding: 0.15em 0.4em; + border-radius: 4px; + font-size: 0.875em; + font-family: ui-monospace, SFMono-Regular, 'SF Mono', Menlo, Consolas, monospace; + } + .markdown-body pre { + background: #161b22; + border: 1px solid #30363d; + border-radius: 6px; + padding: 12px; + margin: 0 0 0.75em; + overflow-x: auto; + } + .markdown-body pre code { + background: none; + padding: 0; + font-size: 0.8125em; + line-height: 1.5; + color: #e6edf3; + } + .markdown-body hr { + border: 0; + height: 1px; + background: #30363d; + margin: 1em 0; + } + .markdown-body a { + color: #58a6ff; + text-decoration: none; + } + .markdown-body a:hover { + text-decoration: underline; + } + .markdown-body strong { font-weight: 600; } + .markdown-body table { + border-collapse: collapse; + margin: 0 0 0.75em; + width: 100%; + } + .markdown-body th, .markdown-body td { + border: 1px solid #30363d; + padding: 6px 13px; + text-align: left; + } + .markdown-body th { + font-weight: 600; + background: rgba(110,118,129,0.1); + } + .markdown-body img { max-width: 100%; border-radius: 6px; } +} diff --git a/web/client/src/lib/markdown.ts b/web/client/src/lib/markdown.ts new file mode 100644 index 0000000..0ec4dcd --- /dev/null +++ b/web/client/src/lib/markdown.ts @@ -0,0 +1,12 @@ +import MarkdownIt from 'markdown-it'; + +const md = new MarkdownIt({ + html: false, + breaks: true, + linkify: true, + typographer: true, +}); + +export function renderMarkdown(text: string): string { + return md.render(text); +} diff --git a/web/client/tailwind.config.ts b/web/client/tailwind.config.ts index 05b7dfb..d221cdc 100644 --- a/web/client/tailwind.config.ts +++ b/web/client/tailwind.config.ts @@ -1,7 +1,7 @@ import type { Config } from 'tailwindcss'; export default { - content: ['./client/src/**/*.{ts,tsx}'], + content: ['./src/**/*.{ts,tsx}'], darkMode: 'class', theme: { extend: { diff --git a/web/data/meitu-agent.db b/web/data/meitu-agent.db new file mode 100644 index 0000000000000000000000000000000000000000..db7a7459cf0d014b0dc2333abb5541c58c2d6ed1 GIT binary patch literal 4096 zcmWFz^vNtqRY=P(%1ta$FlG>7U}9o$P*7lCU|@t|AVoG{WYBBV;st3JAlr;ljiVtj n8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*O6ovo*g{}s{ literal 0 HcmV?d00001 diff --git a/web/data/meitu-agent.db-shm b/web/data/meitu-agent.db-shm new file mode 100644 index 0000000000000000000000000000000000000000..861e062ae45c44b2a588c10940889c2635d319e4 GIT binary patch literal 32768 zcmeI*%}E1c5C-6l8l%Q&{E0?;$kiUiA_QBo20O3-EkO{hz!p4u@S1}q=uG^CTs;VZ zd0rUi+puhQo&sj4AG0pg`>F9+^!=#2i{bs%)5~rD^Zoh!=3#a@d3}4F&#$L{ygweb zNIv&#DedR(q#g4A*P_;=^1Kn%Z|BYEgLdAEzTM6{(T7pH?RwPuUh6r>@;z}W1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pkD<#m2Jrjf25i#lxx}$EgQt#$v2>h==?pqN8H3f3hkPxUT zkh`^nKuv+%C?^DJ3giw&Ay88ww}%RWngY2mR|wP;$W75gpr*iHcUp6BQxk}<8cZn= zC@L`S@(Ub7prXKG#i7kiU?FhKfk0V--!F`n1u!@P0t5&UAV7cs0RjXF5FkK+Kw*Jz DC!Hw= literal 0 HcmV?d00001 diff --git a/web/data/meitu-agent.db-wal b/web/data/meitu-agent.db-wal new file mode 100644 index 0000000000000000000000000000000000000000..505a5d84f949fbb390d054b20447fc518718030b GIT binary patch literal 103032 zcmeI*eP|o^9mjFWjwL@k8MI33g&rB>&O^4{N-2XlLtWAC>PE z-6;=b-f-G%1V%?`C@o{O6vo&Ctl0{sfo!8&7q)INw(gts7>?9jyCTO`rl{q7{)OAnFmKHuMO_m?B5LS2_Phe8`e;!^kCU!S{e^69QKFa2Y) znY{B)&Z2Pn@{P~@{$t;J_5CZOhlX@plQL$0R<)%a&ekJSHW_hlaccb!y5i7qE*s`Cxue6|u82~V~TDn`mR(6|AB7gt_2q1s}0tg_0 z00K9yz^RGaXl(1&@aZ}?x<0E}mO8Cj{`H1HVBKF@EvsH0@%D4+QYnHyqU=*flw?8~ z4JP2@db&dzPC60wA;rnaCk93n1A7%Om!3B(!|zgJzmhl*cN+$iQd}-rT3+syWYw~C z%T^6r%vdKZTbmVAwrOTdF?Z(y+L}?3jrZLq2-!6QTI)a<_BZvov00IagfB*srAbdwBPz zFFpIW(^cvSqGfv{`X^X%9YLh&l~Nr+(Gn!?ma;iO3mLRuJIZQ<5RG5TVE?fGokomexb(+|Apgk2g;jwzp3MphGF zzcUhv-MwYG__}A5>hZ3l)DcuXGt?1yBN?pp6KpnyJufF+6=(Y3tDs==6Wp^TK2v%H2P~T_<>IfRmrYFm8 zn@I!^KmY**5I_Kd^(k;ljz(j$9DcaPjkxFZoR-xMEtM}AR$!(+5O@zXRm;6Ej<{Du zpK~*+RS|ag(!5T$>~_KHSG}}})7|^Nf$^a+NzQ3TS~sTUVlckAvKx)}n>uA@M+~u2 zr<^toCoL~$R3{&wE~Vw2t@5^(PN`n!?%w@obVIjh7M6<6+579S*4KCdB-Im`ZJ>^z zqUZe5a~Mynwr1Y3p|7&<s3TaP{{gdC2q1s} z0tg_000IagfB*srR0xQ3{|)|efm?s~%=aIw8Ge~Mf(j*O5kLR|1Q0*~0R#|0009IL zSf2ucI)d39%+ry`HHyuA^8X0}sDqb}P67j`_)w1vpYA4+O*B64QP^%UP)ERc1eIBJ=iPBs z%Ui0gn?}k{v?qthq~!R}kTjy~Q%00zLK&6HSG3}Kxy=n7Um5TMOx{sr7JgW|nLoa+_$FVI~7$7T8#a2mKEfB*srAbfic4bQjh zMp`?rr;n#*HOo?`H7g}bbp4r@V(I^2a<6jODKV{BO%(6?tH#Sqdz~8KPHACCst)=W z2)WSKk*scODYamm?!3SjJuRf@U!bGI>-2$f3T65ipnriy!}W;)Be!pe#1dPV>)OQs zBu@4CQ+4xmVRzKihx+|~T}s_Fh~8ZmW@;+?cv)zw*3--SAE!|K#&7ZS0!Lo_*K3=f zcs5I3VAWqjxB&zZKmY**5I_I{1Q0*~0R+~9Kp-!0?&Rx-<{r-|Rq_I@WjhvirdOO7 zXsI`r%?mh9To6D20R#|0VBHF2TB5O@E#djL(s_KRH*C$eQtH%{SukvGu65CA{NP&R zJifOiUfizo1U`8INsshiJazPkryB@u;fP%d4X&EtnROg z#16{K1uO!I2{*E}kdml%zGBoDzh;Qv_^p0k;75!*Kp-#h#6R!Y_{F<__+FK~KwH@jh~o<@&I`0g-@i&;z-i=y00Iag zfB*s?hQN{5Xl%C}p5IhDd7smBT2?o-)U0ag8O^dw3!KxSR@1fFHo$wuU6Iipmw6-wDov$@X&a1bn>HWwxCtC`yl}Nmrems2pqoOmyaND_%^M?xp2q1s}0tg_000IagfB*sr z+$;hI!*%1g$UD?sYX4o`T5s3nF0HR?r`p@w)xRs<*EN;Vy3?7y^yE&h$958bC%0ux zbK9P6lWKn3TyJ-8Usw08uI@gm=dL|FckkKR@9b8S*fsySz|3^ldG)J*DTspw;kG}A zoIhL;KmY**5I_I{1Q0*~0R#|0V2ugLwc`yR=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.28.6", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", @@ -1220,6 +1253,12 @@ "win32" ] }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "license": "MIT" + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -1345,6 +1384,31 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/markdown-it": { + "version": "14.1.2", + "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-14.1.2.tgz", + "integrity": "sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/linkify-it": "^5", + "@types/mdurl": "^2" + } + }, + "node_modules/@types/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "25.6.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", @@ -1516,6 +1580,12 @@ "dev": true, "license": "MIT" }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -2211,6 +2281,18 @@ "once": "^1.4.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -2408,6 +2490,12 @@ "node": ">= 6" } }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "license": "Unlicense" + }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -2841,6 +2929,19 @@ "node": ">=6" } }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -2874,6 +2975,15 @@ "dev": true, "license": "MIT" }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "license": "MIT", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -2905,6 +3015,23 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc" } }, + "node_modules/markdown-it": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", + "integrity": "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -2914,6 +3041,12 @@ "node": ">= 0.4" } }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", + "license": "MIT" + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -3445,6 +3578,15 @@ "once": "^1.3.1" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/qs": { "version": "6.14.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", @@ -3969,6 +4111,16 @@ "node": ">=0.10.0" } }, + "node_modules/standardwebhooks": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", + "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -4255,6 +4407,12 @@ "tree-kill": "cli.js" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, "node_modules/ts-interface-checker": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", @@ -4328,6 +4486,12 @@ "node": ">=14.17" } }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "license": "MIT" + }, "node_modules/undici-types": { "version": "7.19.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", diff --git a/web/package.json b/web/package.json index aa4c8ed..8e6e5e6 100644 --- a/web/package.json +++ b/web/package.json @@ -10,12 +10,14 @@ "db:init": "tsx server/db/schema.ts" }, "dependencies": { + "@anthropic-ai/sdk": "^0.95.0", "better-sqlite3": "^11.6.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.0", "cors": "^2.8.5", "express": "^4.21.0", "lucide-react": "^0.460.0", + "markdown-it": "^14.1.1", "react": "^18.3.0", "react-dom": "^18.3.0", "tailwind-merge": "^2.6.0", @@ -27,6 +29,7 @@ "@types/better-sqlite3": "^7.6.0", "@types/cors": "^2.8.0", "@types/express": "^5.0.0", + "@types/markdown-it": "^14.1.2", "@types/react": "^18.3.0", "@types/react-dom": "^18.3.0", "@types/ws": "^8.5.0", diff --git a/web/server/agent/index.ts b/web/server/agent/index.ts index d54b3a1..19d1a50 100644 --- a/web/server/agent/index.ts +++ b/web/server/agent/index.ts @@ -1,4 +1,43 @@ +import Anthropic from '@anthropic-ai/sdk'; import { tools, ToolDefinition } from './tools'; +import { getDb } from '../db'; +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..', '..'); + +function getAnthropicClient(): Anthropic { + const configRow = getDb().prepare('SELECT value FROM configs WHERE key = ?').get('api_keys') as { value: string } | undefined; + let apiKey = process.env.ANTHROPIC_API_KEY || ''; + let baseURL: string | undefined; + + if (configRow) { + try { + const cfg = JSON.parse(configRow.value); + if (cfg.ANTHROPIC_AUTH_TOKEN) apiKey = cfg.ANTHROPIC_AUTH_TOKEN; + if (cfg.ANTHROPIC_BASE_URL) baseURL = cfg.ANTHROPIC_BASE_URL; + } catch {} + } + + return new Anthropic({ + apiKey, + baseURL, + }); +} + +function getModel(): string { + const configRow = getDb().prepare('SELECT value FROM configs WHERE key = ?').get('api_keys') as { value: string } | undefined; + if (configRow) { + try { + const cfg = JSON.parse(configRow.value); + if (cfg.ANTHROPIC_MODEL) return cfg.ANTHROPIC_MODEL; + } catch {} + } + return process.env.ANTHROPIC_MODEL || 'claude-sonnet-4-6'; +} export class VideoAgent { private tools: ToolDefinition[]; @@ -7,11 +46,11 @@ export class VideoAgent { this.tools = tools; } - getToolDefinitions() { + getAnthropicTools(): Anthropic.Tool[] { return this.tools.map((t) => ({ name: t.name, description: t.description, - parameters: t.parameters, + input_schema: t.input_schema, })); } @@ -21,18 +60,60 @@ export class VideoAgent { return tool.execute(params); } - getSystemPrompt(accountContext?: string): string { - return `你是美图 Agent，帮助用户进行短视频创作。 + getSystemPrompt(): string { + // Dynamically list accounts + const accountsDir = path.join(PROJECT_ROOT, 'accounts'); + let accountList = '暂无账号'; + if (fs.existsSync(accountsDir)) { + const dirs = fs.readdirSync(accountsDir, { withFileTypes: true }) + .filter((d) => d.isDirectory() && !d.name.startsWith('_') && !d.name.startsWith('.')); + if (dirs.length > 0) { + accountList = dirs.map((d) => { + const configPath = path.join(accountsDir, d.name, 'account.json'); + if (fs.existsSync(configPath)) { + const cfg = JSON.parse(fs.readFileSync(configPath, 'utf-8')); + return `- ${d.name}: ${cfg.description || '无描述'} (生图:${cfg.imageModel}, 视频:${cfg.videoModel}, 画幅:${cfg.defaultFormat})`; + } + return `- ${d.name}`; + }).join('\n'); + } + } -可用账号：${accountContext || '暂无'} + return `你是美图 Agent，一个专业的短视频创作助手。你可以帮助用户完成从创意到成片的完整流程。 -你可以： -1. 帮用户创建新账号 -2. 查看和管理已有账号 -3. 执行视频创作 pipeline（分镜→生图→生视频→TTS→成片） -4. 管理提示词模板 +## 当前可用账号 +${accountList} -用户想创作视频时，一步步引导他们完成流程。`; +## 你的能力 +1. **查看账号** - 使用 list_accounts 列出所有可用账号及其配置 +2. **创建账号** - 使用 create_account 创建新的短视频账号，配置生图/视频模型、画幅等 +3. **查看账号配置** - 使用 get_account_config 获取账号详细配置 +4. **查看 Pipeline 进度** - 使用 pipeline_status 检查创作进度 +5. **执行创作阶段** - 使用 run_pipeline_phase 执行 pipeline 阶段 + +## 视频创作流程 +1. 确认用户意图（A.幻灯片视频 / B.AI视频） +2. 选择/创建账号 +3. 规划分镜脚本 +4. 生成图片（images 阶段） +5. 生成视频片段（videos 阶段，仅 B 模式） +6. 配音（tts 阶段） +7. 成片组装（assemble 阶段） + +## 行为准则 +- 用中文回复，友好、专业 +- 在用户不清楚时主动询问：成片类型、账号选择、素材来源、画幅等 +- 执行 pipeline 前确认 manifest 路径 +- 如果用户只是闲聊，就闲聊。如果用户想做视频，引导完成流程 +- 不要编造账号或文件路径，使用工具获取真实信息`; + } + + getClient(): Anthropic { + return getAnthropicClient(); + } + + getModel(): string { + return getModel(); } } diff --git a/web/server/agent/tools.ts b/web/server/agent/tools.ts index efd6de3..0bcb773 100644 --- a/web/server/agent/tools.ts +++ b/web/server/agent/tools.ts @@ -1,22 +1,33 @@ import { spawn, execSync } from 'child_process'; import path from 'path'; import fs from 'fs'; +import { fileURLToPath } from 'url'; +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..', '..'); const PIPELINE_SCRIPT = path.join(PROJECT_ROOT, '.claude', 'skills', 'video-from-script', 'scripts', 'pipeline.js'); export interface ToolDefinition { name: string; description: string; - parameters: Record; + input_schema: { + type: 'object'; + properties: Record; + required?: string[]; + }; execute: (params: Record) => Promise; } export const tools: ToolDefinition[] = [ { name: 'list_accounts', - description: '列出所有可用账号', - parameters: { type: 'object', properties: {}, required: [] }, + description: '列出所有可用账号，返回每个账号的名称、描述、生图模型和视频模型', + input_schema: { + type: 'object', + properties: {}, + required: [], + }, execute: async () => { const accountsDir = path.join(PROJECT_ROOT, 'accounts'); const dirs = fs.readdirSync(accountsDir, { withFileTypes: true }) @@ -25,21 +36,69 @@ export const tools: ToolDefinition[] = [ const configPath = path.join(accountsDir, d.name, 'account.json'); if (fs.existsSync(configPath)) { const cfg = JSON.parse(fs.readFileSync(configPath, 'utf-8')); - return `${d.name} - ${cfg.description || '无描述'} (${cfg.imageModel}/${cfg.videoModel})`; + return `${d.name} - ${cfg.description || '无描述'} (生图:${cfg.imageModel} 视频:${cfg.videoModel} 画幅:${cfg.defaultFormat})`; } return d.name; }); - return dirs.join('\n'); + return dirs.join('\n') || '暂无账号'; + }, + }, + { + name: 'create_account', + description: '创建新的短视频账号。需要提供账号ID、名称和描述。创建后可在 accounts/ 目录下找到配置。', + input_schema: { + type: 'object', + properties: { + id: { type: 'string', description: '账号唯一标识，英文小写，如 military-account' }, + name: { type: 'string', description: '账号显示名称，中文，如 军事账号' }, + desc: { type: 'string', description: '账号描述，说明视频风格和主题' }, + imageModel: { type: 'string', description: '生图模型: gemini, mj, gpt, kling' }, + videoModel: { type: 'string', description: '视频模型: veo3-fast, veo3-fast-frames, kling, grok' }, + format: { type: 'string', description: '画幅: 9:16 (竖屏), 16:9 (横屏), 1:1 (方形)' }, + }, + required: ['id', 'name'], + }, + execute: async (params) => { + const { id, name, desc, imageModel, videoModel, format } = params as Record; + const cmd = [ + `node "${PIPELINE_SCRIPT}" create-account`, + `--id "${id}"`, + `--name "${name}"`, + `--desc "${desc || ''}"`, + `--video-model ${videoModel || 'veo3-fast'}`, + imageModel ? `--image-model ${imageModel}` : '', + format ? `--format ${format}` : '', + ].filter(Boolean).join(' '); + const result = execSync(cmd, { cwd: PROJECT_ROOT, encoding: 'utf-8' }); + return `账号「${name}」创建成功。\n${result}`; + }, + }, + { + name: 'pipeline_status', + description: '查看指定 manifest 的 pipeline 执行进度和各阶段状态', + input_schema: { + type: 'object', + properties: { + manifest: { type: 'string', description: 'manifest.json 的绝对路径' }, + }, + required: ['manifest'], + }, + execute: async (params) => { + const { manifest } = params as { manifest: string }; + const result = execSync(`node "${PIPELINE_SCRIPT}" status --manifest "${manifest}"`, { + cwd: PROJECT_ROOT, encoding: 'utf-8', + }); + return result; }, }, { name: 'run_pipeline_phase', - description: '执行 pipeline 阶段 (images/upload/videos/tts/assemble)', - parameters: { + description: '执行视频创作 pipeline 的指定阶段。阶段顺序: images(生图) → upload(上传) → videos(生视频) → tts(配音) → assemble(成片组装)。执行前需确认 manifest.json 已存在。', + input_schema: { type: 'object', properties: { - manifest: { type: 'string', description: 'manifest.json 绝对路径' }, - phase: { type: 'string', description: '阶段名: images, upload, videos, tts, assemble' }, + manifest: { type: 'string', description: 'manifest.json 的绝对路径' }, + phase: { type: 'string', description: '要执行的阶段: images, upload, videos, tts, assemble。多个阶段用逗号分隔如 images,upload' }, }, required: ['manifest', 'phase'], }, @@ -54,48 +113,27 @@ export const tools: ToolDefinition[] = [ proc.stdout.on('data', (d: Buffer) => { output += d.toString(); }); proc.stderr.on('data', (d: Buffer) => { output += d.toString(); }); proc.on('close', (code) => { - code === 0 ? resolve(output) : reject(new Error(`Pipeline exit code ${code}: ${output}`)); + code === 0 ? resolve(output || '执行成功') : reject(new Error(`Pipeline exit code ${code}: ${output.slice(-500)}`)); }); }); }, }, { - name: 'pipeline_status', - description: '查看 pipeline 进度', - parameters: { + name: 'get_account_config', + description: '获取指定账号的完整配置，包括模型选择、TTS语音、字幕风格等', + input_schema: { type: 'object', properties: { - manifest: { type: 'string', description: 'manifest.json 绝对路径' }, + accountId: { type: 'string', description: '账号ID，如 军事账号' }, }, - required: ['manifest'], + required: ['accountId'], }, execute: async (params) => { - const { manifest } = params as { manifest: string }; - const result = execSync(`node "${PIPELINE_SCRIPT}" status --manifest "${manifest}"`, { - cwd: PROJECT_ROOT, encoding: 'utf-8', - }); - return result; - }, - }, - { - name: 'create_account', - description: '创建新账号', - parameters: { - type: 'object', - properties: { - id: { type: 'string', description: '账号 ID' }, - name: { type: 'string', description: '账号名称' }, - desc: { type: 'string', description: '账号描述' }, - }, - required: ['id', 'name'], - }, - execute: async (params) => { - const { id, name, desc } = params as { id: string; name: string; desc?: string }; - const result = execSync( - `node "${PIPELINE_SCRIPT}" create-account --id "${id}" --name "${name}" --desc "${desc || ''}" --video-model veo3-fast`, - { cwd: PROJECT_ROOT, encoding: 'utf-8' } - ); - return result; + const { accountId } = params as { accountId: string }; + const configPath = path.join(PROJECT_ROOT, 'accounts', accountId, 'account.json'); + if (!fs.existsSync(configPath)) return `账号「${accountId}」不存在`; + const cfg = JSON.parse(fs.readFileSync(configPath, 'utf-8')); + return JSON.stringify(cfg, null, 2); }, }, ]; diff --git a/web/server/ws/chat.ts b/web/server/ws/chat.ts index 345cfd4..679858e 100644 --- a/web/server/ws/chat.ts +++ b/web/server/ws/chat.ts @@ -1,47 +1,79 @@ import { WebSocket } from 'ws'; import { randomUUID } from 'crypto'; import { getDb } from '../db'; +import { videoAgent } from '../agent'; +import type { MessageParam, ToolUseBlock, TextBlock } from '@anthropic-ai/sdk/resources/messages.mjs'; + +interface ChatMsg { + type: string; + conversationId?: string; + content?: string; + title?: string; + accountId?: string; + data?: Record; + conversation_id?: string; + role?: string; + tool_calls?: string; + created_at?: string; + id?: string; +} + +interface DbMessage { + id: string; + conversation_id: string; + role: string; + content: string; + tool_calls: string | null; + created_at: string; +} + +function dbToAnthropic(msg: DbMessage): MessageParam { + if (msg.role === 'user') { + return { role: 'user', content: msg.content }; + } + if (msg.role === 'assistant') { + if (msg.tool_calls) { + try { + const parsed = JSON.parse(msg.tool_calls); + return { role: 'assistant', content: parsed }; + } catch { + return { role: 'assistant', content: msg.content }; + } + } + return { role: 'assistant', content: msg.content }; + } + if (msg.role === 'tool') { + try { + const { tool_use_id, content } = JSON.parse(msg.content); + return { + role: 'user', + content: [{ type: 'tool_result', tool_use_id, content }], + }; + } catch { + return { role: 'user', content: msg.content }; + } + } + return { role: 'user', content: msg.content }; +} export function handleChat(ws: WebSocket) { let conversationId: string | null = null; ws.on('message', async (raw) => { try { - const msg = JSON.parse(raw.toString()); + const msg: ChatMsg = JSON.parse(raw.toString()); + // --- Init: load conversation history --- if (msg.type === 'init') { conversationId = msg.conversationId || randomUUID(); const history = getDb().prepare( 'SELECT * FROM messages WHERE conversation_id = ? ORDER BY created_at' - ).all(conversationId); + ).all(conversationId) as DbMessage[]; ws.send(JSON.stringify({ type: 'history', data: { conversationId, messages: history } })); return; } - if (msg.type === 'chat') { - const { content } = msg; - const msgId = randomUUID(); - - getDb().prepare( - 'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)' - ).run(msgId, conversationId, 'user', content); - - ws.send(JSON.stringify({ type: 'message', data: { id: msgId, role: 'user', content } })); - - // Assistant echo (placeholder until LLM integration in Task 3.3) - const assistantId = randomUUID(); - const assistantContent = `收到你的消息：「${content}」。Agent 引擎正在启动中...`; - - getDb().prepare( - 'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)' - ).run(assistantId, conversationId, 'assistant', assistantContent); - - ws.send(JSON.stringify({ - type: 'message', - data: { id: assistantId, role: 'assistant', content: assistantContent }, - })); - } - + // --- Create conversation --- if (msg.type === 'create_conversation') { const { title, accountId } = msg; conversationId = randomUUID(); @@ -49,13 +81,194 @@ export function handleChat(ws: WebSocket) { 'INSERT INTO conversations (id, title, account_id) VALUES (?, ?, ?)' ).run(conversationId, title || '新对话', accountId || null); ws.send(JSON.stringify({ type: 'conversation_created', data: { id: conversationId, title } })); + return; + } + + // --- Chat with LLM --- + if (msg.type === 'chat') { + await handleChatMessage(ws, conversationId!, msg.content!); } } catch (e) { + console.error('WebSocket error:', e); ws.send(JSON.stringify({ type: 'error', data: { message: (e as Error).message } })); } }); - ws.on('close', () => { - // cleanup if needed - }); + ws.on('close', () => {}); +} + +async function handleChatMessage(ws: WebSocket, convId: string, content: string) { + // 1. Save user message + const userMsgId = randomUUID(); + getDb().prepare( + 'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)' + ).run(userMsgId, convId, 'user', content); + ws.send(JSON.stringify({ type: 'message', data: { id: userMsgId, role: 'user', content } })); + + // Update conversation title if first message + const msgCount = getDb().prepare( + 'SELECT COUNT(*) as count FROM messages WHERE conversation_id = ?' + ).get(convId) as { count: number }; + if (msgCount.count <= 1) { + const title = content.slice(0, 30) + (content.length > 30 ? '...' : ''); + getDb().prepare('UPDATE conversations SET title = ?, updated_at = datetime(\'now\') WHERE id = ?') + .run(title, convId); + } + + // Update conversation timestamp + getDb().prepare('UPDATE conversations SET updated_at = datetime(\'now\') WHERE id = ?').run(convId); + + // 2. Build message history for Anthropic + const history = getDb().prepare( + 'SELECT * FROM messages WHERE conversation_id = ? AND id != ? ORDER BY created_at' + ).all(convId, userMsgId) as DbMessage[]; + + const messages: MessageParam[] = history.map(dbToAnthropic); + + // 3. Call LLM with tool loop + const client = videoAgent.getClient(); + const model = videoAgent.getModel(); + const systemPrompt = videoAgent.getSystemPrompt(); + + ws.send(JSON.stringify({ type: 'status', data: { status: 'thinking' } })); + + try { + let currentMessages = messages; + let maxLoops = 10; + + while (maxLoops-- > 0) { + const stream = client.messages.stream({ + model, + max_tokens: 4096, + system: systemPrompt, + tools: videoAgent.getAnthropicTools(), + messages: currentMessages, + }); + + let assistantContent = ''; + let toolUseBlocks: { id: string; name: string; input: Record }[] = []; + const assistantMsgId = randomUUID(); + + // Stream text + ws.send(JSON.stringify({ type: 'message_start', data: { id: assistantMsgId } })); + + for await (const event of stream) { + if (event.type === 'content_block_delta') { + if (event.delta.type === 'text_delta') { + assistantContent += event.delta.text; + ws.send(JSON.stringify({ + type: 'text_delta', + data: { id: assistantMsgId, text: event.delta.text }, + })); + } + if (event.delta.type === 'input_json_delta') { + // Accumulating tool input — handled by SDK internally + } + } + if (event.type === 'content_block_start') { + if (event.content_block.type === 'tool_use') { + toolUseBlocks.push({ + id: event.content_block.id, + name: event.content_block.name, + input: (event.content_block.input || {}) as Record, + }); + } + } + } + + const finalMsg = await stream.finalMessage(); + ws.send(JSON.stringify({ type: 'message_end', data: { id: assistantMsgId } })); + + // Extract tool uses from final message + const toolUses: { id: string; name: string; input: Record }[] = []; + const textBlocks: string[] = []; + + for (const block of finalMsg.content) { + if (block.type === 'text') { + textBlocks.push(block.text); + } + if (block.type === 'tool_use') { + toolUses.push({ id: block.id, name: block.name, input: block.input as Record }); + } + } + + // No tool calls — save assistant message and done + if (toolUses.length === 0) { + const finalText = textBlocks.join(''); + getDb().prepare( + 'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)' + ).run(assistantMsgId, convId, 'assistant', finalText); + return; + } + + // Has tool calls — save assistant message with tool_calls, execute tools, add results + getDb().prepare( + 'INSERT INTO messages (id, conversation_id, role, content, tool_calls) VALUES (?, ?, ?, ?, ?)' + ).run(assistantMsgId, convId, 'assistant', textBlocks.join('') || '(调用工具)', JSON.stringify(finalMsg.content)); + + // Build assistant content blocks for Anthropic + const assistantBlocks: (TextBlock | ToolUseBlock)[] = finalMsg.content + .filter((b): b is TextBlock | ToolUseBlock => b.type === 'text' || b.type === 'tool_use'); + + currentMessages.push({ role: 'assistant', content: assistantBlocks }); + + // Execute tools and send results + const toolResults: { type: 'tool_result'; tool_use_id: string; content: string }[] = []; + + for (const tool of toolUses) { + ws.send(JSON.stringify({ + type: 'tool_start', + data: { tool: tool.name, input: tool.input }, + })); + + try { + const result = await videoAgent.executeTool(tool.name, tool.input); + toolResults.push({ type: 'tool_result', tool_use_id: tool.id, content: result }); + + // Save tool result to DB + const toolMsgId = randomUUID(); + getDb().prepare( + 'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)' + ).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tool.id, content: result })); + + ws.send(JSON.stringify({ + type: 'tool_result', + data: { tool: tool.name, result: result.slice(0, 1000) }, + })); + } catch (err) { + const errMsg = (err as Error).message; + toolResults.push({ type: 'tool_result', tool_use_id: tool.id, content: `Error: ${errMsg}` }); + + const toolMsgId = randomUUID(); + getDb().prepare( + 'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)' + ).run(toolMsgId, convId, 'tool', JSON.stringify({ tool_use_id: tool.id, content: `Error: ${errMsg}` })); + + ws.send(JSON.stringify({ + type: 'tool_error', + data: { tool: tool.name, error: errMsg }, + })); + } + } + + // Add tool results to conversation + currentMessages.push({ + role: 'user', + content: toolResults, + }); + + // Continue loop — LLM will process tool results and possibly call more tools or give final response + } + } catch (err) { + const errMsg = (err as Error).message; + console.error('LLM error:', errMsg); + const errId = randomUUID(); + getDb().prepare( + 'INSERT INTO messages (id, conversation_id, role, content) VALUES (?, ?, ?, ?)' + ).run(errId, convId, 'assistant', `抱歉，出错了：${errMsg}`); + ws.send(JSON.stringify({ + type: 'message', + data: { id: errId, role: 'assistant', content: `抱歉，出错了：${errMsg}` }, + })); + } }