sion/video-create
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(web): add image/video generation and manifest tools to agent

Browse Source 
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
sion123
2026-05-07 04:17:54 +08:00
parent 5104bbc18a
commit 01963aac96
1 changed files with 306 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

313
web/server/agent/tools.ts
View File

@@ -1,12 +1,15 @@
import { spawn, execSync } from 'child_process';
import path from 'path';
import fs from 'fs';
import os from 'os';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..', '..');
const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..');
const PIPELINE_SCRIPT = path.join(PROJECT_ROOT, '.claude', 'skills', 'video-from-script', 'scripts', 'pipeline.js');
const ACCOUNTS_DIR = path.join(PROJECT_ROOT, 'accounts');
const OUTPUT_DIR = path.join(PROJECT_ROOT, 'output');
export interface ToolDefinition {
name: string;
@@ -19,7 +22,55 @@ export interface ToolDefinition {
execute: (params: Record<string, unknown>) => Promise<string>;
}
// ============================================================================
// Helpers
// ============================================================================
function loadJSON(filePath: string): unknown {
return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
}
/** Write items JSON to a temp file and invoke pipeline init, returning the manifest path */
function runInit(params: {
account: string;
mode: string;
items: unknown[];
imageModel?: string;
videoModel?: string;
format?: string;
}): string {
// Write items to a temp file to avoid shell escaping issues with JSON
const tmpFile = path.join(os.tmpdir(), `pipeline-items-${Date.now()}.json`);
fs.writeFileSync(tmpFile, JSON.stringify(params.items), 'utf-8');
try {
const args = [
`"${PIPELINE_SCRIPT}"`, 'init',
`--account "${params.account}"`,
`--mode ${params.mode}`,
`--items-file "${tmpFile}"`,
params.imageModel ? `--image-model ${params.imageModel}` : '',
params.videoModel ? `--video-model ${params.videoModel}` : '',
params.format ? `--format ${params.format}` : '',
].filter(Boolean).join(' ');
const output = execSync(`node ${args}`, { cwd: PROJECT_ROOT, encoding: 'utf-8' });
// Parse manifest path from init output: "Manifest 已创建: <path>"
const match = output.match(/Manifest 已创建: (.+)/);
if (!match) throw new Error(`Failed to parse manifest path from init output: ${output}`);
return match[1].trim();
} finally {
// Clean up temp file (pipeline init also deletes it, but be safe)
try { fs.unlinkSync(tmpFile); } catch { /* ignore */ }
}
}
// ============================================================================
// Tool Registry
// ============================================================================
export const tools: ToolDefinition[] = [
// ---------------------------------------------------------------------------
// Existing tools (kept exactly as before)
// ---------------------------------------------------------------------------
{
name: 'list_accounts',
description: '列出所有可用账号，返回每个账号的名称、描述、生图模型和视频模型',
@@ -29,13 +80,13 @@ export const tools: ToolDefinition[] = [
required: [],
},
execute: async () => {
const accountsDir = path.join(PROJECT_ROOT, 'accounts');
const dirs = fs.readdirSync(accountsDir, { withFileTypes: true })
if (!fs.existsSync(ACCOUNTS_DIR)) return '暂无账号';
const dirs = fs.readdirSync(ACCOUNTS_DIR, { withFileTypes: true })
.filter((d) => d.isDirectory() && !d.name.startsWith('_') && !d.name.startsWith('.'))
.map((d) => {
const configPath = path.join(accountsDir, d.name, 'account.json');
const configPath = path.join(ACCOUNTS_DIR, d.name, 'account.json');
if (fs.existsSync(configPath)) {
const cfg = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
const cfg = loadJSON(configPath) as Record<string, string>;
return `${d.name} - ${cfg.description || '无描述'} (生图:${cfg.imageModel} 视频:${cfg.videoModel} 画幅:${cfg.defaultFormat})`;
}
return d.name;
@@ -130,10 +181,258 @@ export const tools: ToolDefinition[] = [
},
execute: async (params) => {
const { accountId } = params as { accountId: string };
const configPath = path.join(PROJECT_ROOT, 'accounts', accountId, 'account.json');
const configPath = path.join(ACCOUNTS_DIR, accountId, 'account.json');
if (!fs.existsSync(configPath)) return `账号「${accountId}」不存在`;
const cfg = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
const cfg = loadJSON(configPath);
return JSON.stringify(cfg, null, 2);
},
},
// ---------------------------------------------------------------------------
// New tools: image/video generation, manifest management
// ---------------------------------------------------------------------------
{
name: 'generate_images',
description: '文生图：根据文本提示词生成图片。内部创建临时 manifest 并调用 pipeline images 阶段，支持批量生成。生成结果写入 output 目录，返回图片文件路径列表。',
input_schema: {
type: 'object',
properties: {
accountId: { type: 'string', description: '账号ID，用于继承模型、参考图等配置' },
prompt: { type: 'string', description: '图片提示词（imagePrompt），描述要生成的图片内容' },
count: { type: 'number', description: '生成图片数量，默认 1' },
imageModel: { type: 'string', description: '生图模型（可选，默认继承账号配置）: gemini, mj, gpt-image, kling' },
},
required: ['accountId', 'prompt'],
},
execute: async (params) => {
const { accountId, prompt, count, imageModel } = params as Record<string, unknown>;
const num = Math.max(1, Number(count) || 1);
// Build items: each item needs shotDesc + imagePrompt for the images phase
const items = Array.from({ length: num }, (_, i) => ({
id: i + 1,
shotDesc: prompt as string,
script: '',
imagePrompt: prompt as string,
keyword: 'generated',
}));
// Init manifest (items passed as array, helper writes temp file)
const manifestPath = runInit({
account: accountId as string,
mode: 'single',
items,
imageModel: imageModel as string | undefined,
});
// Run images phase
execSync(`node "${PIPELINE_SCRIPT}" run --manifest "${manifestPath}" --phase images`, {
cwd: PROJECT_ROOT, encoding: 'utf-8',
});
// Read back results
const manifest = loadJSON(manifestPath) as { items?: Array<{ id: number; file?: string; candidates?: string[]; status?: string }> };
const results = (manifest.items || []).map((item) => ({
id: item.id,
file: item.file || null,
candidates: item.candidates || [],
status: item.status,
}));
return JSON.stringify({ manifestPath, images: results }, null, 2);
},
},
{
name: 'generate_videos',
description: '图生视频：根据已有图片和提示词生成 AI 视频。内部创建临时 manifest，先上传图片再执行 videos 阶段。返回视频文件路径。',
input_schema: {
type: 'object',
properties: {
accountId: { type: 'string', description: '账号ID，用于继承视频模型等配置' },
imagePath: { type: 'string', description: '图片文件路径（本地绝对路径或相对 PROJECT_ROOT 的路径）' },
prompt: { type: 'string', description: '视频提示词（videoPrompt），描述图片如何动起来' },
videoModel: { type: 'string', description: '视频模型（可选，默认继承账号配置）: veo3-fast, veo3-fast-frames, kling, grok' },
imagePrompt: { type: 'string', description: '图片提示词（可选），用于 manifest 记录' },
},
required: ['accountId', 'imagePath', 'prompt'],
},
execute: async (params) => {
const { accountId, imagePath, prompt, videoModel, imagePrompt } = params as Record<string, string>;
const resolvedImagePath = path.isAbsolute(imagePath)
? imagePath
: path.resolve(PROJECT_ROOT, imagePath);
if (!fs.existsSync(resolvedImagePath)) {
return `错误: 图片文件不存在: ${resolvedImagePath}`;
}
// Determine relative path from output dir
const baseName = path.basename(resolvedImagePath);
const item = {
id: 1,
shotDesc: imagePrompt || prompt,
script: '',
imagePrompt: imagePrompt || prompt,
videoPrompt: prompt,
keyword: 'generated',
file: `images/${baseName}`,
};
// Init manifest (items passed as array, helper writes temp file)
const manifestPath = runInit({
account: accountId,
mode: 'single',
items: [item],
videoModel: videoModel,
});
// Copy image into manifest's images dir
const manifestDir = path.dirname(manifestPath);
const imagesDir = path.join(manifestDir, 'images');
if (!fs.existsSync(imagesDir)) fs.mkdirSync(imagesDir, { recursive: true });
const targetPath = path.join(imagesDir, baseName);
if (resolvedImagePath !== targetPath) {
fs.copyFileSync(resolvedImagePath, targetPath);
}
// Run upload + videos phases
execSync(`node "${PIPELINE_SCRIPT}" run --manifest "${manifestPath}" --phase upload,videos`, {
cwd: PROJECT_ROOT, encoding: 'utf-8',
});
// Read back results
const manifest = loadJSON(manifestPath) as {
items?: Array<{ id: number; video?: string; videoUrl?: string; videoDuration?: number; status?: string }>;
};
const results = (manifest.items || []).map((it) => ({
id: it.id,
video: it.video || null,
videoUrl: it.videoUrl || null,
videoDuration: it.videoDuration || null,
status: it.status,
}));
return JSON.stringify({ manifestPath, videos: results }, null, 2);
},
},
{
name: 'get_account_prompts',
description: '获取指定账号的三个提示词模板内容：分镜提示词、图片提示词、视频提示词。这些模板定义了账号的视频风格和创意方向。',
input_schema: {
type: 'object',
properties: {
accountId: { type: 'string', description: '账号ID' },
},
required: ['accountId'],
},
execute: async (params) => {
const { accountId } = params as { accountId: string };
const accountDir = path.join(ACCOUNTS_DIR, accountId);
if (!fs.existsSync(accountDir)) return `账号「${accountId}」不存在`;
const promptsDir = path.join(accountDir, 'prompts');
const promptFiles = ['分镜.md', '图片提示词.md', '视频提示词.md'];
const result: Record<string, string> = {};
for (const file of promptFiles) {
const filePath = path.join(promptsDir, file);
if (fs.existsSync(filePath)) {
result[file] = fs.readFileSync(filePath, 'utf-8');
} else {
result[file] = `(文件不存在: ${filePath})`;
}
}
return JSON.stringify(result, null, 2);
},
},
{
name: 'list_outputs',
description: '列出 output 目录下所有项目及其 manifest 状态，方便查看历史生成记录和进行中的任务。',
input_schema: {
type: 'object',
properties: {},
required: [],
},
execute: async () => {
if (!fs.existsSync(OUTPUT_DIR)) return 'output 目录不存在';
const entries = fs.readdirSync(OUTPUT_DIR, { withFileTypes: true })
.filter((d) => d.isDirectory())
.map((d) => {
const manifestPath = path.join(OUTPUT_DIR, d.name, 'manifest.json');
if (!fs.existsSync(manifestPath)) return { name: d.name, status: 'no manifest' };
try {
const manifest = loadJSON(manifestPath) as {
account?: string;
imageModel?: string;
videoModel?: string;
mode?: string;
pipeline?: { phases?: Record<string, string> };
items?: Array<{ status?: string }>;
};
const phases = manifest.pipeline?.phases || {};
const itemCount = manifest.items?.length || 0;
const doneCount = manifest.items?.filter((i) => i.status === 'done').length || 0;
return {
name: d.name,
account: manifest.account || '?',
mode: manifest.mode || '?',
items: `${doneCount}/${itemCount} done`,
phases: Object.entries(phases).map(([k, v]) => `${k}:${v}`).join(', ') || 'pending',
};
} catch {
return { name: d.name, status: 'manifest parse error' };
}
});
return JSON.stringify(entries, null, 2);
},
},
{
name: 'get_manifest',
description: '读取指定 manifest.json 的完整内容，返回 JSON 字符串。',
input_schema: {
type: 'object',
properties: {
manifestPath: { type: 'string', description: 'manifest.json 的绝对路径或相对路径' },
},
required: ['manifestPath'],
},
execute: async (params) => {
const { manifestPath } = params as { manifestPath: string };
const resolved = path.isAbsolute(manifestPath)
? manifestPath
: path.resolve(PROJECT_ROOT, manifestPath);
if (!fs.existsSync(resolved)) return `manifest 不存在: ${resolved}`;
const manifest = loadJSON(resolved);
return JSON.stringify(manifest, null, 2);
},
},
{
name: 'create_manifest',
description: '初始化新的 manifest.json，调用 pipeline.js init 创建输出目录和项目骨架。这是视频创作 pipeline 的起点。',
input_schema: {
type: 'object',
properties: {
accountId: { type: 'string', description: '账号ID，决定模型、参考图等配置来源' },
mode: { type: 'string', description: '创作模式: single (单图模式) 或 framePair (首尾帧模式)' },
items: { type: 'string', description: '分镜数据 JSON 数组字符串。每个元素需包含 shotDesc (分镜描述) 和 script (旁白文案)，可选 imagePrompt、videoPrompt、directorRef、keyword、duration' },
imageModel: { type: 'string', description: '生图模型（可选，默认继承账号配置）: gemini, mj, gpt-image, kling' },
videoModel: { type: 'string', description: '视频模型（可选，默认继承账号配置）: veo3-fast, veo3-fast-frames, kling, grok' },
format: { type: 'string', description: '画幅（可选，默认继承账号配置）: 9:16, 16:9, 1:1' },
},
required: ['accountId', 'mode', 'items'],
},
execute: async (params) => {
const { accountId, mode, items, imageModel, videoModel, format } = params as Record<string, string>;
// Validate items JSON
let parsed: unknown;
try { parsed = JSON.parse(items); } catch { return '错误: items 不是合法的 JSON 字符串'; }
if (!Array.isArray(parsed) || (parsed as unknown[]).length === 0) return '错误: items 必须是非空 JSON 数组';
// Write items to temp file to avoid shell escaping issues
const tmpFile = path.join(os.tmpdir(), `pipeline-items-${Date.now()}.json`);
fs.writeFileSync(tmpFile, items, 'utf-8');
try {
const args = [
`"${PIPELINE_SCRIPT}"`, 'init',
`--account "${accountId}"`,
`--mode ${mode}`,
`--items-file "${tmpFile}"`,
imageModel ? `--image-model ${imageModel}` : '',
videoModel ? `--video-model ${videoModel}` : '',
format ? `--format ${format}` : '',
].filter(Boolean).join(' ');
const output = execSync(`node ${args}`, { cwd: PROJECT_ROOT, encoding: 'utf-8' });
const match = output.match(/Manifest 已创建: (.+)/);
const manifestPath = match ? match[1].trim() : '(unable to parse path)';
return `Manifest 已创建: ${manifestPath}\n\n${output}`;
} finally {
try { fs.unlinkSync(tmpFile); } catch { /* ignore */ }
}
},
},
];