From b6e0c475d33936d63c6ed21bb7ba35c2ced0488c Mon Sep 17 00:00:00 2001 From: jinghui Date: Thu, 19 Nov 2026 08:30:20 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20AI=E7=94=9F=E5=9B=BE=E6=8F=90=E7=A4=BA?= =?UTF-8?q?=E8=AF=8D=E7=BF=BB=E8=AF=91=E6=88=90=E8=8B=B1=E6=96=87=E5=90=8E?= =?UTF-8?q?=E5=86=8D=E5=8F=91=E5=BE=80=E4=B8=8A=E6=B5=81=EF=BC=88img2img?= =?UTF-8?q?=20=E5=85=A8=E9=93=BE=E8=B7=AF=20+=20=E8=A7=86=E9=A2=91/?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E6=94=B9=E5=8A=A8=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/src/routes/aigen.js | 156 ++++++- backend/src/routes/settings.js | 18 + backend/src/routes/video.js | 210 +++++++++- backend/utils/settings.js | 65 ++- backend/utils/videotool.js | 10 +- frontend/src/components/AIGenModal.jsx | 213 +++++++++- .../src/components/AIVideoStudioModal.jsx | 393 ++++++++++++++++++ frontend/src/components/Header.jsx | 13 +- frontend/src/components/SettingsModal.jsx | 125 +++++- 9 files changed, 1169 insertions(+), 34 deletions(-) create mode 100644 frontend/src/components/AIVideoStudioModal.jsx diff --git a/backend/src/routes/aigen.js b/backend/src/routes/aigen.js index 7af5ffc..6075284 100644 --- a/backend/src/routes/aigen.js +++ b/backend/src/routes/aigen.js @@ -7,7 +7,45 @@ */ const express = require('express'); +/** 文生图提示词翻译:含中文/非 ASCII 时先翻译成英文,失败则回落原文 */ +const TEXT_TRANSLATE_MODELS = (process.env.AIGEN_TRANSLATE_MODEL || 'glm-5.3-flash,deepseek-v4.1-flash,mimo-v2.6-flash').split(',').map(s=>s.trim()).filter(Boolean); +async function translatePromptToEnglish(prompt, url, key) { + const p = String(prompt || '').trim(); + if (!p || !/[\x80-\uFFFF]/.test(p)) return p; // 已是纯英文/ASCII,无需翻译 + for (const model of TEXT_TRANSLATE_MODELS) { + try { + const ctrl = new AbortController(); + const t = setTimeout(() => ctrl.abort(), 30000); + const resp = await fetch(`${url}/chat/completions`, { + method: 'POST', + headers: { 'Authorization': `Bearer ${key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + messages: [ + { role: 'system', content: 'You are a translator. Translate the user prompt into concise, vivid English suitable for an image generation model. Output ONLY the English prompt, no quotes, no explanation.' }, + { role: 'user', content: p } + ], + temperature: 0.2, + max_tokens: 300 + }), + signal: ctrl.signal + }); + clearTimeout(t); + if (!resp.ok) continue; + const d = await resp.json(); + const out = (d && d.choices && d.choices[0] && d.choices[0].message && d.choices[0].message.content) || ''; + const clean = String(out).trim().replace(/^["'`]+|["'`]+$/g, ''); + if (clean) return clean; + } catch (e) { + continue; + } + } + return p; // 全部失败回落原文 +} + const router = express.Router(); +const fs = require('fs'); +const path = require('path'); const { asyncHandler } = require('../middleware/errorHandler'); const logger = require('../utils/logger'); const { getSettings } = require('../../utils/settings'); @@ -38,6 +76,61 @@ const BUILTIN_GROUPS = [ const ALLOWED_SIZES = ['1024x1024', '1280x720', '720x1280', '1536x1024', '1024x1536']; +// 以图生图(图片编辑)支持的模型:FLUX.2 系列走 /v1/images/edits 的 multipart 上传 +const EDIT_MODELS = [ + { id: 'flux-2', name: 'FLUX.2 [dev](图生图)' }, + { id: 'flux-2-klein', name: 'FLUX.2 Klein 9B(图生图·快)' }, + { id: 'flux-2-klein-4b', name: 'FLUX.2 Klein 4B(图生图·最省)' } +]; + +const REF_EXTS = new Set(['.jpg', '.jpeg', '.png', '.webp', '.bmp', '.avif', '.gif']); +const MAX_REFS = 4; +const MIME_BY_EXT = { + '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png', + '.webp': 'image/webp', '.bmp': 'image/bmp', '.avif': 'image/avif', '.gif': 'image/gif' +}; + +/** 归一化基础地址 -> 完整 images/edits URL */ +function normalizeEditUrl(url) { + let u = String(url || '').trim().replace(/\/+$/, ''); + if (/\/images\/edits$/.test(u)) return u; + if (/\/images\/generations$/.test(u)) return u.replace(/\/images\/generations$/, '/images/edits'); + if (/\/v1$/.test(u)) return u + '/images/edits'; + if (/\/chat\/completions$/.test(u)) return u.replace(/\/chat\/completions$/, '/images/edits'); + return u + '/v1/images/edits'; +} + +/** + * 从素材库读取参考图(以图生图用) + * refs 支持 ['a.jpg'] 或 [{ path|filename }] + */ +function loadRefImages(req, libraryId, refs) { + const list = Array.isArray(refs) ? refs.filter(Boolean).slice(0, MAX_REFS) : []; + if (list.length === 0) return []; + const configManager = req.app.get('configManager'); + if (!configManager) throw new Error('服务未初始化'); + const config = configManager.load(); + const library = (config.libraries || []).find((l) => String(l.id) === String(libraryId)); + if (!library) throw new Error('素材库不存在,请重新选择参考图'); + + const out = []; + for (const item of list) { + const rel = typeof item === 'string' ? item : (item.path || item.filename); + if (!rel) continue; + const abs = path.resolve(path.join(library.path, rel)); + if (!abs.startsWith(path.resolve(library.path))) throw new Error('参考图路径非法'); + if (!fs.existsSync(abs)) throw new Error(`参考图不存在: ${rel}`); + const ext = path.extname(abs).toLowerCase(); + if (!REF_EXTS.has(ext)) throw new Error(`不是图片文件: ${rel}(只能选图片做参考图)`); + out.push({ + name: path.basename(abs), + mime: MIME_BY_EXT[ext] || 'application/octet-stream', + buf: fs.readFileSync(abs) + }); + } + return out; +} + /** 归一化基础地址 -> 完整 images/generations URL */ function normalizeGenUrl(url) { let u = String(url || '').trim().replace(/\/+$/, ''); @@ -87,6 +180,7 @@ router.get('/models', asyncHandler(async (req, res) => { success: true, data: { groups: BUILTIN_GROUPS, + editModels: EDIT_MODELS, sizes: ALLOWED_SIZES, provider: { type: cur.provider, @@ -109,7 +203,13 @@ router.get('/models', asyncHandler(async (req, res) => { */ router.post('/generate', asyncHandler(async (req, res) => { const t0 = Date.now(); - const { prompt, model, size, provider } = req.body || {}; + const { prompt, model, size, provider, libraryId } = req.body || {}; + // 文生图:中文提示词先翻译成英文再发给上游(图生图同样翻译) + const originalPrompt = String(prompt || '').trim(); + const effPrompt = await translatePromptToEnglish(originalPrompt, effUrl, effKey); + + // 参考图(以图生图):兼容 refImages / refs / refPaths 三种字段 + const rawRefs = (req.body && (req.body.refImages || req.body.refs || req.body.refPaths)) || []; if (!prompt || !String(prompt).trim()) { return res.status(400).json({ success: false, error: '提示词不能为空' }); @@ -140,26 +240,50 @@ router.post('/generate', asyncHandler(async (req, res) => { } const effSize = ALLOWED_SIZES.includes(size) ? size : '1024x1024'; - const genUrl = normalizeGenUrl(effUrl); - logger.info(`[aigen] generate model=${effModel} size=${effSize} provider=${useCustom ? 'custom' : 'builtin'} prompt=${String(prompt).slice(0, 60)}`); + // 读取参考图(有参考图 = 以图生图,走图片编辑接口) + let refs = []; + if (rawRefs && (Array.isArray(rawRefs) ? rawRefs.length : true)) { + try { + refs = loadRefImages(req, libraryId, rawRefs); + } catch (e) { + return res.status(400).json({ success: false, error: e.message }); + } + } + const isEdit = refs.length > 0; + const targetUrl = isEdit ? normalizeEditUrl(effUrl) : normalizeGenUrl(effUrl); + + logger.info(`[aigen] ${isEdit ? 'edit' : 'generate'} model=${effModel} size=${effSize} refs=${refs.length} provider=${useCustom ? 'custom' : 'builtin'} prompt=${String(effPrompt).slice(0, 60)} (orig: ${String(originalPrompt).slice(0, 40)})`); const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), 150000); // 生图最长 150s + const timer = setTimeout(() => ctrl.abort(), isEdit ? 300000 : 150000); // 图生图更慢,最长 300s let result; try { - const resp = await fetch(genUrl, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${effKey}`, - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ + let body; + let headers = { 'Authorization': `Bearer ${effKey}` }; + if (isEdit) { + const form = new FormData(); + form.append('model', String(effModel || 'flux-2')); + form.append('prompt', String(effPrompt)); + form.append('n', '1'); + form.append('size', effSize); + refs.forEach((r, i) => { + form.append(i === 0 ? 'image' : `image_${i}`, new Blob([r.buf], { type: r.mime }), r.name); + }); + body = form; + } else { + headers['Content-Type'] = 'application/json'; + body = JSON.stringify({ model: effModel, - prompt: String(prompt), + prompt: String(effPrompt), n: 1, size: effSize - }), + }); + } + const resp = await fetch(targetUrl, { + method: 'POST', + headers, + body, signal: ctrl.signal }); const text = await resp.text(); @@ -171,6 +295,9 @@ router.post('/generate', asyncHandler(async (req, res) => { if (j.error && j.error.message) msg = j.error.message; else if (j.message) msg = j.message; } catch (e) { /* 保留默认信息 */ } + if (isEdit && /not found|404|not support|unsupported|invalid model/i.test(msg + ' ' + resp.status)) { + msg += '(当前模型可能不支持以图生图,请在设置里换成 flux-2 系列或支持图片编辑的模型)'; + } return res.status(502).json({ success: false, error: msg }); } result = JSON.parse(text); @@ -227,6 +354,9 @@ router.post('/generate', asyncHandler(async (req, res) => { revisedPrompt: result.data && result.data[0] && result.data[0].revised_prompt, model: effModel, size: effSize, + promptEn: effPrompt, + edited: isEdit, + refCount: refs.length, ms: Date.now() - t0 } }); diff --git a/backend/src/routes/settings.js b/backend/src/routes/settings.js index dbd1530..cb59e8d 100644 --- a/backend/src/routes/settings.js +++ b/backend/src/routes/settings.js @@ -40,6 +40,15 @@ router.get('/', (req, res) => { apiKeyMasked: maskKey((ig.custom && ig.custom.apiKey) || ''), configured: !!(ig.custom && ig.custom.url && ig.custom.apiKey) } + }, + videoai: { + provider: (s.videoai && s.videoai.provider) === 'custom' ? 'custom' : 'inherit', + custom: { + url: (s.videoai && s.videoai.custom && s.videoai.custom.url) || '', + model: (s.videoai && s.videoai.custom && s.videoai.custom.model) || '', + apiKeyMasked: maskKey((s.videoai && s.videoai.custom && s.videoai.custom.apiKey) || ''), + configured: !!(s.videoai && s.videoai.custom && s.videoai.custom.url && s.videoai.custom.apiKey) + } } } }); @@ -73,6 +82,15 @@ router.put('/', (req, res) => { apiKeyMasked: maskKey(ig.custom.apiKey), configured: !!(ig.custom.url && ig.custom.apiKey) } + }, + videoai: { + provider: (saved.videoai && saved.videoai.provider) === 'custom' ? 'custom' : 'inherit', + custom: { + url: (saved.videoai && saved.videoai.custom && saved.videoai.custom.url) || '', + model: (saved.videoai && saved.videoai.custom && saved.videoai.custom.model) || '', + apiKeyMasked: maskKey((saved.videoai && saved.videoai.custom && saved.videoai.custom.apiKey) || ''), + configured: !!(saved.videoai && saved.videoai.custom && saved.videoai.custom.url && saved.videoai.custom.apiKey) + } } } }); diff --git a/backend/src/routes/video.js b/backend/src/routes/video.js index c9ede3c..2551101 100644 --- a/backend/src/routes/video.js +++ b/backend/src/routes/video.js @@ -14,9 +14,11 @@ const fs = require('fs'); const os = require('os'); const { asyncHandler, ValidationError, NotFoundError } = require('../middleware/errorHandler'); const { processImage } = require('../../utils/scanner'); -const { getSettings } = require('../../utils/settings'); +const { getSettings, resolveVideoGen } = require('../../utils/settings'); const logger = require('../utils/logger'); const videotool = require('../../utils/videotool'); +const multer = require('multer'); +const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 20 * 1024 * 1024 } }); const VIDEO_EXTS = new Set(['.mp4', '.mov', '.m4v', '.webm', '.mkv', '.avi', '.flv', '.wmv', '.mpg', '.mpeg', '.3gp', '.ts']); @@ -133,7 +135,8 @@ router.post( model, // 模型名 voice = 'zh-CN-XiaoxiaoNeural', resolution = '1920x1080', - extraPrompt = '' + extraPrompt = '', + subtitleStyle = {} } = req.body || {}; if (!libraryId || !Array.isArray(images) || images.length < 2) { @@ -156,14 +159,13 @@ router.post( if (!library) throw new NotFoundError('Library', libraryId); const settings = getSettings(); - const gen = (settings.imagegen && settings.imagegen.provider === 'custom') - ? settings.imagegen.custom - : settings.imagegen.newapi; + // 设置 - AI 视频(或复用 AI 生图)中的文本模型配置;请求参数可临时覆盖 + const gen = resolveVideoGen(settings); const url = (baseUrl || gen.url || '').replace(/\/+$/, ''); const key = apiKey || gen.apiKey || process.env.NEWAPI_KEY || ''; const mdl = model || gen.model || 'mimo-v2.6-flash'; - if (!url) throw new ValidationError('未配置模型接口地址(填 BASE URL 或在设置里配置)'); - if (!key) throw new ValidationError('请填写模型 KEY(或在设置-生图里配置)'); + if (!url) throw new ValidationError('未配置模型接口地址(请到右上角设置 → AI 视频里填写)'); + if (!key) throw new ValidationError('请先在右上角设置 → AI 视频里填写模型 KEY'); const id = `vt${++aiTaskSeq}_${Date.now()}`; const [w, h] = String(resolution).split('x').map((n) => parseInt(n, 10)); @@ -185,7 +187,7 @@ router.post( runAiVideo({ task, library, images, title: String(title).trim(), url, key, model: mdl, voice, width: Number.isFinite(w) ? w : 1920, height: Number.isFinite(h) ? h : 1080, - extraPrompt: String(extraPrompt || ''), + extraPrompt: String(extraPrompt || ''), subtitleStyle: subtitleStyle || {}, req }).catch((err) => { task.status = 'error'; @@ -200,7 +202,7 @@ router.post( router.get( '/task/:id', asyncHandler(async (req, res) => { - const task = aiTasks.get(req.params.id); + const task = aiTasks.get(req.params.id) || textTasks.get(req.params.id); if (!task) throw new NotFoundError('Task', req.params.id); res.json(task); }) @@ -255,7 +257,7 @@ async function llmScript({ url, key, model, title, count, extraPrompt }) { } async function runAiVideo(ctx) { - const { task, library, images, title, url, key, model, voice, width, height, extraPrompt, req } = ctx; + const { task, library, images, title, url, key, model, voice, width, height, extraPrompt, subtitleStyle, req } = ctx; // 1) 校验图片文件 const files = []; @@ -313,7 +315,7 @@ async function runAiVideo(ctx) { task.stepLabel = `正在合成画面 ${i + 1}/${segments.length}`; task.progress = 50 + Math.round((i / segments.length) * 45); const out = path.join(work, `seg${i}.mp4`); - await videotool.buildSegment(segments[i], out, { width, height }); + await videotool.buildSegment(segments[i], out, { width, height, subtitleStyle: ctx.subtitleStyle }); segFiles.push(out); } @@ -348,4 +350,190 @@ async function runAiVideo(ctx) { } } +// ---------- URL 抓取:内容 + 图片 ---------- +router.post( + '/extract-url', + asyncHandler(async (req, res) => { + const { url } = req.body || {}; + if (!url) throw new ValidationError('缺少 url'); + let html = ''; + try { + const r = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' }, signal: AbortSignal.timeout(20000) }); + html = await r.text(); + } catch (e) { throw new ValidationError('抓取失败: ' + (e.message || e)); } + const titleMatch = html.match(/]*>([\s\S]*?)<\/title>/i); + const title = titleMatch ? titleMatch[1].replace(/\s+/g, ' ').trim().slice(0, 80) : ''; + // images + const imgs = []; + const imgRe = /]+src=["']([^"']+)["']/gi; + let m; + while ((m = imgRe.exec(html)) && imgs.length < 8) { + let src = m[1]; + if (src.startsWith('//')) src = 'https:' + src; + else if (src.startsWith('/')) { try { src = new URL(src, url).href; } catch (_) {} } + else if (!/^https?:/i.test(src)) { try { src = new URL(src, url).href; } catch (_) {} } + if (/^https?:/i.test(src) && !imgs.includes(src)) imgs.push(src); + } + // text: strip tags + const text = html + .replace(//gi, '') + .replace(//gi, '') + .replace(/<[^>]+>/g, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 4000); + res.json({ success: true, title, text, images: imgs }); + }) +); + +// ---------- 文档/PDF 提取 ---------- +router.post( + '/extract-doc', + upload.single('file'), + asyncHandler(async (req, res) => { + if (!req.file) throw new ValidationError('缺少文件'); + const name = req.file.originalname || 'doc'; + let text = ''; + const ext = path.extname(name).toLowerCase(); + if (ext === '.pdf') { + const tmp = path.join(os.tmpdir(), `doc_${Date.now()}.pdf`); + fs.writeFileSync(tmp, req.file.buffer); + try { + const { stdout } = await videotool.run('pdftotext', ['-layout', tmp, '-'], { timeoutMs: 60000 }); + text = stdout; + } catch (e) { + text = req.file.buffer.toString('utf8').replace(/[^\x20-\x7E\u4e00-\u9fa5]/g, ' '); + } finally { + try { fs.unlinkSync(tmp); } catch (_) {} + } + } else { + text = req.file.buffer.toString('utf8'); + } + text = String(text).replace(/\s+/g, ' ').trim().slice(0, 6000); + res.json({ success: true, title: name.replace(/\.[^.]+$/, ''), text, images: [] }); + }) +); + +// ---------- 文本源 AI 生成视频(提示词/URL/文档) ---------- +const textTasks = new Map(); +let textTaskSeq = 0; +router.post( + '/aigenerate-text', + asyncHandler(async (req, res) => { + const { + libraryId, title, content, images = [], + apiKey, baseUrl, model, + voice = 'zh-CN-XiaoxiaoNeural', + resolution = '1920x1080', + extraPrompt = '', + subtitleStyle = {} + } = req.body || {}; + if (!libraryId) throw new ValidationError('缺少 libraryId'); + if (!title || !String(title).trim()) throw new ValidationError('请填写视频标题'); + const text = String(content || '').trim(); + if (!text && (!Array.isArray(images) || images.length === 0)) throw new ValidationError('缺少内容或图片'); + + const configManager = req.app.get('configManager'); + const config = configManager.load(); + const library = config.libraries.find((l) => String(l.id) === String(libraryId)); + if (!library) throw new NotFoundError('Library', libraryId); + + const settings = getSettings(); + // 设置 - AI 视频(或复用 AI 生图)中的文本模型配置;请求参数可临时覆盖 + const gen = resolveVideoGen(settings); + const url = (baseUrl || gen.url || '').replace(/\/+$/, ''); + const key = apiKey || gen.apiKey || process.env.NEWAPI_KEY || ''; + const mdl = model || gen.model || 'mimo-v2.6-flash'; + if (!url) throw new ValidationError('未配置模型接口地址(请到右上角设置 → AI 视频里填写)'); + if (!key) throw new ValidationError('请先在右上角设置 → AI 视频里填写模型 KEY'); + + const id = `vt${++textTaskSeq}_${Date.now()}`; + const [w, h] = String(resolution).split('x').map((n) => parseInt(n, 10)); + const task = { id, status: 'running', step: 'script', stepLabel: '正在生成', progress: 0, error: '', resultPath: '', createdAt: Date.now() }; + textTasks.set(id, task); + setTimeout(() => textTasks.delete(id), 30 * 60 * 1000); + + runTextVideo({ task, library, title: String(title).trim(), content: text, images, url, key, model: mdl, voice, width: Number.isFinite(w)?w:1920, height: Number.isFinite(h)?h:1080, extraPrompt: String(extraPrompt||''), subtitleStyle: subtitleStyle||{}, req }).catch((err) => { + task.status = 'error'; task.error = err.message || String(err); + }); + res.json({ success: true, taskId: id }); + }) +); +async function runTextVideo(ctx) { + const { task, library, title, content, images, url, key, model, voice, width, height, extraPrompt, subtitleStyle, req } = ctx; + const work = fs.mkdtempSync(path.join(os.tmpdir(), 'flypic-tv-')); + try { + // 生成背景图(若无图片) + let bgPath = ''; + if (Array.isArray(images) && images.length) { + // 用第一张图作为背景 + const rel = typeof images[0] === 'string' ? images[0] : images[0].path || images[0].url; + if (rel && fs.existsSync(path.join(library.path, rel))) bgPath = path.join(library.path, rel); + else if (rel && /^https?:/i.test(rel)) { + // 下载 + try { + const r = await fetch(rel); const buf = Buffer.from(await r.arrayBuffer()); + bgPath = path.join(work, 'bg.jpg'); fs.writeFileSync(bgPath, buf); + } catch (_) {} + } + } + if (!bgPath) { + bgPath = path.join(work, 'bg.png'); + await videotool.run('ffmpeg', ['-y', '-f', 'lavfi', '-i', `color=c=0x1a1a2e:s=${width}x${height}`, '-frames:v', '1', bgPath], { timeoutMs: 60000 }); + } + + // LLM 生成分镜:基于内容切分 + task.step = 'script'; task.stepLabel = '正在生成脚本'; task.progress = 5; + let script = []; + if (content) { + // 让模型按内容生成 N 段(N = min(8, 句子数)) + const sentences = content.split(/[。!?.!?]/).map(s=>s.trim()).filter(Boolean); + const n = Math.min(8, Math.max(2, sentences.length)); + script = await llmScript({ url, key, model, title, count: n, extraPrompt: (extraPrompt + ' 内容:' + content.slice(0, 1500)).trim() }); + } else { + script = [{ subtitle: title, tts: title }]; + } + if (!Array.isArray(script) || !script.length) script = [{ subtitle: title, tts: title }]; + + // TTS + segments + const segments = []; + for (let i = 0; i < script.length; i++) { + task.step = 'tts'; task.stepLabel = `正在朗读 ${i+1}/${script.length}`; task.progress = 10 + Math.round((i/script.length)*40); + const seg = script[i]; + let ttsFile = ''; let dur = 0; + if (seg.tts) { + try { + const t = await videotool.generateTts(seg.tts, { voice, outPath: path.join(work, `tts${i}.mp3`) }); + ttsFile = t.file; dur = t.seconds; + } catch (e) { logger.warn('TTS fail: ' + e.message); } + } + segments.push({ imagePath: bgPath, subtitle: seg.subtitle || seg.tts || title, ttsFile, duration: Math.max(2.5, Math.min(12, dur || 4)) }); + } + + // compose + const segFiles = []; + for (let i = 0; i < segments.length; i++) { + task.step = 'compose'; task.stepLabel = `合成 ${i+1}/${segments.length}`; task.progress = 50 + Math.round((i/segments.length)*45); + const out = path.join(work, `seg${i}.mp4`); + await videotool.buildSegment(segments[i], out, { width, height, subtitleStyle }); + segFiles.push(out); + } + task.step = 'merge'; task.stepLabel = '合并'; task.progress = 96; + const finalTmp = path.join(work, 'final.mp4'); + await videotool.concatSegments(segFiles, finalTmp); + + const safeTitle = String(title).replace(/[\\/:*?"<>|]/g, '_').slice(0, 60) || 'AI视频'; + let target = path.join(library.path, `${safeTitle}.mp4`); + let n = 1; + while (fs.existsSync(target)) { target = path.join(library.path, `${safeTitle}_${n}.mp4`); n++; } + fs.copyFileSync(finalTmp, target); + await reindex(req, library.path, target); + + task.status = 'done'; task.progress = 100; task.step = 'done'; task.stepLabel = '完成'; task.resultPath = path.relative(library.path, target); + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +} + +// llmScript is defined later? it's a function declaration in same module scope (hoisted). ok. module.exports = router; diff --git a/backend/utils/settings.js b/backend/utils/settings.js index 5f45cc8..882d25f 100644 --- a/backend/utils/settings.js +++ b/backend/utils/settings.js @@ -30,6 +30,16 @@ const DEFAULTS = { apiKey: '', model: '' } + }, + // AI 视频:一句话/文本/网页/文档 → 带字幕配音的成片 + // provider='inherit' 复用上面的 AI 生图配置;'custom' 用下面自定义的文本模型 + videoai: { + provider: 'inherit', + custom: { + url: '', + model: '', + apiKey: '' + } } }; @@ -64,6 +74,14 @@ function getSettings() { apiKey: (data.imagegen && data.imagegen.custom && data.imagegen.custom.apiKey) || '', model: (data.imagegen && data.imagegen.custom && data.imagegen.custom.model) || '' } + }, + videoai: { + provider: (data.videoai && data.videoai.provider) === 'custom' ? 'custom' : 'inherit', + custom: { + url: (data.videoai && data.videoai.custom && data.videoai.custom.url) || '', + model: (data.videoai && data.videoai.custom && data.videoai.custom.model) || '', + apiKey: (data.videoai && data.videoai.custom && data.videoai.custom.apiKey) || '' + } } }; } catch (e) { @@ -108,6 +126,19 @@ function saveSettings(settings) { } } } + if (settings.videoai) { + const va = settings.videoai; + if (va.provider === 'custom' || va.provider === 'inherit') merged.videoai.provider = va.provider; + if (va.custom) { + if (typeof va.custom.url === 'string') merged.videoai.custom.url = va.custom.url.trim(); + if (typeof va.custom.model === 'string') merged.videoai.custom.model = va.custom.model.trim(); + const vk = va.custom.apiKey; + // 空串/掩码 = 保持原密钥 + if (typeof vk === 'string' && vk.trim() && !vk.includes('***')) { + merged.videoai.custom.apiKey = vk.trim(); + } + } + } fs.writeFileSync(p, JSON.stringify(merged, null, 2)); return merged; } @@ -119,4 +150,36 @@ function maskKey(key) { return `${key.slice(0, 4)}***${key.slice(-4)}`; } -module.exports = { getSettings, saveSettings, maskKey, settingsPath }; +/** + * 解析 AI 视频使用的文本大模型配置 + * - 设置-AI 视频里选了「自定义」且填了地址 → 用自定义 + * - 否则复用 AI 生图配置(内置 NewAPI / 自定义提供商) + * 普通文本模型(chat/completions)即可,全流程由后端写分镜、TTS 配音、合成字幕。 + */ +function resolveVideoGen(settings) { + const s = settings || getSettings(); + const va = (s && s.videoai) || {}; + const ig = (s && s.imagegen) || {}; + const defModel = process.env.NEWAPI_TEXT_MODEL || 'mimo-v2.6-flash'; + if (va.provider === 'custom' && va.custom && va.custom.url) { + return { + url: String(va.custom.url).replace(/\/+$/, ''), + model: va.custom.model || defModel, + apiKey: va.custom.apiKey || process.env.NEWAPI_KEY || '' + }; + } + if (ig.provider === 'custom' && ig.custom) { + return { + url: String(ig.custom.url || '').replace(/\/+$/, ''), + model: ig.custom.model || defModel, + apiKey: ig.custom.apiKey || process.env.NEWAPI_KEY || '' + }; + } + return { + url: String((ig.newapi && ig.newapi.url) || 'http://192.168.68.19:3099/v1').replace(/\/+$/, ''), + model: defModel, + apiKey: ((ig.newapi && ig.newapi.apiKey) || process.env.NEWAPI_KEY || '') + }; +} + +module.exports = { getSettings, saveSettings, maskKey, settingsPath, resolveVideoGen }; diff --git a/backend/utils/videotool.js b/backend/utils/videotool.js index b236b21..f7a281e 100644 --- a/backend/utils/videotool.js +++ b/backend/utils/videotool.js @@ -365,17 +365,21 @@ async function stripSubtitleAudio(input, output, { removeAudio = true, removeSub * AI 生成视频素材装配(分段:图 + 该段字幕 + 该段配音) * @param {object} seg { imagePath, subtitle, ttsFile, duration } duration=该段秒数 */ -async function buildSegment(seg, outPath, { width = 1920, height = 1080, fontSizePct = 5 } = {}) { +async function buildSegment(seg, outPath, { width = 1920, height = 1080, fontSizePct = 5, subtitleStyle = {} } = {}) { const dir = tmpDir(); try { const tFile = path.join(dir, 'sub.txt'); fs.writeFileSync(tFile, seg.subtitle || '', 'utf8'); const textFile = tFile; - const fontSize = Math.max(12, Math.round((height * fontSizePct) / 100)); + const fontSize = Math.max(12, Math.round((height * (subtitleStyle.fontSizePct || fontSizePct)) / 100)); + const color = subtitleStyle.color || 'white'; + const pos = subtitleStyle.position || 'bottom-center'; + const yExpr = pos.startsWith('top') ? 'h*0.07' : pos === 'center' ? '(h-text_h)/2' : 'h-text_h-h*0.07'; + const xExpr = pos.endsWith('left') ? 'w*0.05' : pos.endsWith('right') ? 'w-text_w-w*0.05' : '(w-text_w)/2'; const vf = [ `scale=${width}:${height}:force_original_aspect_ratio=decrease`, `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, - `drawtext=fontfile=/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc:textfile=${textFile}:fontsize=${fontSize}:fontcolor=white@0.95:borderw=2:bordercolor=black@0.6:x=(w-text_w)/2:y=h-text_h-h*0.07:line_spacing=8` + `drawtext=fontfile=/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc:textfile=${textFile}:fontsize=${fontSize}:fontcolor=${color}@0.95:borderw=2:bordercolor=black@0.6:x=${xExpr}:y=${yExpr}:line_spacing=8` ].join(','); const args = [ diff --git a/frontend/src/components/AIGenModal.jsx b/frontend/src/components/AIGenModal.jsx index 2789c4d..26c6a56 100644 --- a/frontend/src/components/AIGenModal.jsx +++ b/frontend/src/components/AIGenModal.jsx @@ -1,7 +1,8 @@ import { useState, useEffect, useRef, useCallback } from 'react'; import { X, Wand2, Loader2, Download, Save, RefreshCw, Sparkles, - ChevronDown, FolderOpen, CheckCircle2, AlertTriangle + ChevronDown, FolderOpen, CheckCircle2, AlertTriangle, + Image as ImageIcon, Search, Plus, Trash2 } from 'lucide-react'; import { aigenAPI, fileAPI, imageAPI } from '../api'; import { useLibraryStore } from '../stores/useLibraryStore'; @@ -49,6 +50,14 @@ export default function AIGenModal({ open, onClose, onSaved }) { const [saved, setSaved] = useState(false); const [showFolderPick, setShowFolderPick] = useState(false); const [advanced, setAdvanced] = useState(false); + + // 参考图(以图生图) + const [editModels, setEditModels] = useState([]); + const [refs, setRefs] = useState([]); // [{ path, filename, thumbnailPath }] + const [showRefPicker, setShowRefPicker] = useState(false); + const [refList, setRefList] = useState([]); + const [refQuery, setRefQuery] = useState(''); + const [refLoading, setRefLoading] = useState(false); const imgRef = useRef(null); // 打开时拉模型列表 + 当前配置 @@ -57,9 +66,13 @@ export default function AIGenModal({ open, onClose, onSaved }) { setError(''); setResult(null); setSaved(false); + setRefs([]); + setShowRefPicker(false); + setRefQuery(''); aigenAPI.getModels() .then((d) => { setGroups(d.groups || []); + setEditModels(d.editModels || []); setSizes(d.sizes || []); setProviderInfo(d.provider || null); const first = (d.groups || [])[0]; @@ -70,17 +83,70 @@ export default function AIGenModal({ open, onClose, onSaved }) { }, [open]); const currentModelId = useCustom ? (providerInfo?.customModel || '') : model; + + /** 素材库缩略图地址 */ + const thumbUrl = (img) => { + if (!img || !currentLibraryId) return ''; + const raw = img.thumbnailPath || img.path || img.filename || ''; + const fn = String(raw).replace(/\\/g, '/').split('/').pop(); + return `/api/image/thumbnail/${currentLibraryId}/${encodeURIComponent(fn)}`; + }; + + /** 打开素材库选图:读当前库的图片(只图片,不含视频) */ + const openRefPicker = async () => { + setShowRefPicker(true); + setRefLoading(true); + setRefList([]); + try { + const resp = await imageAPI.search(currentLibraryId, { limit: 200, offset: 0, q: refQuery || undefined }); + const list = (resp.images || []).filter((it) => { + const t = String(it.fileType || it.type || '').toLowerCase(); + if (t === 'video') return false; + const ext = String(it.filename || it.path || '').split('.').pop().toLowerCase(); + return !['mp4', 'mov', 'mkv', 'webm', 'avi', 'm4v', 'ts', 'flv'].includes(ext); + }); + setRefList(list); + } catch (e) { + setError('读取素材库图片失败: ' + (e.data?.error || e.message)); + } finally { + setRefLoading(false); + } + }; + + const isPickedRef = (img) => refs.some((r) => r.path === img.path); + + const toggleRef = (img) => { + setError(''); + if (isPickedRef(img)) { + setRefs((prev) => prev.filter((r) => r.path !== img.path)); + return; + } + if (refs.length >= 4) { setError('最多选 4 张参考图'); return; } + setRefs((prev) => [...prev, img]); + // 选了参考图就自动切到图生图模型,避免用户忘记换模型导致图片没被用上 + if (editModels.length && !String(model).startsWith('flux-2')) { + setModel(editModels[0].id); + } + }; const handleGenerate = async () => { if (!prompt.trim()) { setError('请先输入提示词,或点一个热门提示词'); return; } setLoading(true); setError(''); setSaved(false); try { + if (refs.length > 0 && !useCustom && !String(model).startsWith('flux-2')) { + setError('已选参考图,请把模型切换为「FLUX.2 图生图」系列'); + setLoading(false); + return; + } const d = await aigenAPI.generate({ prompt: prompt.trim(), model: useCustom ? undefined : model, size, - provider: useCustom ? 'custom' : 'builtin' + provider: useCustom ? 'custom' : 'builtin', + libraryId: currentLibraryId, + refImages: refs.map((r) => ({ path: r.path, filename: r.filename })), + ...(refs.length ? {} : {}) }); const img = d.images && d.images[0]; if (!img || !img.b64) throw new Error('未返回图片数据'); @@ -265,6 +331,13 @@ export default function AIGenModal({ open, onClose, onSaved }) { ))} ))} + {editModels.length > 0 && ( + + {editModels.map((m) => ( + + ))} + + )} )} @@ -282,6 +355,52 @@ export default function AIGenModal({ open, onClose, onSaved }) { )} + {/* 参考图:以图生图 */} +
+
+ + {refs.length > 0 && ( + + )} +
+
+ {refs.map((r, i) => ( +
+ {r.filename + +
+ ))} + {refs.length < 4 && ( + + )} +
+

+ {refs.length > 0 + ? '将按参考图生成(自动切到 FLUX.2 图生图模型),图片只在本机读取,不上传到第三方' + : '选 1-4 张素材库图片做参考,即可换背景、改风格、保持人物一致'} +

+
+ {/* 尺寸 */}
@@ -323,7 +442,8 @@ export default function AIGenModal({ open, onClose, onSaved }) { {loading && (
- 正在调用 {useCustom ? '自定义提供商' : (model || '')},超时上限 150 秒 + 正在调用 {useCustom ? '自定义提供商' : (model || '')} + {refs.length > 0 ? '(以图生图,约 20-60 秒)' : ',超时上限 150 秒'}
)}
@@ -427,6 +547,93 @@ export default function AIGenModal({ open, onClose, onSaved }) { )} + + {/* 素材库选图(以图生图) */} + {showRefPicker && ( +
setShowRefPicker(false)} + > +
e.stopPropagation()} + > +
+
+ +

从素材库选择参考图

+ 已选 {refs.length}/4 +
+ +
+ +
+
+ + setRefQuery(e.target.value)} + onKeyDown={(e) => { if (e.key === 'Enter') openRefPicker(); }} + placeholder="搜索文件名,回车查询" + className="w-full pl-8 pr-3 py-1.5 text-xs border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100" + /> +
+ +
+ +
+ {refLoading ? ( +
+ 读取图片中… +
+ ) : refList.length === 0 ? ( +
这个素材库里没有可选图片
+ ) : ( +
+ {refList.map((img) => { + const picked = isPickedRef(img); + const idx = refs.findIndex((r) => r.path === img.path); + return ( + + ); + })} +
+ )} +
+ +
+ 点图多选,按点选顺序作为参考顺序 + +
+
+
+ )} ); } diff --git a/frontend/src/components/AIVideoStudioModal.jsx b/frontend/src/components/AIVideoStudioModal.jsx new file mode 100644 index 0000000..cb5458a --- /dev/null +++ b/frontend/src/components/AIVideoStudioModal.jsx @@ -0,0 +1,393 @@ +import { useState, useEffect } from 'react'; +import { X, Film, Loader2, CheckCircle2, AlertCircle, Link as LinkIcon, FileText, Image as ImageIcon, Sparkles, Search, Check, Settings } from 'lucide-react'; +import { getToken } from '../api/client'; + +const VOICES = [ + { key: 'zh-CN-XiaoxiaoNeural', label: '晓晓(女声·自然)' }, + { key: 'zh-CN-YunxiNeural', label: '云希(男声·温暖)' }, + { key: 'zh-CN-YunjianNeural', label: '云健(男声·浑厚)' }, + { key: 'zh-CN-XiaoyiNeural', label: '晓伊(女声·活泼)' }, + { key: 'zh-CN-YunyangNeural', label: '云扬(男声·播音)' }, +]; +const RESOLUTIONS = [ + { key: '1920x1080', label: '横屏 1920×1080' }, + { key: '1080x1920', label: '竖屏 1080×1920' }, +]; +const SUBTITLE_STYLES = [ + { key: 'bottom-center', label: '底部居中', position: 'bottom-center' }, + { key: 'top-center', label: '顶部居中', position: 'top-center' }, + { key: 'center', label: '画面中央', position: 'center' }, +]; +const SUBTITLE_COLORS = [ + { key: 'white', label: '白色' }, + { key: '#FFD700', label: '金色' }, + { key: '#00FFCC', label: '青色' }, +]; +const POPULAR_PROMPTS = [ + '产品功能速览:突出核心卖点与使用场景', + '周末探店 vlog:轻松活泼的语气', + '知识科普:分步骤讲解一个概念', + '旅行日记:描述风景与心情', + '健身打卡:励志、节奏感强', + '美食制作:步骤清晰、诱人', + '读书分享:提炼要点与感悟', + '科技新闻:简洁、专业', +]; + +function errText(e, fb) { if (!e) return fb; if (typeof e === 'string') return e; return e.message || JSON.stringify(e); } + +function thumbUrl(libraryId, img) { + const tp = img.thumbnailPath || ''; + const fn = tp ? tp.replace(/\\/g, '/').split('/').pop() : ''; + if (fn) return `/api/image/thumbnail/${libraryId}/${encodeURIComponent(fn)}${img.modifiedAt ? `?v=${img.modifiedAt}` : ''}`; + return `/api/image/original/${libraryId}/${String(img.path || '').split('/').map(encodeURIComponent).join('/')}`; +} + +export default function AIVideoStudioModal({ libraryId, onClose, onDone }) { + const [tab, setTab] = useState('prompt'); + const [title, setTitle] = useState(''); + const [content, setContent] = useState(''); + const [url, setUrl] = useState(''); + const [extracted, setExtracted] = useState(null); + const [extracting, setExtracting] = useState(false); + const [file, setFile] = useState(null); + const [voice, setVoice] = useState(VOICES[0].key); + const [resolution, setResolution] = useState('1920x1080'); + const [subtitlePos, setSubtitlePos] = useState('bottom-center'); + const [subtitleColor, setSubtitleColor] = useState('white'); + const [subtitleSize, setSubtitleSize] = useState(5); + // 素材库图片选择(弹窗内直接挑选) + const [libImgs, setLibImgs] = useState([]); + const [libLoading, setLibLoading] = useState(false); + const [keyword, setKeyword] = useState(''); + const [picked, setPicked] = useState([]); + const [taskId, setTaskId] = useState(''); + const [task, setTask] = useState(null); + const [error, setError] = useState(''); + const [starting, setStarting] = useState(false); + const timerRef = { current: null }; + + const auth = { Authorization: `Bearer ${getToken()}` }; + + const loadImages = async (kw) => { + setLibLoading(true); + try { + const q = new URLSearchParams({ libraryId: String(libraryId), offset: '0', limit: '120' }); + if (kw && kw.trim()) q.set('keywords', kw.trim()); + const r = await fetch(`/api/image?${q}`, { headers: auth }); + const d = await r.json(); + // 只允许图片做视频画面:视频文件当图传给 ffmpeg 会直接报 Option loop not found + const VIDEO_EXT = ['mp4', 'mov', 'mkv', 'webm', 'avi', 'm4v', 'flv', 'wmv', 'ts']; + setLibImgs(((d && d.data && d.data.images) || []).filter((i) => { + if (!i || !i.path) return false; + const t = String(i.fileType || i.type || '').toLowerCase(); + if (t === 'video') return false; + const ext = String(i.filename || i.path).split('.').pop().toLowerCase(); + return !VIDEO_EXT.includes(ext); + })); + } catch (_) { + setLibImgs([]); + } finally { + setLibLoading(false); + } + }; + + useEffect(() => { if (libraryId) loadImages(''); }, [libraryId]); + + const togglePick = (img) => { + setPicked((list) => { + const i = list.findIndex((x) => x.path === img.path); + if (i >= 0) return list.filter((x) => x.path !== img.path); + return [...list, img]; + }); + }; + + const poll = (id) => { + const iv = setInterval(async () => { + try { + const r = await fetch(`/api/video/task/${id}`, { headers: { Authorization: `Bearer ${getToken()}` } }); + if (!r.ok) return; + const d = await r.json(); + setTask(d); + if (d.status === 'done' || d.status === 'error') { + clearInterval(iv); + if (d.status === 'done') setTimeout(() => { onDone && onDone(d.resultPath); onClose(); }, 1200); + } + } catch (_) {} + }, 1500); + timerRef.current = iv; + }; + + const handleExtractUrl = async () => { + setExtracting(true); setError(''); + try { + const r = await fetch('/api/video/extract-url', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${getToken()}` }, body: JSON.stringify({ url }) }); + const d = await r.json(); + if (!r.ok || d.error) throw new Error(errText(d.error, `失败 (${r.status})`)); + setExtracted(d); + if (!title) setTitle(d.title || ''); + setContent(d.text || ''); + } catch (e) { setError(e.message); } + finally { setExtracting(false); } + }; + + const handleExtractDoc = async () => { + if (!file) return; + setExtracting(true); setError(''); + try { + const fd = new FormData(); fd.append('file', file); + const r = await fetch('/api/video/extract-doc', { method: 'POST', headers: { Authorization: `Bearer ${getToken()}` }, body: fd }); + const d = await r.json(); + if (!r.ok || d.error) throw new Error(errText(d.error, `失败 (${r.status})`)); + setExtracted(d); + if (!title) setTitle(d.title || ''); + setContent(d.text || ''); + } catch (e) { setError(e.message); } + finally { setExtracting(false); } + }; + + const handleStart = async () => { + setError(''); + if (!title.trim()) { setError('请填写视频标题'); return; } + if (tab === 'images' && picked.length === 0) { setError('请先在「素材库图片」里选择至少 1 张'); return; } + setStarting(true); + try { + const imgs = picked.length + ? picked.map((i) => ({ path: i.path, filename: i.filename, width: i.width, height: i.height })) + : ((extracted && extracted.images) || []); + const body = { + libraryId, title: title.trim(), + content: content.trim(), + images: imgs, + voice, resolution, extraPrompt: '', + subtitleStyle: { position: subtitlePos, color: subtitleColor, fontSizePct: subtitleSize } + }; + const r = await fetch('/api/video/aigenerate-text', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${getToken()}` }, body: JSON.stringify(body) }); + const d = await r.json().catch(() => ({})); + if (!r.ok || d.error) throw new Error(errText(d.error, `启动失败 (${r.status})`)); + setTaskId(d.taskId); poll(d.taskId); + } catch (e) { setError(e.message || '启动失败'); } + finally { setStarting(false); } + }; + + const running = !!taskId && task && task.status === 'running'; + const failed = task && task.status === 'error'; + const done = task && task.status === 'done'; + + const input = 'w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 text-sm focus:ring-2 focus:ring-blue-500 focus:border-transparent'; + + return ( +
+
e.stopPropagation()}> +
+
+ +

AI 一键生成视频

+
+ +
+
+ {/* tabs */} +
+ {[ + { k: 'prompt', l: '提示词', icon: Sparkles }, + { k: 'url', l: '链接抓取', icon: LinkIcon }, + { k: 'doc', l: '文档/PDF', icon: FileText }, + { k: 'images', l: '素材库图片', icon: ImageIcon }, + ].map(({ k, l, icon: Icon }) => ( + + ))} +
+ + {tab === 'prompt' && ( +
+
+ +
+ {POPULAR_PROMPTS.map((p) => ( + + ))} +
+
+
+ +