diff --git a/CHANGELOG.md b/CHANGELOG.md index 825a70c..f5bdb7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # 更新日志 +## 2026-11-17 → v1.4.0 + +视频能力大版本:素材库兼容视频目录、点击弹窗播放、四项常用视频编辑(加/去水印、加/去字幕声音),以及多选图片填标题 + 模型 KEY 后 AI 一键生成带字幕和朗读的成片。 + +- ✨ 新功能:**视频目录导入** —— 扫描识别视频文件,瀑布流卡片显示 MP4 徽标与时长角标,与图片混排同库管理(搜索/收藏/回收站通用) +- ✨ 新功能:**单击播放** —— 点击视频卡弹出全屏播放器(原片播放、进度/音量控制),可新窗口打开原片 +- ✨ 新功能:**视频编辑**(右键「编辑视频(水印/字幕声音)…」,后端 ffmpeg 处理): + - **加水印**:文字水印(内容/位置/字号/透明度/颜色,可叠加时间戳) + - **去水印**:框选区域模糊/填充去除 + - **加字幕声音**:SRT 字幕烧录进画面,可同时合成朗读音轨 + - **去字幕声音**:移除画面字幕区与原音轨;处理完自动入库 +- ✨ 新功能:**AI 一键生成视频**(多选 ≥2 张图片 → 右键「AI 生成视频…」)—— 填**视频标题** + **模型 KEY**(OpenAI 兼容,留空用服务器已配 KEY),选朗读音色(6 种)与画幅(横/竖/720p),可加补充要求;流程:AI 写分镜脚本 → 逐段朗读配音 → 图片逐段配字幕合成 → 拼接成片自动入库 +- 🔧 后端新增 `/api/video`(watermark / dewatermark / subtitle / unsubtitle)与 `/api/video/ai-generate` 异步任务;`scanner.js` 识别 `fileType=video` 并入库时长 + ## 2026-11-17 → v1.3.0 本版本为图片管理能力的大版本:图片编辑器(裁剪/旋转/调色等 PS 常用功能)、批量水印、AI 生图、收藏体系、合拼拼图、图片右上角一键新窗口打开。 diff --git a/Dockerfile b/Dockerfile index 4e215b1..b62d3d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ FROM docker.1ms.run/library/node:22-slim AS runner WORKDIR /app -# 运行时依赖:sharp + tesseract OCR + 中英文语言包 +# 运行时依赖:sharp + tesseract OCR + 中英文语言包 + ffmpeg(视频抽帧/转码)+ edge-tts(AI 视频朗读) RUN sed -i 's|deb.debian.org|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null; \ sed -i 's|deb.debian.org|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list 2>/dev/null; \ apt-get update && apt-get install -y --no-install-recommends \ @@ -35,9 +35,15 @@ RUN sed -i 's|deb.debian.org|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.li tesseract-ocr-eng \ fonts-noto-cjk \ fontconfig \ + ffmpeg \ + python3 \ + python3-pip \ && rm -rf /var/lib/apt/lists/* \ && fc-cache -f +# edge-tts:AI 一键生成视频的朗读配音(Microsoft Edge TTS,仅安装脚本层,凭据无) +RUN pip3 install --break-system-packages --no-cache-dir -i https://pypi.tuna.tsinghua.edu.cn/simple edge-tts + # 复制后端代码和依赖 COPY --from=backend-builder /app/backend/node_modules ./backend/node_modules COPY backend/ ./backend/ diff --git a/README.md b/README.md index 9d82d7c..ee45177 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,20 @@ - **图片右上角「在新窗口打开」**:悬停图片右上角出现外链按钮,点击在**新标签页打开原图**,不打断当前浏览(非图片类型的文件类型标识自动左移让位) - **合拼拼图**与**返回顶部按钮** +### 10. 视频功能:目录导入 / 播放 / 常用编辑 / AI 一键成片(v1.4.0 新增) + +- **素材库兼容视频目录**:扫描时识别视频文件(mp4 等),瀑布流卡片显示 **MP4 类型徽标**与**时长角标**(如 `0:09`),与图片混排、同一套搜索/收藏/回收站体系 +- **单击播放**:点击视频卡弹出全屏播放器(原片直接播放,可暂停/进度/音量),右上角可新窗口打开原片 +- **常用视频编辑**(右键「编辑视频(水印/字幕声音)…」,弹窗四选一 + 开始处理,后端 ffmpeg): + - **加水印**:文字水印(内容/位置/字号/透明度/颜色),可叠加时间戳 + - **去水印**:框选区域模糊/填充去除 + - **加字幕声音**:外挂字幕轨烧录进画面(支持 SRT 字幕文件),可同时合成朗读音轨 + - **去字幕声音**:移除画面字幕区与原音轨 + - 处理完自动入库,新文件可在瀑布流直接看到 +- **AI 一键生成视频**(多选 ≥2 张图片 → 右键「AI 生成视频…」): + - 表单:**视频标题**(AI 按此写分镜旁白,成片同名)+ **模型 KEY**(OpenAI 兼容,留空用服务器已配 KEY)+ **朗读音色**(晓晓/云希/云健/晓伊/云扬/晓臻 6 种)+ **画幅**(1920×1080 / 1080×1920 / 1280×720)+ 补充要求 + - 流程全自动:**AI 写分镜脚本 → 逐段朗读配音 → 图片逐段配字幕合成 → 拼接成片入库**(约 10–60 秒/张),完成后成片出现在素材库 + ## 🚀 部署方式 ### 方式一:Docker Compose(推荐) diff --git a/backend/database/db.js b/backend/database/db.js index 693cb27..79c9a17 100644 --- a/backend/database/db.js +++ b/backend/database/db.js @@ -104,6 +104,13 @@ class LibraryDatabase { // 列已存在,忽略错误 } + // 添加 duration 列(如果不存在):视频时长(秒),图片为 0 + try { + this.db.exec(`ALTER TABLE images ADD COLUMN duration REAL DEFAULT 0`); + } catch (e) { + // 列已存在,忽略错误 + } + // Folders table this.db.exec(` CREATE TABLE IF NOT EXISTS folders ( @@ -184,8 +191,8 @@ class LibraryDatabase { insertImage(imageData) { const stmt = this.db.prepare(` INSERT OR REPLACE INTO images - (path, filename, folder, size, width, height, format, file_type, created_at, modified_at, file_hash, thumbnail_path, thumbnail_size, indexed_at, ocr_text, ocr_done) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (path, filename, folder, size, width, height, format, file_type, created_at, modified_at, file_hash, thumbnail_path, thumbnail_size, indexed_at, ocr_text, ocr_done, duration) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); const result = stmt.run( imageData.path, @@ -203,7 +210,8 @@ class LibraryDatabase { imageData.thumbnail_size, Date.now(), imageData.ocr_text || '', - imageData.ocr_done !== undefined ? imageData.ocr_done : 0 + imageData.ocr_done !== undefined ? imageData.ocr_done : 0, + imageData.duration || 0 ); // 更新数据库修改时间 this.updateLastModified(); diff --git a/backend/src/app.js b/backend/src/app.js index a50bdec..f93bb7f 100644 --- a/backend/src/app.js +++ b/backend/src/app.js @@ -88,6 +88,7 @@ function createApp(dependencies) { const editRouter = require('./routes/edit'); const batchRouter = require('./routes/batch'); const aigenRouter = require('./routes/aigen'); + const videoRouter = require('./routes/video'); app.use('/api/auth', authRouter); app.use('/api/library', libraryRouter); @@ -100,6 +101,7 @@ function createApp(dependencies) { app.use('/api/edit', editRouter); app.use('/api/batch', batchRouter); app.use('/api/aigen', aigenRouter); + app.use('/api/video', videoRouter); // 健康检查 app.get('/api/health', (req, res) => { diff --git a/backend/src/routes/video.js b/backend/src/routes/video.js new file mode 100644 index 0000000..c9ede3c --- /dev/null +++ b/backend/src/routes/video.js @@ -0,0 +1,351 @@ +/** + * 视频处理路由 + * - POST /api/video/edit 单视频编辑:watermark-add / watermark-remove / subtitle-audio / strip + * - POST /api/video/aigenerate 多图 + 标题 + 模型KEY → AI 生成带字幕朗读的视频(异步任务) + * - GET /api/video/task/:id 查询 AI 生成进度 + * - GET /api/video/probe ffprobe 宽高/时长(前端预览用) + * + * 输出约定与批量水印一致:output='new' 后缀另存(_wm/_nowm/_sub/_out),'overwrite' 原地覆盖(原子写)。 + */ +const express = require('express'); +const router = express.Router(); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); +const { asyncHandler, ValidationError, NotFoundError } = require('../middleware/errorHandler'); +const { processImage } = require('../../utils/scanner'); +const { getSettings } = require('../../utils/settings'); +const logger = require('../utils/logger'); +const videotool = require('../../utils/videotool'); + +const VIDEO_EXTS = new Set(['.mp4', '.mov', '.m4v', '.webm', '.mkv', '.avi', '.flv', '.wmv', '.mpg', '.mpeg', '.3gp', '.ts']); + +// ---------- 通用:定位文件 ---------- + +function resolveFile(req, libraryId, relPath) { + const configManager = req.app.get('configManager'); + const dbPool = req.app.get('dbPool'); + if (!configManager || !dbPool) throw new ValidationError('服务未初始化'); + const config = configManager.load(); + const library = config.libraries.find((l) => String(l.id) === String(libraryId)); + if (!library) throw new NotFoundError('Library', libraryId); + const abs = path.join(library.path, relPath); + if (!abs.startsWith(path.resolve(library.path))) throw new ValidationError('非法路径'); + if (!fs.existsSync(abs)) throw new NotFoundError('File', relPath); + const ext = path.extname(abs).toLowerCase(); + if (!VIDEO_EXTS.has(ext)) throw new ValidationError(`不是视频文件: ${relPath}`); + return { library, abs }; +} + +async function reindex(req, libraryPath, absPath) { + const dbPool = req.app.get('dbPool'); + const db = dbPool.acquire(libraryPath); + try { + await processImage(absPath, libraryPath, db); + } finally { + dbPool.release(libraryPath); + } +} + +/** 输出路径:new=后缀另存,overwrite=同名临时文件原子替换 */ +function outputPath(abs, mode, suffix) { + if (mode !== 'new') return { target: abs, atomic: true }; + const ext = path.extname(abs); + const base = abs.slice(0, abs.length - ext.length); + let target = `${base}${suffix}${ext}`; + let n = 1; + while (fs.existsSync(target)) { + target = `${base}${suffix}${n}${ext}`; + n++; + } + return { target, atomic: false }; +} + +// ---------- 单视频编辑 ---------- + +router.post( + '/edit', + asyncHandler(async (req, res) => { + const { libraryId, path: relPath, action, options = {}, output = 'new' } = req.body || {}; + if (!libraryId || !relPath) throw new ValidationError('缺少 libraryId / path'); + const { library, abs } = resolveFile(req, libraryId, relPath); + + const SUFFIX = { + 'watermark-add': '_wm', + 'watermark-remove': '_nowm', + 'subtitle-audio': '_sub', + 'strip': '_out' + }; + const { target, atomic } = outputPath(abs, output, SUFFIX[action] || '_edit'); + + let meta; + switch (action) { + case 'watermark-add': + meta = await videotool.addWatermark(abs, target, options); + break; + case 'watermark-remove': + if (!options.region) throw new ValidationError('缺少去水印区域'); + meta = await videotool.removeWatermark(abs, target, options.region); + break; + case 'subtitle-audio': + meta = await videotool.addSubtitleAudio(abs, target, options); + break; + case 'strip': + meta = await videotool.stripSubtitleAudio(abs, target, options); + break; + default: + throw new ValidationError(`未知视频操作: ${action}`); + } + + await reindex(req, library.path, target); + logger.info(`🎬 视频${{ 'watermark-add': '加水印', 'watermark-remove': '去水印', 'subtitle-audio': '加字幕声音', strip: '去字幕声音' }[action]} 完成: ${path.basename(target)}`); + res.json({ success: true, path: path.relative(library.path, target), duration: meta && meta.duration }); + }) +); + +// ---------- ffprobe 预览信息 ---------- + +router.get( + '/probe', + asyncHandler(async (req, res) => { + const { libraryId, path: relPath } = req.query; + if (!libraryId || !relPath) throw new ValidationError('缺少 libraryId / path'); + const { abs } = resolveFile(req, libraryId, relPath); + const info = await videotool.probeVideo(abs); + res.json(info); + }) +); + +// ---------- AI 生成视频(异步任务) ---------- + +const aiTasks = new Map(); +let aiTaskSeq = 0; + +router.post( + '/aigenerate', + asyncHandler(async (req, res) => { + const { + libraryId, + images, // [{ path }] 库内相对路径(同一文件夹顺序展示) + title, // 标题 + apiKey, // 模型 KEY(留空用服务器设置) + baseUrl, // OpenAI 兼容接口(留空用服务器设置) + model, // 模型名 + voice = 'zh-CN-XiaoxiaoNeural', + resolution = '1920x1080', + extraPrompt = '' + } = req.body || {}; + + if (!libraryId || !Array.isArray(images) || images.length < 2) { + throw new ValidationError('请至少选择 2 张图片'); + } + if (!title || !String(title).trim()) throw new ValidationError('请填写视频标题'); + + // 图片扩展名校验(buildSegment 用 image2 demuxer,视频文件会报 Option loop not found) + const IMG_EXTS = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif', '.bmp', '.avif']); + for (const it of images) { + const p = it && it.path ? String(it.path) : ''; + if (!p || !IMG_EXTS.has(path.extname(p).toLowerCase())) { + throw new ValidationError(`仅支持图片文件,收到: ${p || '(空)'}`); + } + } + + const configManager = req.app.get('configManager'); + const config = configManager.load(); + const library = config.libraries.find((l) => String(l.id) === String(libraryId)); + if (!library) throw new NotFoundError('Library', libraryId); + + const settings = getSettings(); + const gen = (settings.imagegen && settings.imagegen.provider === 'custom') + ? settings.imagegen.custom + : settings.imagegen.newapi; + const url = (baseUrl || gen.url || '').replace(/\/+$/, ''); + const key = apiKey || gen.apiKey || process.env.NEWAPI_KEY || ''; + const mdl = model || gen.model || 'mimo-v2.6-flash'; + if (!url) throw new ValidationError('未配置模型接口地址(填 BASE URL 或在设置里配置)'); + if (!key) throw new ValidationError('请填写模型 KEY(或在设置-生图里配置)'); + + const id = `vt${++aiTaskSeq}_${Date.now()}`; + const [w, h] = String(resolution).split('x').map((n) => parseInt(n, 10)); + const task = { + id, + status: 'running', + step: 'script', + stepLabel: '正在生成脚本', + progress: 0, + error: '', + resultPath: '', + createdAt: Date.now() + }; + aiTasks.set(id, task); + // 任务完成后 30 分钟清理 + setTimeout(() => aiTasks.delete(id), 30 * 60 * 1000); + + // 后台执行,不阻塞响应 + runAiVideo({ + task, library, images, title: String(title).trim(), url, key, model: mdl, + voice, width: Number.isFinite(w) ? w : 1920, height: Number.isFinite(h) ? h : 1080, + extraPrompt: String(extraPrompt || ''), + req + }).catch((err) => { + task.status = 'error'; + task.error = err.message || String(err); + logger.warn(`⚠️ AI 生成视频失败: ${task.error}`); + }); + + res.json({ success: true, taskId: id }); + }) +); + +router.get( + '/task/:id', + asyncHandler(async (req, res) => { + const task = aiTasks.get(req.params.id); + if (!task) throw new NotFoundError('Task', req.params.id); + res.json(task); + }) +); + +/** LLM 生成分镜脚本:返回 [{ subtitle, tts }] 数组,长度=图片数 */ +async function llmScript({ url, key, model, title, count, extraPrompt }) { + const sys = '你是短视频编导。根据给定标题与图片数量,输出每个画面的旁白与字幕。要求口语化、有画面感、避免空话。'; + const user = [ + `标题:${title}`, + `共 ${count} 张图片(将按顺序做成视频分镜)。`, + extraPrompt ? `补充要求:${extraPrompt}` : '', + `严格只输出 JSON 数组(不要代码块标记),长度恰好 ${count},每项格式:{"subtitle":"画面字幕(15字内)","tts":"朗读文案(30字内,可与字幕不同)"}` + ].filter(Boolean).join('\n'); + + const resp = await fetch(`${url}/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` }, + body: JSON.stringify({ + model, + messages: [ + { role: 'system', content: sys }, + { role: 'user', content: user } + ], + temperature: 0.7, + max_tokens: 4000 + }), + signal: AbortSignal.timeout(90 * 1000) + }); + if (!resp.ok) { + let msg = `模型接口错误 (${resp.status})`; + try { const j = await resp.json(); msg = (j.error && j.error.message) || j.message || msg; } catch (_) { /* 非 JSON */ } + throw new Error(msg); + } + const data = await resp.json(); + let content = data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content; + if (!content) throw new Error('模型未返回内容'); + content = String(content).replace(/^```(?:json)?\s*/i, '').replace(/```\s*$/, '').trim(); + let arr; + try { + arr = JSON.parse(content); + } catch (_) { + const m = content.match(/\[[\s\S]*\]/); + if (!m) throw new Error('脚本 JSON 解析失败'); + arr = JSON.parse(m[0]); + } + if (!Array.isArray(arr) || !arr.length) throw new Error('脚本为空'); + return arr.map((it) => ({ + subtitle: String((it && it.subtitle) || (it && it.text) || '').slice(0, 40), + tts: String((it && it.tts) || (it && it.subtitle) || '').slice(0, 120) + })); +} + +async function runAiVideo(ctx) { + const { task, library, images, title, url, key, model, voice, width, height, extraPrompt, req } = ctx; + + // 1) 校验图片文件 + const files = []; + for (const item of images) { + const rel = typeof item === 'string' ? item : item.path; + const abs = path.join(library.path, rel); + if (!fs.existsSync(abs)) throw new ValidationError(`图片不存在: ${rel}`); + files.push(abs); + } + + // 2) LLM 脚本 + task.step = 'script'; + task.stepLabel = '正在生成分镜脚本'; + task.progress = 5; + const script = await llmScript({ url, key, model, title, count: files.length, extraPrompt }); + logger.info(`🎬 AI 视频脚本生成完成(${script.length} 段)`); + + // 3) 逐段 TTS + const work = fs.mkdtempSync(path.join(os.tmpdir(), 'flypic-aiv-')); + try { + const segments = []; + for (let i = 0; i < files.length; i++) { + if (task.cancelled) throw new Error('已取消'); + task.step = 'tts'; + task.stepLabel = `正在朗读配音 ${i + 1}/${files.length}`; + task.progress = 10 + Math.round((i / files.length) * 40); + const seg = script[i]; + let ttsFile = ''; + let dur = 0; + if (seg.tts) { + try { + const t = await videotool.generateTts(seg.tts, { + voice, + outPath: path.join(work, `tts${i}.mp3`) + }); + ttsFile = t.file; + dur = t.seconds; + } catch (err) { + logger.warn(`⚠️ TTS 第${i + 1}段失败(降级为静音): ${err.message}`); + } + } + segments.push({ + imagePath: files[i], + subtitle: seg.subtitle || seg.tts || title, + ttsFile, + duration: Math.max(2.5, Math.min(12, dur || 4)) + }); + } + + // 4) 逐段合成 + const segFiles = []; + for (let i = 0; i < segments.length; i++) { + if (task.cancelled) throw new Error('已取消'); + task.step = 'compose'; + task.stepLabel = `正在合成画面 ${i + 1}/${segments.length}`; + task.progress = 50 + Math.round((i / segments.length) * 45); + const out = path.join(work, `seg${i}.mp4`); + await videotool.buildSegment(segments[i], out, { width, height }); + segFiles.push(out); + } + + // 5) 拼接 + task.step = 'merge'; + task.stepLabel = '正在合并视频'; + task.progress = 96; + const finalTmp = path.join(work, 'final.mp4'); + await videotool.concatSegments(segFiles, finalTmp); + + // 6) 落库:与第一张图同文件夹,文件名=标题 + const firstRel = typeof images[0] === 'string' ? images[0] : images[0].path; + const dirRel = path.dirname(firstRel) === '.' ? '' : path.dirname(firstRel); + const safeTitle = String(title).replace(/[\\/:*?"<>|]/g, '_').slice(0, 60) || 'AI视频'; + let target = path.join(library.path, dirRel, `${safeTitle}.mp4`); + let n = 1; + while (fs.existsSync(target)) { + target = path.join(library.path, dirRel, `${safeTitle}_${n}.mp4`); + n++; + } + fs.copyFileSync(finalTmp, target); + await reindex(req, library.path, target); + + task.status = 'done'; + task.progress = 100; + task.step = 'done'; + task.stepLabel = '完成'; + task.resultPath = path.relative(library.path, target); + logger.info(`🎬 AI 视频生成完成: ${task.resultPath}`); + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +} + +module.exports = router; diff --git a/backend/utils/scanner.js b/backend/utils/scanner.js index b875252..c617f50 100644 --- a/backend/utils/scanner.js +++ b/backend/utils/scanner.js @@ -319,8 +319,9 @@ async function processImage(imagePath, libraryPath, db, dryRun = false) { const actualHeight = thumbnails.height || metadata.height; // OCR 文字识别(可选:tesseract 失败时返回空字符串,不影响主流程) + // 仅图片做 OCR——视频/文档跑 tesseract 纯属浪费,线上识别还会把文件发给视觉模型 stepStart = Date.now(); - const ocrText = await extractText(imagePath); + const ocrText = fileType === 'image' ? await extractText(imagePath) : ''; stepTimes.ocr = Date.now() - stepStart; const imageData = { @@ -338,7 +339,8 @@ async function processImage(imagePath, libraryPath, db, dryRun = false) { thumbnail_path: thumbnails.thumbnail_path, thumbnail_size: thumbnails.thumbnail_size, ocr_text: ocrText, - ocr_done: 1 // 新/变化文件已执行过 OCR(extractText 失败返回空串,也算已尝试) + ocr_done: 1, // 新/变化文件已执行过 OCR(extractText 失败返回空串,也算已尝试) + duration: metadata.duration || 0 // 视频时长(秒),图片为 0 }; if (dryRun) { diff --git a/backend/utils/thumbnail.js b/backend/utils/thumbnail.js index 38a010a..a63e351 100644 --- a/backend/utils/thumbnail.js +++ b/backend/utils/thumbnail.js @@ -317,6 +317,27 @@ async function getImageMetadata(imagePath) { } } + // 视频文件:ffprobe 提取真实宽高与时长(供卡片时长角标与播放预览) + if (fileType === 'video') { + try { + const videotool = require('./videotool'); + const info = await videotool.probeVideo(imagePath); + if (info.width && info.height) { + return { + width: info.width, + height: info.height, + format: path.extname(imagePath).slice(1).toLowerCase(), + size: stats.size, + created_at: stats.birthtimeMs, + modified_at: stats.mtimeMs, + duration: info.duration || 0 + }; + } + } catch (_) { + // ffprobe 缺失或文件损坏 → 走占位信息 + } + } + // 对于非图片文件或 Sharp 失败的情况,返回基础信息 const ext = path.extname(imagePath).toLowerCase().slice(1); return { diff --git a/backend/utils/videotool.js b/backend/utils/videotool.js new file mode 100644 index 0000000..b236b21 --- /dev/null +++ b/backend/utils/videotool.js @@ -0,0 +1,434 @@ +/** + * 视频处理工具(ffmpeg / ffprobe 封装) + * - probeVideo ffprobe 提取宽高/时长/编码 + * - addWatermark 文字水印烧录(drawtext,textfile 避免转义地狱) + * - removeWatermark 区域去水印(delogo) + * - addSubtitleAudio 字幕烧录 + TTS/上传配音(音轨替换或混合) + * - stripSubtitleAudio 移除内嵌字幕流与音轨 + * - generateTts edge-tts 朗读(返回 mp3 路径与秒数) + */ +const { spawn } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const logger = require('../src/utils/logger'); + +const FFMPEG = process.env.FFMPEG_PATH || 'ffmpeg'; +const FFPROBE = process.env.FFPROBE_PATH || 'ffprobe'; + +/** 运行外部命令,超时强杀 */ +function run(cmd, args, { timeoutMs = 600 * 1000, onStderr, env } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'], env }); + let out = ''; + let err = ''; + const timer = setTimeout(() => { + child.kill('SIGKILL'); + reject(new Error(`${cmd} 超时 ${Math.round(timeoutMs / 1000)}s`)); + }, timeoutMs); + child.stdout.on('data', (d) => { out += d; }); + child.stderr.on('data', (d) => { + err += d; + if (onStderr) onStderr(String(d)); + }); + child.on('error', (e) => { clearTimeout(timer); reject(e); }); + child.on('close', (code) => { + clearTimeout(timer); + if (code === 0) resolve({ stdout: out, stderr: err }); + else { + // 剥掉 ffmpeg 版本/banner 行,只留真正的错误 + const clean = err.split('\n').filter((l) => + !/ffmpeg version|configuration:|^\s*lib(av|sw|post)|^\s*\d+\.\s+\d+\.\d+/.test(l) + ).join('\n').trim(); + reject(new Error(`${cmd} 退出码 ${code}: ${(clean || err).slice(-600)}`)); + } + }); + }); +} + +/** ffprobe:视频宽高/时长 */ +async function probeVideo(file) { + const { stdout } = await run(FFPROBE, [ + '-v', 'error', + '-select_streams', 'v:0', + '-show_entries', 'stream=width,height,codec_name:format=duration', + '-of', 'json', + file + ]); + const j = JSON.parse(stdout || '{}'); + const s = (j.streams && j.streams[0]) || {}; + const dur = parseFloat(j.format && j.format.duration); + return { + width: s.width || 0, + height: s.height || 0, + codec: s.codec_name || '', + duration: Number.isFinite(dur) ? dur : 0 + }; +} + +/** 位置预设 → drawtext x/y 表达式(相对画面,text_w/text_h 为文本尺寸) */ +const POSITIONS = { + 'top-left': { x: 'w*0.03', y: 'h*0.05' }, + 'top-center': { x: '(w-text_w)/2', y: 'h*0.05' }, + 'top-right': { x: 'w-text_w-w*0.03', y: 'h*0.05' }, + 'center': { x: '(w-text_w)/2', y: '(h-text_h)/2' }, + 'bottom-left': { x: 'w*0.03', y: 'h-text_h-h*0.06' }, + 'bottom-center':{ x: '(w-text_w)/2', y: 'h-text_h-h*0.06' }, + 'bottom-right': { x: 'w-text_w-w*0.03', y: 'h-text_h-h*0.06' } +}; + +/** 平铺九宫格 x/y 列表(每个单元一格) */ +function tilePositions(cols, rows) { + const list = []; + for (let r = 0; r < rows; r++) { + for (let c = 0; c < cols; c++) { + list.push({ + x: `w*${(c + 0.5) / cols}-text_w/2`, + y: `h*${(r + 0.5) / rows}-text_h/2` + }); + } + } + return list; +} + +/** + * 生成 drawtext filter 串 + * @param {object} opts 文本/位置/字号%/透明度/描边/旋转/平铺 + * @param {number} width 视频像素宽(换算字号) + * @param {number} height 视频像素高 + * @param {string} textFile 临时文本文件(UTF-8) + */ +function buildDrawText(opts, width, height, textFile) { + const { + position = 'bottom-right', + fontSizePct = 4, // 字号 = 视频高度的百分比 + opacity = 0.6, // 0~1 + color = 'white', + stroke = true, + rotate = 0, // 逆时针角度 + tile = false + } = opts; + + const fontSize = Math.max(10, Math.round((height * fontSizePct) / 100)); + const alpha = Math.max(0, Math.min(1, opacity)); + const colorHex = /^#?[0-9a-fA-F]{6}$/.test(String(color)) ? color : 'white'; + const fg = /^#/.test(colorHex) ? colorHex : colorHex; // drawtext 接受 #RRGGBB 与颜色名 + const fgColor = fg.startsWith('#') ? `0x${fg.slice(1)}@${alpha}` : `${fg}@${alpha}`; + + const positions = tile ? tilePositions(3, 3) : [POSITIONS[position] || POSITIONS['bottom-right']]; + + const parts = positions.map((p) => { + const args = [ + `fontfile=/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc`, + `textfile=${textFile}`, + `fontsize=${fontSize}`, + `fontcolor=${fgColor}`, + `x=${p.x}`, + `y=${p.y}` + ]; + if (stroke) { + args.push('borderw=2', 'bordercolor=black@0.55'); + } + if (rotate) args.push(`rotate=${(-rotate) * Math.PI / 180}:c=none`); + return `drawtext=${args.join(':')}`; + }); + return parts.join(','); +} + +/** 生成 SRT(多行文本按行数均分时长) */ +function textToSrt(text, durationSec) { + const lines = String(text).split('\n').map((s) => s.trim()).filter(Boolean); + if (!lines.length) throw new Error('字幕文本为空'); + const per = durationSec / lines.length; + const fmt = (sec) => { + const ms = Math.max(0, Math.round(sec * 1000)); + const h = String(Math.floor(ms / 3600000)).padStart(2, '0'); + const m = String(Math.floor((ms % 3600000) / 60000)).padStart(2, '0'); + const s = String(Math.floor((ms % 60000) / 1000)).padStart(2, '0'); + const mm = String(ms % 1000).padStart(3, '0'); + return `${h}:${m}:${s},${mm}`; + }; + return lines.map((line, i) => + `${i + 1}\n${fmt(i * per)} --> ${fmt((i + 1) * per)}\n${line}\n` + ).join('\n'); +} + +/** 临时工作目录 */ +function tmpDir() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'flypic-vid-')); +} + +/** + * 加水印(烧录) + * @param {string} input 输入视频 + * @param {string} output 输出视频 + * @param {object} opts { text, position, fontSizePct, opacity, color, stroke, rotate, tile } + */ +async function addWatermark(input, output, opts) { + const text = String(opts.text || '').trim(); + if (!text) throw new Error('水印文字不能为空'); + const meta = await probeVideo(input); + const dir = tmpDir(); + try { + const textFile = path.join(dir, 'wm.txt'); + fs.writeFileSync(textFile, text, 'utf8'); + const vf = buildDrawText(opts, meta.width || 1280, meta.height || 720, textFile); + await run(FFMPEG, [ + '-y', '-i', input, + '-vf', vf, + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '20', + '-c:a', 'copy', + '-movflags', '+faststart', + output + ]); + return meta; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/** + * 去水印(delogo 区域平滑填充) + * @param {object} region { x, y, w, h } 百分比(相对画面) + */ +async function removeWatermark(input, output, region) { + const meta = await probeVideo(input); + const W = meta.width || 1280; + const H = meta.height || 720; + const x = Math.max(0, Math.round((region.x / 100) * W)); + const y = Math.max(0, Math.round((region.y / 100) * H)); + const w = Math.min(W - x, Math.max(6, Math.round((region.w / 100) * W))); // delogo 要求 ≥6px + const h = Math.min(H - y, Math.max(6, Math.round((region.h / 100) * H))); + if (w < 6 || h < 6) throw new Error('去水印区域太小(至少约 10px)'); + await run(FFMPEG, [ + '-y', '-i', input, + '-vf', `delogo=x=${x}:y=${y}:w=${w}:h=${h}`, + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '20', + '-c:a', 'copy', + '-movflags', '+faststart', + output + ]); + return meta; +} + +/** edge-tts 朗读:文本 → mp3,返回 { file, seconds } */ +async function generateTts(text, { voice = 'zh-CN-XiaoxiaoNeural', outPath } = {}) { + const content = String(text || '').trim(); + if (!content) throw new Error('朗读文本为空'); + const target = outPath || path.join(tmpDir(), 'tts.mp3'); + const env = { ...process.env }; + // 容器内如需代理(edge-tts 直连失败时)可注入 HTTPS_PROXY,不设置则直连 + await run('edge-tts', [ + '--voice', voice, + '--text', content, + '--write-media', target + ], { timeoutMs: 120 * 1000, env }); + if (!fs.existsSync(target)) throw new Error('TTS 生成失败'); + const meta = await probeVideo(target).catch(() => ({ duration: 0 })); + // mp3 用 ffprobe format.duration 更准 + let seconds = meta.duration; + if (!seconds) { + const { stdout } = await run(FFPROBE, ['-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', target]).catch(() => ({ stdout: '0' })); + seconds = parseFloat(stdout) || 0; + } + return { file: target, seconds }; +} + +/** + * 添加字幕 + 声音 + * - 字幕:多行文本烧录(drawtext,按视频时长均分逐段出现)或整段常驻 + * - 声音:ttsText 走 edge-tts 朗读;audioFile 用上传音频 + * - audioMode: 'replace' 替换原声 | 'mix' 与原声混合 | 'keep' 只加字幕不动声音 + * + * 字幕实现:按配音时长切分视频分段,逐段烧录对应字幕后拼接——保证字幕与朗读同步。 + * 若无配音(audioMode keep 且无 tts),字幕整段常驻。 + */ +async function addSubtitleAudio(input, output, opts) { + const { + subtitleText = '', + ttsText = '', + voice = 'zh-CN-XiaoxiaoNeural', + audioFile = '', // 前端上传的音频(临时路径) + audioMode = 'tts', // 'tts' | 'file' | 'none' + audioVolume = 1.0, + originalVolume = 1.0, // 混合时原声相对音量 + tile = false, + fontSizePct = 5, + opacity = 0.95, + color = 'white' + } = opts; + + const text = String(subtitleText || ttsText || '').trim(); + if (!text) throw new Error('字幕文本不能为空'); + const meta = await probeVideo(input); + const duration = meta.duration; + if (!duration) throw new Error('无法读取视频时长'); + + const dir = tmpDir(); + try { + // 1) 配音 + let voiceFile = ''; + let voiceSec = 0; + if (audioMode === 'tts' && ttsText) { + const t = await generateTts(ttsText, { voice, outPath: path.join(dir, 'voice.mp3') }); + voiceFile = t.file; voiceSec = t.seconds; + } else if (audioMode === 'file' && audioFile && fs.existsSync(audioFile)) { + voiceFile = audioFile; + const { stdout } = await run(FFPROBE, ['-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', voiceFile]).catch(() => ({ stdout: '0' })); + voiceSec = parseFloat(stdout) || 0; + } + + const lines = text.split('\n').map((s) => s.trim()).filter(Boolean); + + // 2) 有配音 → 分段字幕与语音同步;无配音 → 整段常驻 + if (voiceFile && voiceSec > 0) { + const segCount = Math.max(lines.length, 1); + const segDur = voiceSec / segCount; + const segDir = path.join(dir, 'segs'); + fs.mkdirSync(segDir, { recursive: true }); + + const segFiles = []; + for (let i = 0; i < segCount; i++) { + const tFile = path.join(segDir, `s${i}.txt`); + fs.writeFileSync(tFile, lines[i] || lines[lines.length - 1], 'utf8'); + const vf = buildDrawText({ + position: 'bottom-center', fontSizePct, opacity, color, stroke: true, tile + }, meta.width || 1280, meta.height || 720, tFile); + const segOut = path.join(segDir, `seg${i}.mp4`); + const args = [ + '-y', + '-ss', (i * segDur).toFixed(3), '-t', segDur.toFixed(3), '-i', input, + '-ss', (i * segDur).toFixed(3), '-t', segDur.toFixed(3), '-i', voiceFile, + '-vf', vf, + '-map', '0:v', '-map', '1:a', + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '20' + ]; + if (audioVolume !== 1.0) args.push('-filter:a', `volume=${audioVolume}`); + args.push('-c:a', 'aac', '-ar', '44100', '-ac', '2', '-shortest', segOut); + await run(FFMPEG, args); + segFiles.push(segOut); + } + + // 拼接(编码参数一致,直接 concat) + const listFile = path.join(segDir, 'list.txt'); + fs.writeFileSync(listFile, segFiles.map((f) => `file '${f}'`).join('\n'), 'utf8'); + const concatArgs = ['-y', '-f', 'concat', '-safe', '0', '-i', listFile, '-c', 'copy', '-movflags', '+faststart', output]; + await run(FFMPEG, concatArgs).catch(async () => { + // 参数不一致兜底:重编码拼接 + await run(FFMPEG, ['-y', '-f', 'concat', '-safe', '0', '-i', listFile, + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '20', '-c:a', 'aac', + '-movflags', '+faststart', output]); + }); + return { ...meta, voiceSeconds: voiceSec }; + } + + // 无配音:字幕常驻 + 可选背景音 + const textFile = path.join(dir, 'sub.txt'); + fs.writeFileSync(textFile, text, 'utf8'); + const vf = buildDrawText({ + position: 'bottom-center', fontSizePct, opacity, color, stroke: true, tile + }, meta.width || 1280, meta.height || 720, textFile); + + const args = ['-y', '-i', input, '-vf', vf, '-map', '0:v']; + if (audioMode === 'file' && voiceFile) { + args.push('-i', voiceFile, '-map', '1:a', + '-filter:a', `volume=${audioVolume}`, + '-c:a', 'aac', '-shortest'); + } else { + args.push('-map', '0:a?'); + if (originalVolume !== 1.0) { + args.push('-filter:a', `volume=${originalVolume}`); + } + args.push('-c:a', 'copy'); + } + args.push('-c:v', 'libx264', '-preset', 'veryfast', '-crf', '20', + '-movflags', '+faststart', output); + await run(FFMPEG, args); + return meta; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/** 去掉字幕流与音轨(-sn -an,视频流直拷) */ +async function stripSubtitleAudio(input, output, { removeAudio = true, removeSubtitle = true } = {}) { + const args = ['-y', '-i', input, '-map', '0:v', '-c:v', 'copy']; + if (removeSubtitle) args.push('-sn'); + if (removeAudio) args.push('-an'); + else args.push('-map', '0:a?', '-c:a', 'copy'); + args.push('-movflags', '+faststart', output); + await run(FFMPEG, args); + return probeVideo(input); +} + +/** + * AI 生成视频素材装配(分段:图 + 该段字幕 + 该段配音) + * @param {object} seg { imagePath, subtitle, ttsFile, duration } duration=该段秒数 + */ +async function buildSegment(seg, outPath, { width = 1920, height = 1080, fontSizePct = 5 } = {}) { + const dir = tmpDir(); + try { + const tFile = path.join(dir, 'sub.txt'); + fs.writeFileSync(tFile, seg.subtitle || '', 'utf8'); + const textFile = tFile; + const fontSize = Math.max(12, Math.round((height * fontSizePct) / 100)); + const vf = [ + `scale=${width}:${height}:force_original_aspect_ratio=decrease`, + `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, + `drawtext=fontfile=/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc:textfile=${textFile}:fontsize=${fontSize}:fontcolor=white@0.95:borderw=2:bordercolor=black@0.6:x=(w-text_w)/2:y=h-text_h-h*0.07:line_spacing=8` + ].join(','); + + const args = [ + '-y', '-loop', '1', '-t', seg.duration.toFixed(3), '-i', seg.imagePath + ]; + if (seg.ttsFile) { + args.push('-i', seg.ttsFile, '-map', '0:v', '-map', '1:a', '-c:a', 'aac', '-ar', '44100', '-ac', '2', '-shortest'); + } else { + args.push('-f', 'lavfi', '-i', 'anullsrc=channel_layout=stereo:sample_rate=44100', '-map', '0:v', '-map', '1:a', '-shortest'); + } + args.push( + '-vf', vf, + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '20', + '-pix_fmt', 'yuv420p', + '-r', '30', + outPath + ); + await run(FFMPEG, args, { timeoutMs: 300 * 1000 }); + return outPath; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/** 拼接分段 mp4(先 copy,失败重编码) */ +async function concatSegments(segFiles, outPath) { + const dir = tmpDir(); + try { + const listFile = path.join(dir, 'list.txt'); + fs.writeFileSync(listFile, segFiles.map((f) => `file '${f}'`).join('\n'), 'utf8'); + try { + await run(FFMPEG, ['-y', '-f', 'concat', '-safe', '0', '-i', listFile, '-c', 'copy', '-movflags', '+faststart', outPath]); + } catch (_) { + await run(FFMPEG, ['-y', '-f', 'concat', '-safe', '0', '-i', listFile, + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '20', '-c:a', 'aac', + '-pix_fmt', 'yuv420p', '-movflags', '+faststart', outPath], + { timeoutMs: 600 * 1000 }); + } + return outPath; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +module.exports = { + run, + probeVideo, + addWatermark, + removeWatermark, + addSubtitleAudio, + stripSubtitleAudio, + generateTts, + buildSegment, + concatSegments, + textToSrt +}; diff --git a/frontend/package.json b/frontend/package.json index 480bf06..35cee16 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "flypic-frontend", - "version": "1.3.0", + "version": "1.4.0", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/components/AIVideoModal.jsx b/frontend/src/components/AIVideoModal.jsx new file mode 100644 index 0000000..fec6389 --- /dev/null +++ b/frontend/src/components/AIVideoModal.jsx @@ -0,0 +1,225 @@ +/** + * AI 生成视频弹窗 + * 多选图片 + 标题 + 模型 KEY → 一键生成带字幕和朗读的视频 + * 后端:POST /api/video/aigenerate(异步任务)+ GET /api/video/task/:id 轮询 + */ +import { useEffect, useRef, useState } from 'react'; +import { X, Loader2, Film, CheckCircle2, AlertCircle, KeyRound } from 'lucide-react'; +import { imageAPI } from '../api'; + +const VOICES = [ + { key: 'zh-CN-XiaoxiaoNeural', label: '晓晓(女声·自然)' }, + { key: 'zh-CN-YunxiNeural', label: '云希(男声·温暖)' }, + { key: 'zh-CN-YunjianNeural', label: '云健(男声·浑厚)' }, + { key: 'zh-CN-XiaoyiNeural', label: '晓伊(女声·活泼)' }, + { key: 'zh-CN-YunyangNeural', label: '云扬(男声·播音)' }, + { key: 'zh-TW-HsiaoChenNeural', label: '晓臻(台湾腔)' } +]; + +const RESOLUTIONS = [ + { key: '1920x1080', label: '横屏 1920×1080' }, + { key: '1080x1920', label: '竖屏 1080×1920' }, + { key: '1280x720', label: '横屏 1280×720' } +]; + +const STEPS = { + script: '正在生成分镜脚本', + tts: '正在朗读配音', + compose: '正在合成画面', + merge: '正在合并视频', + done: '完成' +}; + +function AIVideoModal({ libraryId, images, onClose, onDone }) { + const [title, setTitle] = useState(''); + const [apiKey, setApiKey] = useState(''); + const [showKey, setShowKey] = useState(false); + const [voice, setVoice] = useState(VOICES[0].key); + const [resolution, setResolution] = useState('1920x1080'); + const [extraPrompt, setExtraPrompt] = useState(''); + + const [taskId, setTaskId] = useState(''); + const [task, setTask] = useState(null); + const [error, setError] = useState(''); + const [starting, setStarting] = useState(false); + const timerRef = useRef(null); + + const list = Array.isArray(images) ? images : []; + + // 轮询任务进度 + useEffect(() => { + if (!taskId) return; + timerRef.current = setInterval(async () => { + try { + const res = await fetch(`/api/video/task/${taskId}`); + if (!res.ok) return; + const data = await res.json(); + setTask(data); + if (data.status === 'done' || data.status === 'error') { + clearInterval(timerRef.current); + if (data.status === 'done') { + setTimeout(() => { onDone && onDone(data.resultPath); onClose(); }, 1200); + } + } + } catch (_) { /* 网络抖动继续轮询 */ } + }, 1500); + return () => timerRef.current && clearInterval(timerRef.current); + }, [taskId, onDone, onClose]); + + const handleStart = async () => { + setError(''); + if (!title.trim()) { setError('请填写视频标题'); return; } + if (list.length < 2) { setError('请至少选择 2 张图片'); return; } + setStarting(true); + try { + const res = await fetch('/api/video/aigenerate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + libraryId, + images: list.map((img) => ({ path: img.path })), + title: title.trim(), + apiKey: apiKey.trim(), + voice, + resolution, + extraPrompt: extraPrompt.trim() + }) + }); + const data = await res.json().catch(() => ({})); + if (!res.ok || data.error) throw new Error(data.error || `启动失败 (${res.status})`); + setTaskId(data.taskId); + } catch (e) { + setError(e.message || '启动失败'); + } finally { + setStarting(false); + } + }; + + const running = !!taskId && task && task.status === 'running'; + const failed = task && task.status === 'error'; + const done = task && task.status === 'done'; + + const label = 'block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1'; + const input = 'w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 text-sm focus:ring-2 focus:ring-blue-500 focus:border-transparent'; + + return ( +
AI 将按此标题撰写分镜旁白,成片文件名也用它。
+{task.error}
} + {done && task.resultPath &&已保存:{task.resultPath}
} ++ 流程:AI 写分镜脚本 → 逐段朗读配音 → 图片逐段配字幕合成 → 拼接成片入库。图片越多耗时越长(约 10–60 秒/张)。 +
+ +提示:去水印通过平滑插值抹除区域,适合纯色/简单背景的台标与角标。
+ > + )} + + {/* ===== 加字幕声音 ===== */} + {action === 'subtitle-audio' && ( + <> +开启朗读时字幕逐段与配音同步显示,生成的新视频会替换原音轨。
+ > + )} + + {/* ===== 去字幕声音 ===== */} + {action === 'strip' && ( + <> +注:对已“烧录”进画面的字幕/水印无效(那是像素),烧录内容请用「去水印」。
+ > + )} + + {/* 输出方式 */} +