优化 拖入文件上传速度

This commit is contained in:
Chen committed 2025-12-07 17:33:38 +08:00
1 parent aeb403f3c0
commit a71230bce4
4 files changed
+134 -41

No files matched your search

+18 -18
View File
@@ -29,11 +29,9 @@ const upload = multer({
* conflictAction: 'skip' | 'replace' | 'rename'
*/
router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
console.log('📤 收到上传请求');
console.log(' libraryId:', req.body.libraryId);
console.log(' targetFolder:', req.body.targetFolder);
console.log(' conflictAction:', req.body.conflictAction);
console.log(' 文件数量:', req.files?.length);
const requestStartTime = Date.now();
const totalSize = req.files?.length > 0 ? req.files.reduce((sum, f) => sum + f.size, 0) : 0;
console.log(`\n📤 上传 ${req.files?.length} 个文件 (${(totalSize / 1024 / 1024).toFixed(2)}MB) 到 [${req.body.targetFolder || '根目录'}]`);
const { libraryId, targetFolder, conflictAction } = req.body;
const files = req.files;
@@ -151,7 +149,6 @@ router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
// 写入文件(同步,快速)
fs.writeFileSync(filePath, file.buffer);
console.log(`📥 已保存: ${filename} (${(file.size / 1024).toFixed(2)} KB)`);
// 计算相对路径(使用最终的文件名)
const relativePath = targetFolder
@@ -171,7 +168,7 @@ router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
filesToProcess.push({ filePath, filename });
}
} catch (error) {
console.error(`❌ 保存失败: ${file.originalname}`, error);
console.error(`❌ 保存失败: ${file.originalname} -`, error.message);
results.failed.push({
filename: file.originalname,
error: error.message
@@ -180,26 +177,30 @@ router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
}
// 第二阶段:后台异步处理图片(生成缩略图、提取元数据)
// 不阻塞响应,让前端可以立即刷新
if (filesToProcess.length > 0) {
setImmediate(async () => {
console.log(`🔄 开始后台处理 ${filesToProcess.length} 个图片...`);
const processStartTime = Date.now();
let processedCount = 0;
let failedCount = 0;
for (const { filePath, filename } of filesToProcess) {
try {
await processImage(filePath, libraryPath, db.db);
console.log(`✅ 已处理: ${filename}`);
await processImage(filePath, libraryPath, db);
processedCount++;
} catch (processError) {
console.error(`❌ 处理图片失败: ${filename}`, processError.message);
failedCount++;
console.error(`❌ 处理失败: ${filename} -`, processError.message);
}
}
console.log(`✅ 后台处理完成`);
const totalProcessTime = Date.now() - processStartTime;
console.log(`✅ 后台处理完成: ${processedCount}/${filesToProcess.length} (${totalProcessTime}ms)`);
});
}
// 更新文件夹图片计数
if (targetFolder && results.success.length > 0) {
try {
console.log(`📊 更新文件夹计数: ${targetFolder}`);
const updateStmt = db.db.prepare(`
UPDATE folders
SET image_count = (
@@ -214,14 +215,13 @@ router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
Date.now(),
targetFolder
);
console.log(`✅ 文件夹计数已更新`);
} catch (error) {
console.error('❌ 更新文件夹计数失败:', error);
console.error(' 错误详情:', error.stack);
console.error('❌ 更新文件夹计数失败:', error.message);
}
}
console.log(`📋 上传结果: 成功 ${results.success.length}, 失败 ${results.failed.length}, 冲突 ${results.conflicts.length}`);
const totalRequestTime = Date.now() - requestStartTime;
console.log(`✅ HTTP响应: 成功 ${results.success.length}, 失败 ${results.failed.length}, 冲突 ${results.conflicts.length} (${totalRequestTime}ms)\n`);
res.json({
success: true,
+39 -4
View File
@@ -237,6 +237,9 @@ async function getFolderStructure(libraryPath) {
* @param {boolean} dryRun - If true, return data instead of inserting into DB (for batch write)
*/
async function processImage(imagePath, libraryPath, db, dryRun = false) {
const processStartTime = Date.now();
const stepTimes = {}; // 记录每个步骤的耗时
try {
const relativePath = path.relative(libraryPath, imagePath);
const filename = path.basename(imagePath);
@@ -244,8 +247,13 @@ async function processImage(imagePath, libraryPath, db, dryRun = false) {
const folder = folderRaw === '.' ? '' : folderRaw.replace(/\\/g, '/');
// Check if image already exists in database
let stepStart = Date.now();
const existing = db.getImageByPath(relativePath.replace(/\\/g, '/'));
stepTimes.dbCheck = Date.now() - stepStart;
stepStart = Date.now();
const currentHash = calculateFileHash(imagePath);
stepTimes.hashCalc = Date.now() - stepStart;
// For unchanged files, check whether thumbnails need upgrade/regeneration
let needRegenThumbs = false;
@@ -270,17 +278,26 @@ async function processImage(imagePath, libraryPath, db, dryRun = false) {
// Skip only if unchanged and thumbnails are up-to-date
if (existing && existing.file_hash === currentHash && !needRegenThumbs) {
const totalTime = Date.now() - processStartTime;
// logger.info(`⏭️ 跳过 (未变化): ${filename} (${totalTime}ms)`);
return { status: 'skipped', path: relativePath };
}
// Get image metadata
stepStart = Date.now();
const metadata = await getImageMetadata(imagePath);
stepTimes.metadata = Date.now() - stepStart;
if (!metadata) {
const totalTime = Date.now() - processStartTime;
logger.warn(`❌ 元数据读取失败: ${filename} (${totalTime}ms)`);
return { status: 'error', path: relativePath, error: 'Failed to read metadata' };
}
// Generate thumbnails (also for unchanged files when thumbnails missing/outdated)
stepStart = Date.now();
const thumbnails = await generateImageThumbnails(imagePath, libraryPath);
stepTimes.thumbnail = Date.now() - stepStart;
const fileType = getFileType(imagePath);
// 使用缩略图的实际尺寸(对于视频/PSD,这是提取后的真实尺寸)
@@ -308,12 +325,30 @@ async function processImage(imagePath, libraryPath, db, dryRun = false) {
}
// Insert/update in database
stepStart = Date.now();
db.insertImage(imageData);
return { status: 'processed', path: relativePath };
stepTimes.dbInsert = Date.now() - stepStart;
const totalTime = Date.now() - processStartTime;
// 输出详细的性能日志
const logParts = [
`总计${totalTime}ms`,
`哈希${stepTimes.hashCalc}ms`,
`元数据${stepTimes.metadata}ms`,
`缩略图${stepTimes.thumbnail}ms`
];
// 只有当总耗时超过 500ms 时才输出警告
if (totalTime > 500) {
logger.warn(`⚠️ 处理较慢: ${filename} (${logParts.join(', ')})`);
}
return { status: 'processed', path: relativePath, timing: stepTimes, totalTime };
} catch (error) {
logger.error('处理图片失败:', path.basename(imagePath), error.message);
return { status: 'error', path: imagePath, error: error.message };
const totalTime = Date.now() - processStartTime;
logger.error(`❌ 处理图片失败: ${path.basename(imagePath)} (${totalTime}ms)`, error.message);
return { status: 'error', path: imagePath, error: error.message, totalTime };
}
}
+46 -2
View File
@@ -183,6 +183,9 @@ function getThumbnailConfig(originalWidth, originalHeight, targetHeight = 200) {
* 4. 不修改色彩(饱和度/亮度),保持原图风格
*/
async function generateThumbnail(inputPath, outputPath, targetHeight = 200) {
const startTime = Date.now();
const filename = path.basename(inputPath);
try {
// 先读取文件到 Buffer,避免 Sharp 锁定文件句柄
let inputBuffer = fs.readFileSync(inputPath);
@@ -220,6 +223,7 @@ async function generateThumbnail(inputPath, outputPath, targetHeight = 200) {
const effort = constants.THUMBNAIL_GENERATION.EFFORT;
// 生成缩略图(使用 buffer,不锁定原文件)
const processStart = Date.now();
await sharp(inputBuffer)
.rotate() // 按EXIF旋转
.resize(config.width, config.height, {
@@ -247,6 +251,7 @@ async function generateThumbnail(inputPath, outputPath, targetHeight = 200) {
alphaQuality: hasAlpha ? 100 : undefined
})
.toFile(outputPath);
const processTime = Date.now() - processStart;
// 显式释放 Buffer 内存
inputBuffer = null;
@@ -261,13 +266,21 @@ async function generateThumbnail(inputPath, outputPath, targetHeight = 200) {
const stats = fs.statSync(outputPath);
const finalSize = stats.size;
const totalTime = Date.now() - startTime;
// 只有当耗时超过 200ms 时才输出警告
if (totalTime > 200) {
console.log(`⚠️ Sharp处理较慢: ${filename} (总计${totalTime}ms, 处理${processTime}ms)`);
}
return {
width: config.width,
height: config.height,
size: finalSize,
quality: quality,
originalPixels: config.originalPixels,
path: outputPath
path: outputPath,
timing: { total: totalTime, process: processTime }
};
} catch (error) {
console.error('Error generating thumbnail:', error);
@@ -325,6 +338,10 @@ async function getImageMetadata(imagePath) {
* 使用 480px 高度(与 Billfish 一致)
*/
async function generateImageThumbnails(imagePath, libraryPath) {
const startTime = Date.now();
const filename = path.basename(imagePath);
const stepTimes = {};
const flypicDir = path.join(libraryPath, '.flypic');
const relativePath = path.relative(libraryPath, imagePath);
const hash = crypto.createHash('md5').update(relativePath).digest('hex');
@@ -339,45 +356,70 @@ async function generateImageThumbnails(imagePath, libraryPath) {
fs.mkdirSync(path.dirname(out480), { recursive: true });
let thumbnailResult;
let stepStart = Date.now();
// 根据文件类型生成不同的缩略图
const ext = path.extname(imagePath).slice(1).toUpperCase();
if (fileType === 'image' && canGenerateThumbnail(imagePath)) {
// 图片:使用 Sharp 生成真实缩略图
stepStart = Date.now();
thumbnailResult = await generateThumbnail(imagePath, out480, targetHeight);
stepTimes.generate = Date.now() - stepStart;
} else if (fileType === 'video') {
// 视频:尝试提取封面
stepStart = Date.now();
thumbnailResult = await extractVideoThumbnail(imagePath, out480);
stepTimes.videoExtract = Date.now() - stepStart;
// 如果提取失败,生成占位图
if (!thumbnailResult) {
stepStart = Date.now();
thumbnailResult = await generatePlaceholderThumbnail(out480, 'video', ext);
stepTimes.placeholder = Date.now() - stepStart;
}
} else if (fileType === 'design') {
// 设计文件:尝试提取嵌入缩略图(仅 PSD)
if (ext.toLowerCase() === 'psd') {
stepStart = Date.now();
thumbnailResult = await extractPSDThumbnail(imagePath, out480);
stepTimes.psdExtract = Date.now() - stepStart;
}
// 如果提取失败或不是 PSD,生成占位图
if (!thumbnailResult) {
stepStart = Date.now();
thumbnailResult = await generatePlaceholderThumbnail(out480, 'design', ext);
stepTimes.placeholder = Date.now() - stepStart;
}
} else {
// 其他类型(音频/文档/未知):生成占位图
stepStart = Date.now();
thumbnailResult = await generatePlaceholderThumbnail(out480, fileType, ext);
stepTimes.placeholder = Date.now() - stepStart;
}
// 返回相对于 libraryPath 的路径(包含 .flypic 前缀)
const thumbnailPath = path.relative(libraryPath, out480).replace(/\\/g, '/');
const totalTime = Date.now() - startTime;
// 只有当总耗时超过 300ms 时才输出警告
if (totalTime > 300) {
const details = Object.entries(stepTimes)
.map(([key, time]) => `${key}:${time}ms`)
.join(', ');
console.log(`⚠️ 缩略图较慢: ${filename} (总计${totalTime}ms, ${details})`);
}
return {
thumbnail_path: thumbnailPath,
thumbnail_size: thumbnailResult.size,
width: thumbnailResult.width,
height: thumbnailResult.height,
file_type: fileType
file_type: fileType,
timing: stepTimes,
totalTime
};
}
@@ -456,6 +498,7 @@ async function extractPSDThumbnail(psdPath, outputPath) {
const targetWidth = Math.round(targetHeight * aspectRatio);
// 简化处理策略
const processStart = Date.now();
await sharp(jpegData)
.resize(targetWidth, targetHeight, {
fit: 'inside',
@@ -469,6 +512,7 @@ async function extractPSDThumbnail(psdPath, outputPath) {
smartSubsample: false
})
.toFile(outputPath);
const processTime = Date.now() - processStart;
const stats = fs.statSync(outputPath);
return {
+31 -17
View File
@@ -98,7 +98,8 @@ export const useImageUpload = () => {
const files = Array.from(e.dataTransfer.files);
if (files.length === 0) return;
logger.file(`准备上传 ${files.length} 个文件到: ${selectedFolder || '根目录'}`);
const uploadStartTime = Date.now();
logger.file(`📤 开始上传 ${files.length} 个文件到 [${selectedFolder || '根目录'}]`);
// 开始上传
setUploadProgress({ isUploading: true, percent: 0, current: 0, total: files.length });
@@ -115,24 +116,30 @@ export const useImageUpload = () => {
);
// 上传完成
setUploadProgress({ isUploading: false, percent: 100, current: result.success.length, total: files.length });
const successList = result.data?.success || [];
setUploadProgress({ isUploading: false, percent: 100, current: successList.length, total: files.length });
// 检查是否有冲突
const conflicts = result.conflicts || [];
const conflicts = result.data?.conflicts || [];
if (conflicts.length > 0 && onConflict) {
// 有冲突,回调处理
logger.file(`检测到 ${conflicts.length} 个文件冲突`);
logger.file(`⚠️ 检测到 ${conflicts.length} 个文件冲突,等待处理...`);
onConflict(conflicts, files, selectedFolder || '');
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
return { hasConflicts: true, conflicts };
}
// 成功后刷新
const successCount = result.success?.length || 0;
const failedCount = result.failed?.length || 0;
const successCount = result.data?.success?.length || 0;
const failedCount = result.data?.failed?.length || 0;
if (failedCount > 0) {
logger.file(`📊 上传结果: 成功 ${successCount}, 失败 ${failedCount}`);
}
if (successCount > 0) {
// 立即刷新当前文件夹的图片列表
// 延迟150ms后刷新,等待后台缩略图生成完成
await new Promise(resolve => setTimeout(resolve, 150));
const params = { folder: selectedFolder || '' };
const response = await imageAPI.search(currentLibraryId, params);
setImages(response.images);
@@ -146,7 +153,6 @@ export const useImageUpload = () => {
try {
const response2 = await imageAPI.search(currentLibraryId, params);
setImages(response2.images);
logger.data('缩略图刷新完成');
} catch (err) {
logger.error('刷新缩略图失败:', err);
}
@@ -156,6 +162,9 @@ export const useImageUpload = () => {
setTimeout(() => {
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
}, 3000);
const totalTime = Date.now() - uploadStartTime;
logger.file(`✅ 上传完成 (${totalTime}ms)`);
return {
success: true,
@@ -164,7 +173,7 @@ export const useImageUpload = () => {
hasConflicts: false
};
} catch (error) {
logger.error('上传失败:', error);
logger.error('❌ 上传失败:', error.message);
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
throw error;
}
@@ -176,10 +185,10 @@ export const useImageUpload = () => {
const uploadWithConflictAction = useCallback(async (files, targetFolder, conflictAction) => {
if (!currentLibraryId) return;
logger.file(`处理上传冲突: ${conflictAction}`);
const uploadStartTime = Date.now();
logger.file(`📤 处理冲突后重新上传 (${conflictAction})`);
if (conflictAction === 'skip') {
logger.file('用户选择跳过上传');
return { success: true, skipped: true };
}
@@ -199,18 +208,21 @@ export const useImageUpload = () => {
);
// 上传完成
const successList = result.data?.success || [];
setUploadProgress({
isUploading: false,
percent: 100,
current: result.success.length,
current: successList.length,
total: files.length
});
const successCount = result.success?.length || 0;
const failedCount = result.failed?.length || 0;
const successCount = result.data?.success?.length || 0;
const failedCount = result.data?.failed?.length || 0;
// 刷新图片列表
if (successCount > 0) {
await new Promise(resolve => setTimeout(resolve, 150));
const params = { folder: selectedFolder || '' };
const response = await imageAPI.search(currentLibraryId, params);
setImages(response.images);
@@ -223,7 +235,6 @@ export const useImageUpload = () => {
try {
const response2 = await imageAPI.search(currentLibraryId, params);
setImages(response2.images);
logger.data('缩略图刷新完成');
} catch (err) {
logger.error('刷新缩略图失败:', err);
}
@@ -233,10 +244,13 @@ export const useImageUpload = () => {
setTimeout(() => {
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
}, 3000);
const totalTime = Date.now() - uploadStartTime;
logger.file(`✅ 上传完成 (${totalTime}ms)`);
return { success: true, successCount, failedCount };
} catch (error) {
logger.error('上传失败:', error);
logger.error('❌ 上传失败:', error.message);
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
throw error;
}