优化 拖入文件上传速度
This commit is contained in:
1 parent
aeb403f3c0
commit
a71230bce4
4 files changed
+134
-41
No files matched your search
@@ -29,11 +29,9 @@ const upload = multer({
|
||||
* conflictAction: 'skip' | 'replace' | 'rename'
|
||||
*/
|
||||
router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
|
||||
console.log('📤 收到上传请求');
|
||||
console.log(' libraryId:', req.body.libraryId);
|
||||
console.log(' targetFolder:', req.body.targetFolder);
|
||||
console.log(' conflictAction:', req.body.conflictAction);
|
||||
console.log(' 文件数量:', req.files?.length);
|
||||
const requestStartTime = Date.now();
|
||||
const totalSize = req.files?.length > 0 ? req.files.reduce((sum, f) => sum + f.size, 0) : 0;
|
||||
console.log(`\n📤 上传 ${req.files?.length} 个文件 (${(totalSize / 1024 / 1024).toFixed(2)}MB) 到 [${req.body.targetFolder || '根目录'}]`);
|
||||
|
||||
const { libraryId, targetFolder, conflictAction } = req.body;
|
||||
const files = req.files;
|
||||
@@ -151,7 +149,6 @@ router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
|
||||
|
||||
// 写入文件(同步,快速)
|
||||
fs.writeFileSync(filePath, file.buffer);
|
||||
console.log(`📥 已保存: ${filename} (${(file.size / 1024).toFixed(2)} KB)`);
|
||||
|
||||
// 计算相对路径(使用最终的文件名)
|
||||
const relativePath = targetFolder
|
||||
@@ -171,7 +168,7 @@ router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
|
||||
filesToProcess.push({ filePath, filename });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`❌ 保存失败: ${file.originalname}`, error);
|
||||
console.error(`❌ 保存失败: ${file.originalname} -`, error.message);
|
||||
results.failed.push({
|
||||
filename: file.originalname,
|
||||
error: error.message
|
||||
@@ -180,26 +177,30 @@ router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
|
||||
}
|
||||
|
||||
// 第二阶段:后台异步处理图片(生成缩略图、提取元数据)
|
||||
// 不阻塞响应,让前端可以立即刷新
|
||||
if (filesToProcess.length > 0) {
|
||||
setImmediate(async () => {
|
||||
console.log(`🔄 开始后台处理 ${filesToProcess.length} 个图片...`);
|
||||
const processStartTime = Date.now();
|
||||
let processedCount = 0;
|
||||
let failedCount = 0;
|
||||
|
||||
for (const { filePath, filename } of filesToProcess) {
|
||||
try {
|
||||
await processImage(filePath, libraryPath, db.db);
|
||||
console.log(`✅ 已处理: ${filename}`);
|
||||
await processImage(filePath, libraryPath, db);
|
||||
processedCount++;
|
||||
} catch (processError) {
|
||||
console.error(`❌ 处理图片失败: ${filename}`, processError.message);
|
||||
failedCount++;
|
||||
console.error(`❌ 处理失败: ${filename} -`, processError.message);
|
||||
}
|
||||
}
|
||||
console.log(`✅ 后台处理完成`);
|
||||
|
||||
const totalProcessTime = Date.now() - processStartTime;
|
||||
console.log(`✅ 后台处理完成: ${processedCount}/${filesToProcess.length} (${totalProcessTime}ms)`);
|
||||
});
|
||||
}
|
||||
|
||||
// 更新文件夹图片计数
|
||||
if (targetFolder && results.success.length > 0) {
|
||||
try {
|
||||
console.log(`📊 更新文件夹计数: ${targetFolder}`);
|
||||
const updateStmt = db.db.prepare(`
|
||||
UPDATE folders
|
||||
SET image_count = (
|
||||
@@ -214,14 +215,13 @@ router.post('/', upload.array('files', 50), asyncHandler(async (req, res) => {
|
||||
Date.now(),
|
||||
targetFolder
|
||||
);
|
||||
console.log(`✅ 文件夹计数已更新`);
|
||||
} catch (error) {
|
||||
console.error('❌ 更新文件夹计数失败:', error);
|
||||
console.error(' 错误详情:', error.stack);
|
||||
console.error('❌ 更新文件夹计数失败:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`📋 上传结果: 成功 ${results.success.length}, 失败 ${results.failed.length}, 冲突 ${results.conflicts.length}`);
|
||||
const totalRequestTime = Date.now() - requestStartTime;
|
||||
console.log(`✅ HTTP响应: 成功 ${results.success.length}, 失败 ${results.failed.length}, 冲突 ${results.conflicts.length} (${totalRequestTime}ms)\n`);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
|
||||
@@ -237,6 +237,9 @@ async function getFolderStructure(libraryPath) {
|
||||
* @param {boolean} dryRun - If true, return data instead of inserting into DB (for batch write)
|
||||
*/
|
||||
async function processImage(imagePath, libraryPath, db, dryRun = false) {
|
||||
const processStartTime = Date.now();
|
||||
const stepTimes = {}; // 记录每个步骤的耗时
|
||||
|
||||
try {
|
||||
const relativePath = path.relative(libraryPath, imagePath);
|
||||
const filename = path.basename(imagePath);
|
||||
@@ -244,8 +247,13 @@ async function processImage(imagePath, libraryPath, db, dryRun = false) {
|
||||
const folder = folderRaw === '.' ? '' : folderRaw.replace(/\\/g, '/');
|
||||
|
||||
// Check if image already exists in database
|
||||
let stepStart = Date.now();
|
||||
const existing = db.getImageByPath(relativePath.replace(/\\/g, '/'));
|
||||
stepTimes.dbCheck = Date.now() - stepStart;
|
||||
|
||||
stepStart = Date.now();
|
||||
const currentHash = calculateFileHash(imagePath);
|
||||
stepTimes.hashCalc = Date.now() - stepStart;
|
||||
|
||||
// For unchanged files, check whether thumbnails need upgrade/regeneration
|
||||
let needRegenThumbs = false;
|
||||
@@ -270,17 +278,26 @@ async function processImage(imagePath, libraryPath, db, dryRun = false) {
|
||||
|
||||
// Skip only if unchanged and thumbnails are up-to-date
|
||||
if (existing && existing.file_hash === currentHash && !needRegenThumbs) {
|
||||
const totalTime = Date.now() - processStartTime;
|
||||
// logger.info(`⏭️ 跳过 (未变化): ${filename} (${totalTime}ms)`);
|
||||
return { status: 'skipped', path: relativePath };
|
||||
}
|
||||
|
||||
// Get image metadata
|
||||
stepStart = Date.now();
|
||||
const metadata = await getImageMetadata(imagePath);
|
||||
stepTimes.metadata = Date.now() - stepStart;
|
||||
if (!metadata) {
|
||||
const totalTime = Date.now() - processStartTime;
|
||||
logger.warn(`❌ 元数据读取失败: ${filename} (${totalTime}ms)`);
|
||||
return { status: 'error', path: relativePath, error: 'Failed to read metadata' };
|
||||
}
|
||||
|
||||
// Generate thumbnails (also for unchanged files when thumbnails missing/outdated)
|
||||
stepStart = Date.now();
|
||||
const thumbnails = await generateImageThumbnails(imagePath, libraryPath);
|
||||
stepTimes.thumbnail = Date.now() - stepStart;
|
||||
|
||||
const fileType = getFileType(imagePath);
|
||||
|
||||
// 使用缩略图的实际尺寸(对于视频/PSD,这是提取后的真实尺寸)
|
||||
@@ -308,12 +325,30 @@ async function processImage(imagePath, libraryPath, db, dryRun = false) {
|
||||
}
|
||||
|
||||
// Insert/update in database
|
||||
stepStart = Date.now();
|
||||
db.insertImage(imageData);
|
||||
|
||||
return { status: 'processed', path: relativePath };
|
||||
stepTimes.dbInsert = Date.now() - stepStart;
|
||||
|
||||
const totalTime = Date.now() - processStartTime;
|
||||
|
||||
// 输出详细的性能日志
|
||||
const logParts = [
|
||||
`总计${totalTime}ms`,
|
||||
`哈希${stepTimes.hashCalc}ms`,
|
||||
`元数据${stepTimes.metadata}ms`,
|
||||
`缩略图${stepTimes.thumbnail}ms`
|
||||
];
|
||||
|
||||
// 只有当总耗时超过 500ms 时才输出警告
|
||||
if (totalTime > 500) {
|
||||
logger.warn(`⚠️ 处理较慢: ${filename} (${logParts.join(', ')})`);
|
||||
}
|
||||
|
||||
return { status: 'processed', path: relativePath, timing: stepTimes, totalTime };
|
||||
} catch (error) {
|
||||
logger.error('处理图片失败:', path.basename(imagePath), error.message);
|
||||
return { status: 'error', path: imagePath, error: error.message };
|
||||
const totalTime = Date.now() - processStartTime;
|
||||
logger.error(`❌ 处理图片失败: ${path.basename(imagePath)} (${totalTime}ms)`, error.message);
|
||||
return { status: 'error', path: imagePath, error: error.message, totalTime };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -183,6 +183,9 @@ function getThumbnailConfig(originalWidth, originalHeight, targetHeight = 200) {
|
||||
* 4. 不修改色彩(饱和度/亮度),保持原图风格
|
||||
*/
|
||||
async function generateThumbnail(inputPath, outputPath, targetHeight = 200) {
|
||||
const startTime = Date.now();
|
||||
const filename = path.basename(inputPath);
|
||||
|
||||
try {
|
||||
// 先读取文件到 Buffer,避免 Sharp 锁定文件句柄
|
||||
let inputBuffer = fs.readFileSync(inputPath);
|
||||
@@ -220,6 +223,7 @@ async function generateThumbnail(inputPath, outputPath, targetHeight = 200) {
|
||||
const effort = constants.THUMBNAIL_GENERATION.EFFORT;
|
||||
|
||||
// 生成缩略图(使用 buffer,不锁定原文件)
|
||||
const processStart = Date.now();
|
||||
await sharp(inputBuffer)
|
||||
.rotate() // 按EXIF旋转
|
||||
.resize(config.width, config.height, {
|
||||
@@ -247,6 +251,7 @@ async function generateThumbnail(inputPath, outputPath, targetHeight = 200) {
|
||||
alphaQuality: hasAlpha ? 100 : undefined
|
||||
})
|
||||
.toFile(outputPath);
|
||||
const processTime = Date.now() - processStart;
|
||||
|
||||
// 显式释放 Buffer 内存
|
||||
inputBuffer = null;
|
||||
@@ -261,13 +266,21 @@ async function generateThumbnail(inputPath, outputPath, targetHeight = 200) {
|
||||
const stats = fs.statSync(outputPath);
|
||||
const finalSize = stats.size;
|
||||
|
||||
const totalTime = Date.now() - startTime;
|
||||
|
||||
// 只有当耗时超过 200ms 时才输出警告
|
||||
if (totalTime > 200) {
|
||||
console.log(`⚠️ Sharp处理较慢: ${filename} (总计${totalTime}ms, 处理${processTime}ms)`);
|
||||
}
|
||||
|
||||
return {
|
||||
width: config.width,
|
||||
height: config.height,
|
||||
size: finalSize,
|
||||
quality: quality,
|
||||
originalPixels: config.originalPixels,
|
||||
path: outputPath
|
||||
path: outputPath,
|
||||
timing: { total: totalTime, process: processTime }
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error generating thumbnail:', error);
|
||||
@@ -325,6 +338,10 @@ async function getImageMetadata(imagePath) {
|
||||
* 使用 480px 高度(与 Billfish 一致)
|
||||
*/
|
||||
async function generateImageThumbnails(imagePath, libraryPath) {
|
||||
const startTime = Date.now();
|
||||
const filename = path.basename(imagePath);
|
||||
const stepTimes = {};
|
||||
|
||||
const flypicDir = path.join(libraryPath, '.flypic');
|
||||
const relativePath = path.relative(libraryPath, imagePath);
|
||||
const hash = crypto.createHash('md5').update(relativePath).digest('hex');
|
||||
@@ -339,45 +356,70 @@ async function generateImageThumbnails(imagePath, libraryPath) {
|
||||
fs.mkdirSync(path.dirname(out480), { recursive: true });
|
||||
|
||||
let thumbnailResult;
|
||||
let stepStart = Date.now();
|
||||
|
||||
// 根据文件类型生成不同的缩略图
|
||||
const ext = path.extname(imagePath).slice(1).toUpperCase();
|
||||
|
||||
if (fileType === 'image' && canGenerateThumbnail(imagePath)) {
|
||||
// 图片:使用 Sharp 生成真实缩略图
|
||||
stepStart = Date.now();
|
||||
thumbnailResult = await generateThumbnail(imagePath, out480, targetHeight);
|
||||
stepTimes.generate = Date.now() - stepStart;
|
||||
} else if (fileType === 'video') {
|
||||
// 视频:尝试提取封面
|
||||
stepStart = Date.now();
|
||||
thumbnailResult = await extractVideoThumbnail(imagePath, out480);
|
||||
stepTimes.videoExtract = Date.now() - stepStart;
|
||||
|
||||
// 如果提取失败,生成占位图
|
||||
if (!thumbnailResult) {
|
||||
stepStart = Date.now();
|
||||
thumbnailResult = await generatePlaceholderThumbnail(out480, 'video', ext);
|
||||
stepTimes.placeholder = Date.now() - stepStart;
|
||||
}
|
||||
} else if (fileType === 'design') {
|
||||
// 设计文件:尝试提取嵌入缩略图(仅 PSD)
|
||||
if (ext.toLowerCase() === 'psd') {
|
||||
stepStart = Date.now();
|
||||
thumbnailResult = await extractPSDThumbnail(imagePath, out480);
|
||||
stepTimes.psdExtract = Date.now() - stepStart;
|
||||
}
|
||||
|
||||
// 如果提取失败或不是 PSD,生成占位图
|
||||
if (!thumbnailResult) {
|
||||
stepStart = Date.now();
|
||||
thumbnailResult = await generatePlaceholderThumbnail(out480, 'design', ext);
|
||||
stepTimes.placeholder = Date.now() - stepStart;
|
||||
}
|
||||
} else {
|
||||
// 其他类型(音频/文档/未知):生成占位图
|
||||
stepStart = Date.now();
|
||||
thumbnailResult = await generatePlaceholderThumbnail(out480, fileType, ext);
|
||||
stepTimes.placeholder = Date.now() - stepStart;
|
||||
}
|
||||
|
||||
// 返回相对于 libraryPath 的路径(包含 .flypic 前缀)
|
||||
const thumbnailPath = path.relative(libraryPath, out480).replace(/\\/g, '/');
|
||||
|
||||
const totalTime = Date.now() - startTime;
|
||||
|
||||
// 只有当总耗时超过 300ms 时才输出警告
|
||||
if (totalTime > 300) {
|
||||
const details = Object.entries(stepTimes)
|
||||
.map(([key, time]) => `${key}:${time}ms`)
|
||||
.join(', ');
|
||||
console.log(`⚠️ 缩略图较慢: ${filename} (总计${totalTime}ms, ${details})`);
|
||||
}
|
||||
|
||||
return {
|
||||
thumbnail_path: thumbnailPath,
|
||||
thumbnail_size: thumbnailResult.size,
|
||||
width: thumbnailResult.width,
|
||||
height: thumbnailResult.height,
|
||||
file_type: fileType
|
||||
file_type: fileType,
|
||||
timing: stepTimes,
|
||||
totalTime
|
||||
};
|
||||
}
|
||||
|
||||
@@ -456,6 +498,7 @@ async function extractPSDThumbnail(psdPath, outputPath) {
|
||||
const targetWidth = Math.round(targetHeight * aspectRatio);
|
||||
|
||||
// 简化处理策略
|
||||
const processStart = Date.now();
|
||||
await sharp(jpegData)
|
||||
.resize(targetWidth, targetHeight, {
|
||||
fit: 'inside',
|
||||
@@ -469,6 +512,7 @@ async function extractPSDThumbnail(psdPath, outputPath) {
|
||||
smartSubsample: false
|
||||
})
|
||||
.toFile(outputPath);
|
||||
const processTime = Date.now() - processStart;
|
||||
|
||||
const stats = fs.statSync(outputPath);
|
||||
return {
|
||||
|
||||
@@ -98,7 +98,8 @@ export const useImageUpload = () => {
|
||||
const files = Array.from(e.dataTransfer.files);
|
||||
if (files.length === 0) return;
|
||||
|
||||
logger.file(`准备上传 ${files.length} 个文件到: ${selectedFolder || '根目录'}`);
|
||||
const uploadStartTime = Date.now();
|
||||
logger.file(`📤 开始上传 ${files.length} 个文件到 [${selectedFolder || '根目录'}]`);
|
||||
|
||||
// 开始上传
|
||||
setUploadProgress({ isUploading: true, percent: 0, current: 0, total: files.length });
|
||||
@@ -115,24 +116,30 @@ export const useImageUpload = () => {
|
||||
);
|
||||
|
||||
// 上传完成
|
||||
setUploadProgress({ isUploading: false, percent: 100, current: result.success.length, total: files.length });
|
||||
const successList = result.data?.success || [];
|
||||
setUploadProgress({ isUploading: false, percent: 100, current: successList.length, total: files.length });
|
||||
|
||||
// 检查是否有冲突
|
||||
const conflicts = result.conflicts || [];
|
||||
const conflicts = result.data?.conflicts || [];
|
||||
if (conflicts.length > 0 && onConflict) {
|
||||
// 有冲突,回调处理
|
||||
logger.file(`检测到 ${conflicts.length} 个文件冲突`);
|
||||
logger.file(`⚠️ 检测到 ${conflicts.length} 个文件冲突,等待处理...`);
|
||||
onConflict(conflicts, files, selectedFolder || '');
|
||||
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
|
||||
return { hasConflicts: true, conflicts };
|
||||
}
|
||||
|
||||
// 成功后刷新
|
||||
const successCount = result.success?.length || 0;
|
||||
const failedCount = result.failed?.length || 0;
|
||||
const successCount = result.data?.success?.length || 0;
|
||||
const failedCount = result.data?.failed?.length || 0;
|
||||
|
||||
if (failedCount > 0) {
|
||||
logger.file(`📊 上传结果: 成功 ${successCount}, 失败 ${failedCount}`);
|
||||
}
|
||||
|
||||
if (successCount > 0) {
|
||||
// 立即刷新当前文件夹的图片列表
|
||||
// 延迟150ms后刷新,等待后台缩略图生成完成
|
||||
await new Promise(resolve => setTimeout(resolve, 150));
|
||||
|
||||
const params = { folder: selectedFolder || '' };
|
||||
const response = await imageAPI.search(currentLibraryId, params);
|
||||
setImages(response.images);
|
||||
@@ -146,7 +153,6 @@ export const useImageUpload = () => {
|
||||
try {
|
||||
const response2 = await imageAPI.search(currentLibraryId, params);
|
||||
setImages(response2.images);
|
||||
logger.data('缩略图刷新完成');
|
||||
} catch (err) {
|
||||
logger.error('刷新缩略图失败:', err);
|
||||
}
|
||||
@@ -156,6 +162,9 @@ export const useImageUpload = () => {
|
||||
setTimeout(() => {
|
||||
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
|
||||
}, 3000);
|
||||
|
||||
const totalTime = Date.now() - uploadStartTime;
|
||||
logger.file(`✅ 上传完成 (${totalTime}ms)`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
@@ -164,7 +173,7 @@ export const useImageUpload = () => {
|
||||
hasConflicts: false
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('上传失败:', error);
|
||||
logger.error('❌ 上传失败:', error.message);
|
||||
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
|
||||
throw error;
|
||||
}
|
||||
@@ -176,10 +185,10 @@ export const useImageUpload = () => {
|
||||
const uploadWithConflictAction = useCallback(async (files, targetFolder, conflictAction) => {
|
||||
if (!currentLibraryId) return;
|
||||
|
||||
logger.file(`处理上传冲突: ${conflictAction}`);
|
||||
const uploadStartTime = Date.now();
|
||||
logger.file(`📤 处理冲突后重新上传 (${conflictAction})`);
|
||||
|
||||
if (conflictAction === 'skip') {
|
||||
logger.file('用户选择跳过上传');
|
||||
return { success: true, skipped: true };
|
||||
}
|
||||
|
||||
@@ -199,18 +208,21 @@ export const useImageUpload = () => {
|
||||
);
|
||||
|
||||
// 上传完成
|
||||
const successList = result.data?.success || [];
|
||||
setUploadProgress({
|
||||
isUploading: false,
|
||||
percent: 100,
|
||||
current: result.success.length,
|
||||
current: successList.length,
|
||||
total: files.length
|
||||
});
|
||||
|
||||
const successCount = result.success?.length || 0;
|
||||
const failedCount = result.failed?.length || 0;
|
||||
const successCount = result.data?.success?.length || 0;
|
||||
const failedCount = result.data?.failed?.length || 0;
|
||||
|
||||
// 刷新图片列表
|
||||
if (successCount > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, 150));
|
||||
|
||||
const params = { folder: selectedFolder || '' };
|
||||
const response = await imageAPI.search(currentLibraryId, params);
|
||||
setImages(response.images);
|
||||
@@ -223,7 +235,6 @@ export const useImageUpload = () => {
|
||||
try {
|
||||
const response2 = await imageAPI.search(currentLibraryId, params);
|
||||
setImages(response2.images);
|
||||
logger.data('缩略图刷新完成');
|
||||
} catch (err) {
|
||||
logger.error('刷新缩略图失败:', err);
|
||||
}
|
||||
@@ -233,10 +244,13 @@ export const useImageUpload = () => {
|
||||
setTimeout(() => {
|
||||
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
|
||||
}, 3000);
|
||||
|
||||
const totalTime = Date.now() - uploadStartTime;
|
||||
logger.file(`✅ 上传完成 (${totalTime}ms)`);
|
||||
|
||||
return { success: true, successCount, failedCount };
|
||||
} catch (error) {
|
||||
logger.error('上传失败:', error);
|
||||
logger.error('❌ 上传失败:', error.message);
|
||||
setUploadProgress({ isUploading: false, percent: 0, current: 0, total: 0 });
|
||||
throw error;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user