664 lines
20 KiB
JavaScript
664 lines
20 KiB
JavaScript
const axios = require('axios');
|
|
const fs = require('fs-extra');
|
|
const path = require('path');
|
|
const crypto = require('crypto');
|
|
const ConfigManager = require('../config/config-manager');
|
|
const CacheConfigManager = require('../config/cache-config');
|
|
const FileUtils = require('../utils/file-utils');
|
|
const ErrorHandler = require('../utils/error-handler');
|
|
const { defaultLogger } = require('../utils/logger');
|
|
|
|
// 创建logger实例
|
|
const logger = defaultLogger.child('FileManager');
|
|
|
|
/**
|
|
* 文件管理器 - 负责文件下载、检查和目录管理
|
|
*/
|
|
class FileManager {
|
|
constructor() {
|
|
this.configManager = new ConfigManager();
|
|
this.cacheConfigManager = new CacheConfigManager();
|
|
|
|
// 默认下载配置（作为后备）
|
|
this.defaultDownloadConfig = {
|
|
timeout: 300000, // 5分钟超时
|
|
chunkSize: 1024 * 1024, // 1MB块大小
|
|
retryAttempts: 3, // 重试次数
|
|
retryDelay: 2000, // 重试延迟
|
|
concurrentDownloads: 3 // 并发下载数
|
|
};
|
|
|
|
// 初始化时设置线程池大小
|
|
this.initializeThreadPool();
|
|
}
|
|
|
|
/**
|
|
* 初始化线程池大小
|
|
*/
|
|
async initializeThreadPool() {
|
|
try {
|
|
const config = await this.getDownloadConfig();
|
|
if (config.threadPoolSize && !process.env.UV_THREADPOOL_SIZE) {
|
|
process.env.UV_THREADPOOL_SIZE = config.threadPoolSize.toString();
|
|
logger.info(`设置线程池大小为: ${config.threadPoolSize}`);
|
|
}
|
|
} catch (error) {
|
|
logger.warn('初始化线程池大小失败，使用默认值:', error.message);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 获取下载配置
|
|
*/
|
|
async getDownloadConfig() {
|
|
try {
|
|
const cacheConfig = await this.cacheConfigManager.loadConfig();
|
|
return {
|
|
timeout: cacheConfig.download?.downloadTimeout || this.defaultDownloadConfig.timeout,
|
|
chunkSize: cacheConfig.download?.chunkSize || this.defaultDownloadConfig.chunkSize,
|
|
retryAttempts: cacheConfig.download?.retryAttempts || this.defaultDownloadConfig.retryAttempts,
|
|
retryDelay: cacheConfig.download?.retryDelay || this.defaultDownloadConfig.retryDelay,
|
|
concurrentDownloads: cacheConfig.download?.concurrentDownloads || this.defaultDownloadConfig.concurrentDownloads,
|
|
maxConcurrentFiles: cacheConfig.download?.maxConcurrentFiles || 5,
|
|
threadPoolSize: cacheConfig.download?.threadPoolSize || 16,
|
|
maxFileSize: cacheConfig.download?.maxFileSize || 50 * 1024 * 1024,
|
|
};
|
|
} catch (error) {
|
|
logger.warn('获取下载配置失败，使用默认配置:', error.message);
|
|
return this.defaultDownloadConfig;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 获取当前下载路径
|
|
*/
|
|
async getDownloadPath() {
|
|
try {
|
|
const config = await this.configManager.readConfig();
|
|
const downloadDir = config.downloadDir || './downloads';
|
|
|
|
// 如果是相对路径，转换为绝对路径
|
|
return path.isAbsolute(downloadDir)
|
|
? downloadDir
|
|
: path.resolve(process.cwd(), downloadDir);
|
|
} catch (error) {
|
|
logger.error('获取下载路径失败:', error);
|
|
// 返回默认路径
|
|
return path.resolve(process.cwd(), 'downloads');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 计算文件MD5
|
|
*/
|
|
async calculateFileMD5(filePath) {
|
|
return new Promise((resolve, reject) => {
|
|
const hash = crypto.createHash('md5');
|
|
const stream = fs.createReadStream(filePath);
|
|
|
|
stream.on('data', (data) => {
|
|
hash.update(data);
|
|
});
|
|
|
|
stream.on('end', () => {
|
|
resolve(hash.digest('hex'));
|
|
});
|
|
|
|
stream.on('error', reject);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* 检查文件完整性
|
|
*/
|
|
/**
|
|
* 检查文件完整性
|
|
* @param {string} filePath - 文件路径
|
|
* @param {number} expectedSize - 期望的文件大小
|
|
* @param {string} expectedMimeType - 期望的MIME类型
|
|
* @returns {Object} 检查结果
|
|
*/
|
|
async checkFileIntegrity(filePath, expectedSize = null, expectedMimeType = null) {
|
|
try {
|
|
if (!await fs.pathExists(filePath)) {
|
|
return { valid: false, reason: '文件不存在' };
|
|
}
|
|
|
|
const stats = await fs.stat(filePath);
|
|
|
|
// 检查文件大小
|
|
if (expectedSize && stats.size !== expectedSize) {
|
|
return { valid: false, reason: '文件大小不匹配', actual: stats.size, expected: expectedSize };
|
|
}
|
|
|
|
// 检查文件是否为空
|
|
if (stats.size === 0) {
|
|
return { valid: false, reason: '文件为空' };
|
|
}
|
|
|
|
// 检查文件是否过小（可能下载不完整）
|
|
const minSize = this.getMinimumFileSize(filePath, expectedMimeType);
|
|
if (stats.size < minSize) {
|
|
return { valid: false, reason: '文件过小，可能下载不完整', size: stats.size, minSize };
|
|
}
|
|
|
|
// 检查文件头部是否符合预期格式
|
|
const headerCheck = await this.checkFileHeader(filePath, expectedMimeType);
|
|
if (!headerCheck.valid) {
|
|
return headerCheck;
|
|
}
|
|
|
|
// 文件存在且大小正常，认为有效
|
|
return { valid: true, size: stats.size };
|
|
} catch (error) {
|
|
return { valid: false, reason: '检查文件失败', error: error.message };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 获取文件的最小合理大小
|
|
* @param {string} filePath - 文件路径
|
|
* @param {string} expectedMimeType - 期望的MIME类型
|
|
* @returns {number} 最小文件大小
|
|
*/
|
|
getMinimumFileSize(filePath, expectedMimeType) {
|
|
const ext = path.extname(filePath).toLowerCase();
|
|
|
|
// 根据文件类型设置最小大小
|
|
if (expectedMimeType) {
|
|
if (expectedMimeType.startsWith('image/')) {
|
|
return 1024; // 图片文件至少1KB
|
|
}
|
|
}
|
|
|
|
// 根据扩展名判断
|
|
switch (ext) {
|
|
case '.jpg':
|
|
case '.jpeg':
|
|
return 1024; // JPEG文件至少1KB
|
|
case '.png':
|
|
return 512; // PNG文件至少512字节
|
|
case '.gif':
|
|
return 256; // GIF文件至少256字节
|
|
case '.webp':
|
|
return 512; // WebP文件至少512字节
|
|
case '.bmp':
|
|
return 1024; // BMP文件至少1KB
|
|
default:
|
|
return 256; // 其他文件至少256字节
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 检查文件头部格式
|
|
* @param {string} filePath - 文件路径
|
|
* @param {string} expectedMimeType - 期望的MIME类型
|
|
* @returns {Object} 检查结果
|
|
*/
|
|
async checkFileHeader(filePath, expectedMimeType) {
|
|
try {
|
|
// 读取文件前几个字节来检查文件头
|
|
const buffer = Buffer.alloc(16);
|
|
const fd = await fs.open(filePath, 'r');
|
|
|
|
try {
|
|
const { bytesRead } = await fs.read(fd, buffer, 0, 16, 0);
|
|
|
|
if (bytesRead < 4) {
|
|
return { valid: false, reason: '文件头部数据不足' };
|
|
}
|
|
|
|
// 检查常见图片格式的文件头
|
|
const header = buffer.toString('hex', 0, Math.min(bytesRead, 8));
|
|
|
|
// JPEG文件头: FFD8FF
|
|
if (header.startsWith('ffd8ff')) {
|
|
if (expectedMimeType && !expectedMimeType.includes('jpeg') && !expectedMimeType.includes('jpg')) {
|
|
return { valid: false, reason: '文件格式不匹配：检测到JPEG但期望其他格式' };
|
|
}
|
|
return { valid: true, detectedType: 'image/jpeg' };
|
|
}
|
|
|
|
// PNG文件头: 89504E47
|
|
if (header.startsWith('89504e47')) {
|
|
if (expectedMimeType && !expectedMimeType.includes('png')) {
|
|
return { valid: false, reason: '文件格式不匹配：检测到PNG但期望其他格式' };
|
|
}
|
|
return { valid: true, detectedType: 'image/png' };
|
|
}
|
|
|
|
// GIF文件头: 474946
|
|
if (header.startsWith('474946')) {
|
|
if (expectedMimeType && !expectedMimeType.includes('gif')) {
|
|
return { valid: false, reason: '文件格式不匹配：检测到GIF但期望其他格式' };
|
|
}
|
|
return { valid: true, detectedType: 'image/gif' };
|
|
}
|
|
|
|
// WebP文件头: 52494646...57454250
|
|
if (header.startsWith('52494646') && buffer.toString('hex', 8, 12) === '57454250') {
|
|
if (expectedMimeType && !expectedMimeType.includes('webp')) {
|
|
return { valid: false, reason: '文件格式不匹配：检测到WebP但期望其他格式' };
|
|
}
|
|
return { valid: true, detectedType: 'image/webp' };
|
|
}
|
|
|
|
// 如果没有明确的期望类型，且检测到了有效的图片头部，则认为有效
|
|
if (!expectedMimeType) {
|
|
return { valid: true, detectedType: 'unknown' };
|
|
}
|
|
|
|
// 如果有期望类型但未匹配到已知格式，可能是损坏的文件
|
|
return { valid: false, reason: '无法识别的文件格式或文件头部损坏' };
|
|
|
|
} finally {
|
|
await fs.close(fd);
|
|
}
|
|
} catch (error) {
|
|
return { valid: false, reason: '检查文件头部失败', error: error.message };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 简单的文件下载方法
|
|
*/
|
|
async downloadFile(url, filePath, abortController = null) {
|
|
const downloadConfig = await this.getDownloadConfig();
|
|
const maxRetries = downloadConfig.retryAttempts;
|
|
let lastError = null;
|
|
|
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
let writer = null;
|
|
let response = null;
|
|
|
|
try {
|
|
// 检查是否已被中断
|
|
if (abortController && abortController.signal.aborted) {
|
|
throw new Error('下载已被中断');
|
|
}
|
|
|
|
// 使用增强的文件工具类确保目录存在
|
|
const dirPath = path.dirname(filePath);
|
|
const dirCreated = await FileUtils.safeEnsureDirEnhanced(dirPath);
|
|
|
|
if (!dirCreated) {
|
|
throw new Error(`无法创建目录: ${dirPath}`);
|
|
}
|
|
|
|
// 检查文件是否被占用
|
|
if (await fs.pathExists(filePath)) {
|
|
const fileReleased = await FileUtils.waitForFileRelease(filePath);
|
|
if (!fileReleased) {
|
|
throw new Error(`文件被占用，无法写入: ${filePath}`);
|
|
}
|
|
}
|
|
|
|
// 再次检查是否已被中断
|
|
if (abortController && abortController.signal.aborted) {
|
|
throw new Error('下载已被中断');
|
|
}
|
|
|
|
response = await axios({
|
|
method: 'GET',
|
|
url: url,
|
|
responseType: 'stream',
|
|
headers: {
|
|
'Referer': 'https://www.pixiv.net/',
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
|
},
|
|
timeout: downloadConfig.timeout,
|
|
signal: abortController ? abortController.signal : undefined
|
|
});
|
|
|
|
// 使用增强的写入流创建方法
|
|
writer = await FileUtils.safeCreateWriteStream(filePath);
|
|
response.data.pipe(writer);
|
|
|
|
return new Promise((resolve, reject) => {
|
|
let isResolved = false;
|
|
|
|
const cleanup = () => {
|
|
if (writer && !writer.destroyed) {
|
|
writer.destroy();
|
|
}
|
|
if (response && response.data && !response.data.destroyed) {
|
|
response.data.destroy();
|
|
}
|
|
};
|
|
|
|
// 监听中断信号
|
|
if (abortController) {
|
|
abortController.signal.addEventListener('abort', () => {
|
|
if (isResolved) return;
|
|
isResolved = true;
|
|
|
|
logger.info(`下载被中断: ${filePath}`);
|
|
cleanup();
|
|
|
|
// 删除未完成的文件
|
|
this.safeDeleteFile(filePath).catch(error => {
|
|
logger.warn('删除被中断的文件失败', { filePath, error: error.message });
|
|
});
|
|
|
|
reject(new Error('下载被中断'));
|
|
});
|
|
}
|
|
|
|
writer.on('finish', async () => {
|
|
if (isResolved) return;
|
|
isResolved = true;
|
|
|
|
try {
|
|
// 验证文件是否写入成功
|
|
await fs.access(filePath);
|
|
cleanup();
|
|
resolve();
|
|
} catch (error) {
|
|
logger.error(`文件写入验证失败: ${filePath}`, error.message);
|
|
cleanup();
|
|
reject(error);
|
|
}
|
|
});
|
|
|
|
writer.on('error', async (error) => {
|
|
if (isResolved) return;
|
|
isResolved = true;
|
|
|
|
logger.error(`文件写入流错误: ${filePath}`, {
|
|
error: error.message,
|
|
stack: error.stack,
|
|
attempt,
|
|
url
|
|
});
|
|
|
|
// 下载失败时删除文件
|
|
try {
|
|
if (await fs.pathExists(filePath)) {
|
|
await this.safeDeleteFile(filePath);
|
|
logger.debug('已清理失败的下载文件', { filePath });
|
|
}
|
|
} catch (removeError) {
|
|
logger.warn('清理失败文件时出错:', {
|
|
filePath,
|
|
error: removeError.message
|
|
});
|
|
}
|
|
|
|
cleanup();
|
|
reject(error);
|
|
});
|
|
|
|
// 添加超时处理
|
|
const timeout = setTimeout(() => {
|
|
if (isResolved) return;
|
|
isResolved = true;
|
|
|
|
logger.error(`下载超时: ${filePath}`, {
|
|
url,
|
|
timeout: downloadConfig.timeout + 60000,
|
|
attempt
|
|
});
|
|
|
|
const timeoutError = new Error('下载超时');
|
|
cleanup();
|
|
reject(timeoutError);
|
|
}, downloadConfig.timeout + 60000); // 动态超时 + 1分钟缓冲
|
|
|
|
// 清理超时定时器
|
|
writer.on('finish', () => clearTimeout(timeout));
|
|
writer.on('error', () => clearTimeout(timeout));
|
|
if (abortController) {
|
|
abortController.signal.addEventListener('abort', () => clearTimeout(timeout));
|
|
}
|
|
});
|
|
|
|
} catch (error) {
|
|
// 确保清理资源
|
|
if (writer && !writer.destroyed) {
|
|
writer.destroy();
|
|
}
|
|
if (response && response.data && !response.data.destroyed) {
|
|
response.data.destroy();
|
|
}
|
|
|
|
lastError = error;
|
|
|
|
// 首先检查是否是中断错误，如果是则直接抛出，不重试
|
|
if (error.message === '下载已被中断' ||
|
|
error.code === 'ERR_CANCELED' ||
|
|
error.name === 'AbortError' ||
|
|
(error.message && error.message.includes('canceled'))) {
|
|
|
|
logger.error(`下载文件失败 (尝试 ${attempt}/${maxRetries}): ${filePath}`, {
|
|
error: error.message,
|
|
stack: error.stack,
|
|
url,
|
|
retryable: false,
|
|
attempt,
|
|
reason: 'download_interrupted'
|
|
});
|
|
|
|
throw error;
|
|
}
|
|
|
|
// 处理其他文件系统错误
|
|
const errorResult = ErrorHandler.handleFileSystemError(error, filePath, 'download');
|
|
|
|
logger.error(`下载文件失败 (尝试 ${attempt}/${maxRetries}): ${filePath}`, {
|
|
error: error.message,
|
|
stack: error.stack,
|
|
url,
|
|
retryable: errorResult.retryable,
|
|
attempt
|
|
});
|
|
|
|
// 如果不是可重试的错误，直接抛出
|
|
if (!errorResult.retryable) {
|
|
throw error;
|
|
}
|
|
|
|
// 如果是最后一次尝试，抛出错误
|
|
if (attempt === maxRetries) {
|
|
logger.error(`下载文件最终失败: ${filePath}`, {
|
|
error: error.message,
|
|
stack: error.stack,
|
|
url,
|
|
totalAttempts: maxRetries
|
|
});
|
|
throw error;
|
|
}
|
|
|
|
// 等待后重试
|
|
const retryDelay = ErrorHandler.getRetryDelay(error, attempt);
|
|
logger.info(`等待 ${retryDelay}ms 后重试下载: ${filePath}`);
|
|
await new Promise(resolve => setTimeout(resolve, retryDelay));
|
|
}
|
|
}
|
|
|
|
// 如果所有重试都失败了
|
|
throw lastError;
|
|
}
|
|
|
|
/**
|
|
* 获取文件扩展名
|
|
*/
|
|
getFileExtension(url) {
|
|
const match = url.match(/\.([a-zA-Z0-9]+)(\?|$)/);
|
|
return match ? `.${match[1]}` : '.jpg';
|
|
}
|
|
|
|
/**
|
|
* 获取目录大小
|
|
*/
|
|
async getDirectorySize(dirPath) {
|
|
try {
|
|
const files = await fs.readdir(dirPath);
|
|
let totalSize = 0;
|
|
|
|
for (const file of files) {
|
|
const filePath = path.join(dirPath, file);
|
|
const stat = await fs.stat(filePath);
|
|
if (stat.isFile()) {
|
|
totalSize += stat.size;
|
|
}
|
|
}
|
|
|
|
return totalSize;
|
|
} catch (error) {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 创建安全的目录名
|
|
*/
|
|
createSafeDirectoryName(name) {
|
|
if (!name) return 'Untitled';
|
|
|
|
// 移除或替换Windows文件系统不允许的字符
|
|
let safeName = name
|
|
// 替换Windows文件系统不允许的字符
|
|
.replace(/[<>:"/\\|?*]/g, '_')
|
|
// 替换波浪号和其他可能导致问题的字符
|
|
.replace(/[~`!@#$%^&*()+=\[\]{};',]/g, '_')
|
|
// 替换控制字符
|
|
.replace(/[\x00-\x1f\x7f]/g, '_')
|
|
// 替换Unicode控制字符
|
|
.replace(/[\u0000-\u001F\u007F-\u009F]/g, '_')
|
|
// 替换零宽字符
|
|
.replace(/[\u200B-\u200D\uFEFF]/g, '_')
|
|
// 替换其他可能导致问题的Unicode字符
|
|
.replace(/[\uFFFD]/g, '_');
|
|
|
|
// 移除前后空格和点
|
|
safeName = safeName.trim().replace(/^\.+|\.+$/g, '');
|
|
|
|
// 如果处理后为空，使用默认名称
|
|
if (!safeName) {
|
|
safeName = 'Untitled';
|
|
}
|
|
|
|
// 限制长度，避免路径过长（Windows路径限制为260字符）
|
|
if (safeName.length > 100) {
|
|
safeName = safeName.substring(0, 100);
|
|
}
|
|
|
|
// 确保不以数字开头（避免与Windows保留名称冲突）
|
|
if (/^\d/.test(safeName)) {
|
|
safeName = 'artwork_' + safeName;
|
|
}
|
|
|
|
// 检查是否为Windows保留名称
|
|
const reservedNames = ['CON', 'PRN', 'AUX', 'NUL', 'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9', 'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9'];
|
|
if (reservedNames.includes(safeName.toUpperCase())) {
|
|
safeName = 'artwork_' + safeName;
|
|
}
|
|
|
|
return safeName;
|
|
}
|
|
|
|
/**
|
|
* 确保目录存在
|
|
*/
|
|
async ensureDirectory(dirPath) {
|
|
const success = await FileUtils.safeEnsureDirEnhanced(dirPath);
|
|
if (!success) {
|
|
throw new Error(`目录创建失败: ${dirPath}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 删除目录
|
|
*/
|
|
async removeDirectory(dirPath) {
|
|
if (await fs.pathExists(dirPath)) {
|
|
await fs.remove(dirPath);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 检查目录是否存在
|
|
*/
|
|
async directoryExists(dirPath) {
|
|
return await fs.pathExists(dirPath);
|
|
}
|
|
|
|
/**
|
|
* 列出目录内容
|
|
*/
|
|
async listDirectory(dirPath) {
|
|
try {
|
|
return await fs.readdir(dirPath);
|
|
} catch (error) {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 复制文件
|
|
*/
|
|
async copyFile(src, dest) {
|
|
await fs.copy(src, dest);
|
|
}
|
|
|
|
/**
|
|
* 移动文件
|
|
*/
|
|
async moveFile(src, dest) {
|
|
await fs.move(src, dest);
|
|
}
|
|
|
|
/**
|
|
* 删除文件
|
|
*/
|
|
async deleteFile(filePath) {
|
|
try {
|
|
if (await fs.pathExists(filePath)) {
|
|
await fs.unlink(filePath);
|
|
logger.debug('文件删除成功', { filePath });
|
|
}
|
|
} catch (error) {
|
|
logger.error(`文件删除失败: ${filePath}`, {
|
|
error: error.message,
|
|
stack: error.stack,
|
|
code: error.code
|
|
});
|
|
// 不抛出错误，避免影响其他操作
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 安全删除文件（兼容 pkg 打包）
|
|
*/
|
|
async safeDeleteFile(filePath) {
|
|
return await FileUtils.safeDeleteFile(filePath);
|
|
}
|
|
|
|
/**
|
|
* 检查文件是否存在
|
|
*/
|
|
async fileExists(filePath) {
|
|
return await fs.pathExists(filePath);
|
|
}
|
|
|
|
/**
|
|
* 获取文件信息
|
|
*/
|
|
async getFileInfo(filePath) {
|
|
try {
|
|
const stats = await fs.stat(filePath);
|
|
return {
|
|
exists: true,
|
|
size: stats.size,
|
|
created: stats.birthtime,
|
|
modified: stats.mtime,
|
|
isFile: stats.isFile(),
|
|
isDirectory: stats.isDirectory()
|
|
};
|
|
} catch (error) {
|
|
return { exists: false };
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = FileManager; |