From 1142d55b24dc5da9f9f79afa4bf42b16ac5eb596 Mon Sep 17 00:00:00 2001 From: kjqwer <2990346238@qq.com> Date: Sat, 6 Sep 2025 11:04:57 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=BA=BF=E7=A8=8B=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E9=87=87=E7=94=A8=E5=8A=A8=E6=80=81=E5=B9=B6?= =?UTF-8?q?=E5=8F=91=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/config/cache-config.js | 11 ++ backend/services/download-executor.js | 12 +- backend/services/download.js | 25 ++- backend/services/file-manager.js | 53 +++++- backend/start.js | 6 + ui/src/components/common/SettingsWidget.vue | 185 ++++++++++++++++++++ ui/src/services/cache.ts | 17 ++ 7 files changed, 300 insertions(+), 9 deletions(-) diff --git a/backend/config/cache-config.js b/backend/config/cache-config.js index f51fe92..7dec384 100644 --- a/backend/config/cache-config.js +++ b/backend/config/cache-config.js @@ -39,6 +39,17 @@ class CacheConfigManager { retryDelay: 1000, }, allowedExtensions: ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp'], + // 新增并发下载配置 + download: { + concurrentDownloads: 3, // 同时下载任务数 + maxConcurrentFiles: 5, // 单个任务内最大并发文件数 + threadPoolSize: 16, // Node.js 线程池大小 + downloadTimeout: 300000, // 5分钟下载超时 + chunkSize: 1024 * 1024, // 1MB块大小 + retryAttempts: 3, // 重试次数 + retryDelay: 2000, // 重试延迟 + maxFileSize: 50 * 1024 * 1024, // 最大文件大小 50MB + }, // 新增Windows特定配置 windows: { skipInUseFiles: true, // 跳过被占用的文件 diff --git a/backend/services/download-executor.js b/backend/services/download-executor.js index b42b54b..df1da09 100644 --- a/backend/services/download-executor.js +++ b/backend/services/download-executor.js @@ -170,7 +170,9 @@ class DownloadExecutor { * 执行批量下载 */ async executeBatchDownload(task, artworkIds, options) { - const { concurrent = 3, size = 'original', quality = 'high', format = 'auto' } = options; + // 获取动态并发配置 + const concurrentConfig = await this.downloadService.getConcurrentConfig(); + const { concurrent = concurrentConfig.concurrentDownloads, size = 'original', quality = 'high', format = 'auto' } = options; try { const results = []; @@ -292,7 +294,9 @@ class DownloadExecutor { * 执行作者作品下载 */ async executeArtistDownload(task, newArtworks, options) { - const { maxConcurrent = 3, size = 'original', quality = 'high', format = 'auto' } = options; + // 获取动态并发配置 + const concurrentConfig = await this.downloadService.getConcurrentConfig(); + const { maxConcurrent = concurrentConfig.maxConcurrentFiles, size = 'original', quality = 'high', format = 'auto' } = options; try { const results = []; @@ -409,7 +413,9 @@ class DownloadExecutor { * 执行排行榜作品下载 */ async executeRankingDownload(task, newArtworks, options) { - const { maxConcurrent = 3, size = 'original', quality = 'high', format = 'auto' } = options; + // 获取动态并发配置 + const concurrentConfig = await this.downloadService.getConcurrentConfig(); + const { maxConcurrent = concurrentConfig.maxConcurrentFiles, size = 'original', quality = 'high', format = 'auto' } = options; try { const results = []; diff --git a/backend/services/download.js b/backend/services/download.js index 7a60013..447175f 100644 --- a/backend/services/download.js +++ b/backend/services/download.js @@ -6,6 +6,7 @@ const FileManager = require('./file-manager'); const ProgressManager = require('./progress-manager'); const HistoryManager = require('./history-manager'); const DownloadExecutor = require('./download-executor'); +const CacheConfigManager = require('../config/cache-config'); const fs = require('fs-extra'); // Added for fs-extra const { defaultLogger } = require('../utils/logger'); @@ -21,6 +22,7 @@ class DownloadService { this.auth = auth; this.artworkService = new ArtworkService(auth); this.artistService = new ArtistService(auth); + this.cacheConfigManager = new CacheConfigManager(); // 检测是否在pkg打包环境中运行 const isPkg = process.pkg !== undefined; @@ -44,6 +46,25 @@ class DownloadService { this.initialized = false; } + /** + * 获取动态并发配置 + */ + async getConcurrentConfig() { + try { + const cacheConfig = await this.cacheConfigManager.loadConfig(); + return { + concurrentDownloads: cacheConfig.download?.concurrentDownloads || 3, + maxConcurrentFiles: cacheConfig.download?.maxConcurrentFiles || 5, + }; + } catch (error) { + logger.warn('获取并发配置失败，使用默认值:', error.message); + return { + concurrentDownloads: 3, + maxConcurrentFiles: 5, + }; + } + } + /** * 初始化服务 */ @@ -605,7 +626,9 @@ class DownloadService { * 批量下载作品 */ async downloadMultipleArtworks(artworkIds, options = {}) { - const { concurrent = 3, size = 'original', quality = 'high', format = 'auto', skipExisting = true } = options; + // 获取动态并发配置 + const concurrentConfig = await this.getConcurrentConfig(); + const { concurrent = concurrentConfig.concurrentDownloads, size = 'original', quality = 'high', format = 'auto', skipExisting = true } = options; try { // 检查重复下载 diff --git a/backend/services/file-manager.js b/backend/services/file-manager.js index e066235..ea6c901 100644 --- a/backend/services/file-manager.js +++ b/backend/services/file-manager.js @@ -3,6 +3,7 @@ const fs = require('fs-extra'); const path = require('path'); const crypto = require('crypto'); const ConfigManager = require('../config/config-manager'); +const CacheConfigManager = require('../config/cache-config'); const FileUtils = require('../utils/file-utils'); const ErrorHandler = require('../utils/error-handler'); const { defaultLogger } = require('../utils/logger'); @@ -16,15 +17,56 @@ const logger = defaultLogger.child('FileManager'); class FileManager { constructor() { this.configManager = new ConfigManager(); + this.cacheConfigManager = new CacheConfigManager(); - // 下载配置 - this.downloadConfig = { + // 默认下载配置（作为后备） + this.defaultDownloadConfig = { timeout: 300000, // 5分钟超时 chunkSize: 1024 * 1024, // 1MB块大小 retryAttempts: 3, // 重试次数 retryDelay: 2000, // 重试延迟 concurrentDownloads: 3 // 并发下载数 }; + + // 初始化时设置线程池大小 + this.initializeThreadPool(); + } + + /** + * 初始化线程池大小 + */ + async initializeThreadPool() { + try { + const config = await this.getDownloadConfig(); + if (config.threadPoolSize && !process.env.UV_THREADPOOL_SIZE) { + process.env.UV_THREADPOOL_SIZE = config.threadPoolSize.toString(); + logger.info(`设置线程池大小为: ${config.threadPoolSize}`); + } + } catch (error) { + logger.warn('初始化线程池大小失败，使用默认值:', error.message); + } + } + + /** + * 获取下载配置 + */ + async getDownloadConfig() { + try { + const cacheConfig = await this.cacheConfigManager.loadConfig(); + return { + timeout: cacheConfig.download?.downloadTimeout || this.defaultDownloadConfig.timeout, + chunkSize: cacheConfig.download?.chunkSize || this.defaultDownloadConfig.chunkSize, + retryAttempts: cacheConfig.download?.retryAttempts || this.defaultDownloadConfig.retryAttempts, + retryDelay: cacheConfig.download?.retryDelay || this.defaultDownloadConfig.retryDelay, + concurrentDownloads: cacheConfig.download?.concurrentDownloads || this.defaultDownloadConfig.concurrentDownloads, + maxConcurrentFiles: cacheConfig.download?.maxConcurrentFiles || 5, + threadPoolSize: cacheConfig.download?.threadPoolSize || 16, + maxFileSize: cacheConfig.download?.maxFileSize || 50 * 1024 * 1024, + }; + } catch (error) { + logger.warn('获取下载配置失败，使用默认配置:', error.message); + return this.defaultDownloadConfig; + } } /** @@ -103,7 +145,8 @@ class FileManager { * 简单的文件下载方法 */ async downloadFile(url, filePath) { - const maxRetries = this.downloadConfig.retryAttempts; + const downloadConfig = await this.getDownloadConfig(); + const maxRetries = downloadConfig.retryAttempts; let lastError = null; for (let attempt = 1; attempt <= maxRetries; attempt++) { @@ -135,7 +178,7 @@ class FileManager { 'Referer': 'https://www.pixiv.net/', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' }, - timeout: 60000 + timeout: downloadConfig.timeout }); // 使用增强的写入流创建方法 @@ -193,7 +236,7 @@ class FileManager { const timeoutError = new Error('下载超时'); cleanup(); reject(timeoutError); - }, 120000); // 2分钟超时 + }, downloadConfig.timeout + 60000); // 动态超时 + 1分钟缓冲 // 清理超时定时器 writer.on('finish', () => clearTimeout(timeout)); diff --git a/backend/start.js b/backend/start.js index f27d0d5..d53f9d3 100644 --- a/backend/start.js +++ b/backend/start.js @@ -4,6 +4,12 @@ * Pixiv 后端服务器启动脚本 */ +// 重要：必须在任何其他模块导入之前设置线程池大小 +// 解决多个下载任务时的SSH连接阻塞问题 +if (!process.env.UV_THREADPOOL_SIZE) { + process.env.UV_THREADPOOL_SIZE = '16'; // 增加到16个线程 +} + const PixivServer = require('./server'); const { defaultLogger } = require('./utils/logger'); diff --git a/ui/src/components/common/SettingsWidget.vue b/ui/src/components/common/SettingsWidget.vue index fab26cd..e94e6bc 100644 --- a/ui/src/components/common/SettingsWidget.vue +++ b/ui/src/components/common/SettingsWidget.vue @@ -102,6 +102,44 @@ + +
+
下载配置
+ +
+ + + 建议值: 3-5 +
+ +
+ + + 建议值: 3-8 +
+ +
+ + + 建议值: 16-32，需要重启生效 +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+
+