增加线程配置,采用动态并发管理
This commit is contained in:
@@ -170,7 +170,9 @@ class DownloadExecutor {
|
||||
* 执行批量下载
|
||||
*/
|
||||
async executeBatchDownload(task, artworkIds, options) {
|
||||
const { concurrent = 3, size = 'original', quality = 'high', format = 'auto' } = options;
|
||||
// 获取动态并发配置
|
||||
const concurrentConfig = await this.downloadService.getConcurrentConfig();
|
||||
const { concurrent = concurrentConfig.concurrentDownloads, size = 'original', quality = 'high', format = 'auto' } = options;
|
||||
|
||||
try {
|
||||
const results = [];
|
||||
@@ -292,7 +294,9 @@ class DownloadExecutor {
|
||||
* 执行作者作品下载
|
||||
*/
|
||||
async executeArtistDownload(task, newArtworks, options) {
|
||||
const { maxConcurrent = 3, size = 'original', quality = 'high', format = 'auto' } = options;
|
||||
// 获取动态并发配置
|
||||
const concurrentConfig = await this.downloadService.getConcurrentConfig();
|
||||
const { maxConcurrent = concurrentConfig.maxConcurrentFiles, size = 'original', quality = 'high', format = 'auto' } = options;
|
||||
|
||||
try {
|
||||
const results = [];
|
||||
@@ -409,7 +413,9 @@ class DownloadExecutor {
|
||||
* 执行排行榜作品下载
|
||||
*/
|
||||
async executeRankingDownload(task, newArtworks, options) {
|
||||
const { maxConcurrent = 3, size = 'original', quality = 'high', format = 'auto' } = options;
|
||||
// 获取动态并发配置
|
||||
const concurrentConfig = await this.downloadService.getConcurrentConfig();
|
||||
const { maxConcurrent = concurrentConfig.maxConcurrentFiles, size = 'original', quality = 'high', format = 'auto' } = options;
|
||||
|
||||
try {
|
||||
const results = [];
|
||||
|
||||
@@ -6,6 +6,7 @@ const FileManager = require('./file-manager');
|
||||
const ProgressManager = require('./progress-manager');
|
||||
const HistoryManager = require('./history-manager');
|
||||
const DownloadExecutor = require('./download-executor');
|
||||
const CacheConfigManager = require('../config/cache-config');
|
||||
const fs = require('fs-extra'); // Added for fs-extra
|
||||
const { defaultLogger } = require('../utils/logger');
|
||||
|
||||
@@ -21,6 +22,7 @@ class DownloadService {
|
||||
this.auth = auth;
|
||||
this.artworkService = new ArtworkService(auth);
|
||||
this.artistService = new ArtistService(auth);
|
||||
this.cacheConfigManager = new CacheConfigManager();
|
||||
|
||||
// 检测是否在pkg打包环境中运行
|
||||
const isPkg = process.pkg !== undefined;
|
||||
@@ -44,6 +46,25 @@ class DownloadService {
|
||||
this.initialized = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取动态并发配置
|
||||
*/
|
||||
async getConcurrentConfig() {
|
||||
try {
|
||||
const cacheConfig = await this.cacheConfigManager.loadConfig();
|
||||
return {
|
||||
concurrentDownloads: cacheConfig.download?.concurrentDownloads || 3,
|
||||
maxConcurrentFiles: cacheConfig.download?.maxConcurrentFiles || 5,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.warn('获取并发配置失败,使用默认值:', error.message);
|
||||
return {
|
||||
concurrentDownloads: 3,
|
||||
maxConcurrentFiles: 5,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化服务
|
||||
*/
|
||||
@@ -605,7 +626,9 @@ class DownloadService {
|
||||
* 批量下载作品
|
||||
*/
|
||||
async downloadMultipleArtworks(artworkIds, options = {}) {
|
||||
const { concurrent = 3, size = 'original', quality = 'high', format = 'auto', skipExisting = true } = options;
|
||||
// 获取动态并发配置
|
||||
const concurrentConfig = await this.getConcurrentConfig();
|
||||
const { concurrent = concurrentConfig.concurrentDownloads, size = 'original', quality = 'high', format = 'auto', skipExisting = true } = options;
|
||||
|
||||
try {
|
||||
// 检查重复下载
|
||||
|
||||
@@ -3,6 +3,7 @@ const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const ConfigManager = require('../config/config-manager');
|
||||
const CacheConfigManager = require('../config/cache-config');
|
||||
const FileUtils = require('../utils/file-utils');
|
||||
const ErrorHandler = require('../utils/error-handler');
|
||||
const { defaultLogger } = require('../utils/logger');
|
||||
@@ -16,15 +17,56 @@ const logger = defaultLogger.child('FileManager');
|
||||
class FileManager {
|
||||
constructor() {
|
||||
this.configManager = new ConfigManager();
|
||||
this.cacheConfigManager = new CacheConfigManager();
|
||||
|
||||
// 下载配置
|
||||
this.downloadConfig = {
|
||||
// 默认下载配置(作为后备)
|
||||
this.defaultDownloadConfig = {
|
||||
timeout: 300000, // 5分钟超时
|
||||
chunkSize: 1024 * 1024, // 1MB块大小
|
||||
retryAttempts: 3, // 重试次数
|
||||
retryDelay: 2000, // 重试延迟
|
||||
concurrentDownloads: 3 // 并发下载数
|
||||
};
|
||||
|
||||
// 初始化时设置线程池大小
|
||||
this.initializeThreadPool();
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化线程池大小
|
||||
*/
|
||||
async initializeThreadPool() {
|
||||
try {
|
||||
const config = await this.getDownloadConfig();
|
||||
if (config.threadPoolSize && !process.env.UV_THREADPOOL_SIZE) {
|
||||
process.env.UV_THREADPOOL_SIZE = config.threadPoolSize.toString();
|
||||
logger.info(`设置线程池大小为: ${config.threadPoolSize}`);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('初始化线程池大小失败,使用默认值:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取下载配置
|
||||
*/
|
||||
async getDownloadConfig() {
|
||||
try {
|
||||
const cacheConfig = await this.cacheConfigManager.loadConfig();
|
||||
return {
|
||||
timeout: cacheConfig.download?.downloadTimeout || this.defaultDownloadConfig.timeout,
|
||||
chunkSize: cacheConfig.download?.chunkSize || this.defaultDownloadConfig.chunkSize,
|
||||
retryAttempts: cacheConfig.download?.retryAttempts || this.defaultDownloadConfig.retryAttempts,
|
||||
retryDelay: cacheConfig.download?.retryDelay || this.defaultDownloadConfig.retryDelay,
|
||||
concurrentDownloads: cacheConfig.download?.concurrentDownloads || this.defaultDownloadConfig.concurrentDownloads,
|
||||
maxConcurrentFiles: cacheConfig.download?.maxConcurrentFiles || 5,
|
||||
threadPoolSize: cacheConfig.download?.threadPoolSize || 16,
|
||||
maxFileSize: cacheConfig.download?.maxFileSize || 50 * 1024 * 1024,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.warn('获取下载配置失败,使用默认配置:', error.message);
|
||||
return this.defaultDownloadConfig;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -103,7 +145,8 @@ class FileManager {
|
||||
* 简单的文件下载方法
|
||||
*/
|
||||
async downloadFile(url, filePath) {
|
||||
const maxRetries = this.downloadConfig.retryAttempts;
|
||||
const downloadConfig = await this.getDownloadConfig();
|
||||
const maxRetries = downloadConfig.retryAttempts;
|
||||
let lastError = null;
|
||||
|
||||
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
@@ -135,7 +178,7 @@ class FileManager {
|
||||
'Referer': 'https://www.pixiv.net/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
},
|
||||
timeout: 60000
|
||||
timeout: downloadConfig.timeout
|
||||
});
|
||||
|
||||
// 使用增强的写入流创建方法
|
||||
@@ -193,7 +236,7 @@ class FileManager {
|
||||
const timeoutError = new Error('下载超时');
|
||||
cleanup();
|
||||
reject(timeoutError);
|
||||
}, 120000); // 2分钟超时
|
||||
}, downloadConfig.timeout + 60000); // 动态超时 + 1分钟缓冲
|
||||
|
||||
// 清理超时定时器
|
||||
writer.on('finish', () => clearTimeout(timeout));
|
||||
|
||||
Reference in New Issue
Block a user