kjqwdw/pixiv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加线程配置，采用动态并发管理

Browse Source
This commit is contained in:
kjqwer
2025-09-06 11:04:57 +08:00
parent 3e910e14a4
commit 1142d55b24
7 changed files with 300 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/config/cache-config.js
+11
View File
@@ -39,6 +39,17 @@ class CacheConfigManager {
retryDelay: 1000,
},
allowedExtensions: ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp'],
// 新增并发下载配置
download: {
concurrentDownloads: 3, // 同时下载任务数
maxConcurrentFiles: 5, // 单个任务内最大并发文件数
threadPoolSize: 16, // Node.js 线程池大小
downloadTimeout: 300000, // 5分钟下载超时
chunkSize: 1024 * 1024, // 1MB块大小
retryAttempts: 3, // 重试次数
retryDelay: 2000, // 重试延迟
maxFileSize: 50 * 1024 * 1024, // 最大文件大小 50MB
},
// 新增Windows特定配置
windows: {
skipInUseFiles: true, // 跳过被占用的文件
backend/services/download-executor.js
+9 -3
View File
@@ -170,7 +170,9 @@ class DownloadExecutor {
* 执行批量下载
*/
async executeBatchDownload(task, artworkIds, options) {
const { concurrent = 3, size = 'original', quality = 'high', format = 'auto' } = options;
// 获取动态并发配置
const concurrentConfig = await this.downloadService.getConcurrentConfig();
const { concurrent = concurrentConfig.concurrentDownloads, size = 'original', quality = 'high', format = 'auto' } = options;
try {
const results = [];
@@ -292,7 +294,9 @@ class DownloadExecutor {
* 执行作者作品下载
*/
async executeArtistDownload(task, newArtworks, options) {
const { maxConcurrent = 3, size = 'original', quality = 'high', format = 'auto' } = options;
// 获取动态并发配置
const concurrentConfig = await this.downloadService.getConcurrentConfig();
const { maxConcurrent = concurrentConfig.maxConcurrentFiles, size = 'original', quality = 'high', format = 'auto' } = options;
try {
const results = [];
@@ -409,7 +413,9 @@ class DownloadExecutor {
* 执行排行榜作品下载
*/
async executeRankingDownload(task, newArtworks, options) {
const { maxConcurrent = 3, size = 'original', quality = 'high', format = 'auto' } = options;
// 获取动态并发配置
const concurrentConfig = await this.downloadService.getConcurrentConfig();
const { maxConcurrent = concurrentConfig.maxConcurrentFiles, size = 'original', quality = 'high', format = 'auto' } = options;
try {
const results = [];
backend/services/download.js
+24 -1
View File
@@ -6,6 +6,7 @@ const FileManager = require('./file-manager');
const ProgressManager = require('./progress-manager');
const HistoryManager = require('./history-manager');
const DownloadExecutor = require('./download-executor');
const CacheConfigManager = require('../config/cache-config');
const fs = require('fs-extra'); // Added for fs-extra
const { defaultLogger } = require('../utils/logger');
@@ -21,6 +22,7 @@ class DownloadService {
this.auth = auth;
this.artworkService = new ArtworkService(auth);
this.artistService = new ArtistService(auth);
this.cacheConfigManager = new CacheConfigManager();
// 检测是否在pkg打包环境中运行
const isPkg = process.pkg !== undefined;
@@ -44,6 +46,25 @@ class DownloadService {
this.initialized = false;
}
/**
* 获取动态并发配置
*/
async getConcurrentConfig() {
try {
const cacheConfig = await this.cacheConfigManager.loadConfig();
return {
concurrentDownloads: cacheConfig.download?.concurrentDownloads || 3,
maxConcurrentFiles: cacheConfig.download?.maxConcurrentFiles || 5,
};
} catch (error) {
logger.warn('获取并发配置失败，使用默认值:', error.message);
return {
concurrentDownloads: 3,
maxConcurrentFiles: 5,
};
}
}
/**
* 初始化服务
*/
@@ -605,7 +626,9 @@ class DownloadService {
* 批量下载作品
*/
async downloadMultipleArtworks(artworkIds, options = {}) {
const { concurrent = 3, size = 'original', quality = 'high', format = 'auto', skipExisting = true } = options;
// 获取动态并发配置
const concurrentConfig = await this.getConcurrentConfig();
const { concurrent = concurrentConfig.concurrentDownloads, size = 'original', quality = 'high', format = 'auto', skipExisting = true } = options;
try {
// 检查重复下载
backend/services/file-manager.js
+48 -5
View File
@@ -3,6 +3,7 @@ const fs = require('fs-extra');
const path = require('path');
const crypto = require('crypto');
const ConfigManager = require('../config/config-manager');
const CacheConfigManager = require('../config/cache-config');
const FileUtils = require('../utils/file-utils');
const ErrorHandler = require('../utils/error-handler');
const { defaultLogger } = require('../utils/logger');
@@ -16,15 +17,56 @@ const logger = defaultLogger.child('FileManager');
class FileManager {
constructor() {
this.configManager = new ConfigManager();
this.cacheConfigManager = new CacheConfigManager();
// 下载配置
this.downloadConfig = {
// 默认下载配置（作为后备）
this.defaultDownloadConfig = {
timeout: 300000, // 5分钟超时
chunkSize: 1024 * 1024, // 1MB块大小
retryAttempts: 3, // 重试次数
retryDelay: 2000, // 重试延迟
concurrentDownloads: 3 // 并发下载数
};
// 初始化时设置线程池大小
this.initializeThreadPool();
}
/**
* 初始化线程池大小
*/
async initializeThreadPool() {
try {
const config = await this.getDownloadConfig();
if (config.threadPoolSize && !process.env.UV_THREADPOOL_SIZE) {
process.env.UV_THREADPOOL_SIZE = config.threadPoolSize.toString();
logger.info(`设置线程池大小为: ${config.threadPoolSize}`);
}
} catch (error) {
logger.warn('初始化线程池大小失败，使用默认值:', error.message);
}
}
/**
* 获取下载配置
*/
async getDownloadConfig() {
try {
const cacheConfig = await this.cacheConfigManager.loadConfig();
return {
timeout: cacheConfig.download?.downloadTimeout || this.defaultDownloadConfig.timeout,
chunkSize: cacheConfig.download?.chunkSize || this.defaultDownloadConfig.chunkSize,
retryAttempts: cacheConfig.download?.retryAttempts || this.defaultDownloadConfig.retryAttempts,
retryDelay: cacheConfig.download?.retryDelay || this.defaultDownloadConfig.retryDelay,
concurrentDownloads: cacheConfig.download?.concurrentDownloads || this.defaultDownloadConfig.concurrentDownloads,
maxConcurrentFiles: cacheConfig.download?.maxConcurrentFiles || 5,
threadPoolSize: cacheConfig.download?.threadPoolSize || 16,
maxFileSize: cacheConfig.download?.maxFileSize || 50 * 1024 * 1024,
};
} catch (error) {
logger.warn('获取下载配置失败，使用默认配置:', error.message);
return this.defaultDownloadConfig;
}
}
/**
@@ -103,7 +145,8 @@ class FileManager {
* 简单的文件下载方法
*/
async downloadFile(url, filePath) {
const maxRetries = this.downloadConfig.retryAttempts;
const downloadConfig = await this.getDownloadConfig();
const maxRetries = downloadConfig.retryAttempts;
let lastError = null;
for (let attempt = 1; attempt <= maxRetries; attempt++) {
@@ -135,7 +178,7 @@ class FileManager {
'Referer': 'https://www.pixiv.net/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
},
timeout: 60000
timeout: downloadConfig.timeout
});
// 使用增强的写入流创建方法
@@ -193,7 +236,7 @@ class FileManager {
const timeoutError = new Error('下载超时');
cleanup();
reject(timeoutError);
}, 120000); // 2分钟超时
}, downloadConfig.timeout + 60000); // 动态超时 + 1分钟缓冲
// 清理超时定时器
writer.on('finish', () => clearTimeout(timeout));
backend/start.js
+6
View File
@@ -4,6 +4,12 @@
* Pixiv 后端服务器启动脚本
*/
// 重要：必须在任何其他模块导入之前设置线程池大小
// 解决多个下载任务时的SSH连接阻塞问题
if (!process.env.UV_THREADPOOL_SIZE) {
process.env.UV_THREADPOOL_SIZE = '16'; // 增加到16个线程
}
const PixivServer = require('./server');
const { defaultLogger } = require('./utils/logger');
ui/src/components/common/SettingsWidget.vue
+185
View File
@@ -102,6 +102,44 @@
</div>
</div>
<!-- 下载配置 -->
<div class="config-section">
<h5>下载配置</h5>
<div class="form-group">
<label>同时下载任务数</label>
<input type="number" v-model.number="concurrentDownloads" min="1" max="10" />
<span class="form-help">建议值: 3-5</span>
</div>
<div class="form-group">
<label>单任务最大并发文件数</label>
<input type="number" v-model.number="maxConcurrentFiles" min="1" max="20" />
<span class="form-help">建议值: 3-8</span>
</div>
<div class="form-group">
<label>线程池大小</label>
<input type="number" v-model.number="threadPoolSize" min="4" max="64" />
<span class="form-help">建议值: 16-32，需要重启生效</span>
</div>
<div class="form-group">
<label>下载超时 (分钟)</label>
<input type="number" v-model.number="downloadTimeoutMinutes" min="1" max="30" />
</div>
<div class="form-group">
<label>重试延迟 (秒)</label>
<input type="number" v-model.number="retryDelaySeconds" min="1" max="30" />
</div>
<div class="form-group">
<label>最大文件大小 (MB)</label>
<input type="number" v-model.number="maxFileSizeMB" min="1" max="500" />
</div>
</div>
<!-- 缓存操作 -->
<div class="cache-actions">
<button @click="saveConfig" class="btn btn-primary" :disabled="saving">
@@ -153,6 +191,16 @@ const config = ref<CacheConfig>({
retryDelay: 1000,
},
allowedExtensions: ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp'],
download: {
concurrentDownloads: 3,
maxConcurrentFiles: 5,
threadPoolSize: 16,
downloadTimeout: 300000,
chunkSize: 1024 * 1024,
retryAttempts: 3,
retryDelay: 2000,
maxFileSize: 50 * 1024 * 1024,
},
lastUpdated: new Date().toISOString()
});
@@ -185,6 +233,121 @@ const timeoutSeconds = computed({
}
});
// 下载配置相关的计算属性
const concurrentDownloads = computed({
get: () => config.value.download?.concurrentDownloads || 3,
set: (value) => {
if (!config.value.download) {
config.value.download = {
concurrentDownloads: 3,
maxConcurrentFiles: 5,
threadPoolSize: 16,
downloadTimeout: 300000,
chunkSize: 1024 * 1024,
retryAttempts: 3,
retryDelay: 2000,
maxFileSize: 50 * 1024 * 1024,
};
}
config.value.download.concurrentDownloads = value;
}
});
const maxConcurrentFiles = computed({
get: () => config.value.download?.maxConcurrentFiles || 5,
set: (value) => {
if (!config.value.download) {
config.value.download = {
concurrentDownloads: 3,
maxConcurrentFiles: 5,
threadPoolSize: 16,
downloadTimeout: 300000,
chunkSize: 1024 * 1024,
retryAttempts: 3,
retryDelay: 2000,
maxFileSize: 50 * 1024 * 1024,
};
}
config.value.download.maxConcurrentFiles = value;
}
});
const threadPoolSize = computed({
get: () => config.value.download?.threadPoolSize || 16,
set: (value) => {
if (!config.value.download) {
config.value.download = {
concurrentDownloads: 3,
maxConcurrentFiles: 5,
threadPoolSize: 16,
downloadTimeout: 300000,
chunkSize: 1024 * 1024,
retryAttempts: 3,
retryDelay: 2000,
maxFileSize: 50 * 1024 * 1024,
};
}
config.value.download.threadPoolSize = value;
}
});
const downloadTimeoutMinutes = computed({
get: () => Math.round((config.value.download?.downloadTimeout || 300000) / (1000 * 60)),
set: (value) => {
if (!config.value.download) {
config.value.download = {
concurrentDownloads: 3,
maxConcurrentFiles: 5,
threadPoolSize: 16,
downloadTimeout: 300000,
chunkSize: 1024 * 1024,
retryAttempts: 3,
retryDelay: 2000,
maxFileSize: 50 * 1024 * 1024,
};
}
config.value.download.downloadTimeout = value * 1000 * 60;
}
});
const retryDelaySeconds = computed({
get: () => Math.round((config.value.download?.retryDelay || 2000) / 1000),
set: (value) => {
if (!config.value.download) {
config.value.download = {
concurrentDownloads: 3,
maxConcurrentFiles: 5,
threadPoolSize: 16,
downloadTimeout: 300000,
chunkSize: 1024 * 1024,
retryAttempts: 3,
retryDelay: 2000,
maxFileSize: 50 * 1024 * 1024,
};
}
config.value.download.retryDelay = value * 1000;
}
});
const maxFileSizeMB = computed({
get: () => Math.round((config.value.download?.maxFileSize || 50 * 1024 * 1024) / (1024 * 1024)),
set: (value) => {
if (!config.value.download) {
config.value.download = {
concurrentDownloads: 3,
maxConcurrentFiles: 5,
threadPoolSize: 16,
downloadTimeout: 300000,
chunkSize: 1024 * 1024,
retryAttempts: 3,
retryDelay: 2000,
maxFileSize: 50 * 1024 * 1024,
};
}
config.value.download.maxFileSize = value * 1024 * 1024;
}
});
// 方法
const toggleSettings = () => {
isOpen.value = !isOpen.value;
@@ -550,6 +713,28 @@ onMounted(() => {
margin-right: 0.5rem;
}
.config-section {
margin-bottom: 1.5rem;
padding: 1rem;
background: #f8fafc;
border-radius: 0.5rem;
border: 1px solid #e2e8f0;
}
.config-section h5 {
margin: 0 0 1rem 0;
font-size: 1rem;
font-weight: 600;
color: #2d3748;
}
.form-help {
display: block;
font-size: 0.75rem;
color: #718096;
margin-top: 0.25rem;
}
.cache-actions {
display: flex;
flex-direction: column;
ui/src/services/cache.ts
+17
View File
@@ -13,6 +13,23 @@ export interface CacheConfig {
retryDelay: number;
};
allowedExtensions: string[];
download?: {
concurrentDownloads: number;
maxConcurrentFiles: number;
threadPoolSize: number;
downloadTimeout: number;
chunkSize: number;
retryAttempts: number;
retryDelay: number;
maxFileSize: number;
};
windows?: {
skipInUseFiles: boolean;
maxRetries: number;
retryDelay: number;
waitForRelease: boolean;
maxWaitTime: number;
};
lastUpdated: string;
}