kjqwdw/pixiv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重构批量下载逻辑，优化历史显示和下载管理

Browse Source
This commit is contained in:
kjqwer
2025-08-25 09:10:00 +08:00
parent 29ef34b5a9
commit 4033e4eed2
10 changed files with 1240 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/download-executor.js
+257 -50
View File
@@ -1,15 +1,16 @@
const fs = require('fs-extra');
const path = require('path');
const fs = require('fs-extra');
/**
* 下载执行器 - 负责具体的下载逻辑执行
*/
class DownloadExecutor {
constructor(fileManager, taskManager, progressManager, historyManager) {
constructor(fileManager, taskManager, progressManager, historyManager, downloadService) {
this.fileManager = fileManager;
this.taskManager = taskManager;
this.progressManager = progressManager;
this.historyManager = historyManager;
this.downloadService = downloadService; // 添加下载服务引用
}
/**
@@ -17,38 +18,53 @@ class DownloadExecutor {
*/
async executeArtworkDownload(task, images, size, artworkDir, artwork) {
try {
// 检查哪些文件已经存在（断点续传）
const existingFiles = new Set();
if (await this.fileManager.directoryExists(artworkDir)) {
const files = await this.fileManager.listDirectory(artworkDir);
for (const file of files) {
if (/\.(jpg|jpeg|png|gif|webp)$/i.test(file)) {
existingFiles.add(file);
}
}
}
// 逐个下载图片，实时更新进度
const results = [];
for (let index = 0; index < images.length; index++) {
if (task.status === 'cancelled') {
break;
}
const image = images[index];
const imageUrl = image[size] || image.original;
// 使用安全处理的标题创建文件名
const safeTitle = this.fileManager.createSafeDirectoryName(artwork.title || 'Untitled');
const fileName = `${safeTitle}_${artwork.id}_${index + 1}${this.fileManager.getFileExtension(imageUrl)}`;
// 从图片对象中获取指定尺寸的URL
const imageObj = images[index];
let imageUrl;
// 根据size参数选择对应的URL
switch (size) {
case 'original':
imageUrl = imageObj.original;
break;
case 'large':
imageUrl = imageObj.large;
break;
case 'medium':
imageUrl = imageObj.medium;
break;
case 'square_medium':
imageUrl = imageObj.square_medium;
break;
default:
imageUrl = imageObj.original || imageObj.large || imageObj.medium;
}
// 确保imageUrl是字符串
if (typeof imageUrl !== 'string') {
console.error(`图片URL不是字符串:`, imageUrl);
task.failed_files++;
this.progressManager.notifyProgressUpdate(task.id, task);
results.push({ success: false, error: '图片URL格式错误' });
continue;
}
const fileName = `image_${index + 1}.${this.getFileExtension(imageUrl)}`;
const filePath = path.join(artworkDir, fileName);
// 如果文件已存在，跳过下载
if (existingFiles.has(fileName)) {
// 检查文件是否已存在
if (await this.fileManager.fileExists(filePath)) {
task.completed_files++;
task.progress = Math.round((task.completed_files / task.total_files) * 100);
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
results.push({ success: true, file: fileName, skipped: true });
continue;
}
@@ -95,7 +111,7 @@ class DownloadExecutor {
completed_files: task.completed_files,
failed_files: task.failed_files,
start_time: task.start_time,
end_time: task.end_time,
end_time: task.end_time instanceof Date ? task.end_time.toISOString() : task.end_time,
status: task.status,
};
@@ -118,23 +134,66 @@ class DownloadExecutor {
try {
const results = [];
const recentCompleted = []; // 最近完成的作品列表
// 分批下载
for (let i = 0; i < task.filtered_ids.length; i += concurrent) {
for (let i = 0; i < artworkIds.length; i += concurrent) {
if (task.status === 'cancelled') {
break;
}
const batch = task.filtered_ids.slice(i, i + concurrent);
const batch = artworkIds.slice(i, i + concurrent);
const batchPromises = batch.map(async artworkId => {
try {
// 这里需要调用主下载服务的方法，暂时返回模拟结果
task.completed++;
const result = { artwork_id: artworkId, success: true };
results.push(result);
return result;
// 使用专门的批量下载方法，避免创建重复任务
const downloadResult = await this.downloadService.downloadSingleArtworkForBatch(artworkId, {
size,
quality,
format,
skipExisting: true
});
if (downloadResult.success) {
// 检查是否跳过下载
if (downloadResult.skipped) {
// 跳过下载，不计入失败，但也不计入完成
const result = { artwork_id: artworkId, success: true, skipped: true };
results.push(result);
return result;
} else {
// 真正下载成功
task.completed_files++;
// 添加到最近完成列表
const completedItem = {
artwork_id: artworkId,
artwork_title: downloadResult.artwork_title || `作品 ${artworkId}`,
artist_name: downloadResult.artist_name || '未知作者'
};
recentCompleted.unshift(completedItem);
// 只保留最近5个
if (recentCompleted.length > 5) {
recentCompleted.pop();
}
// 更新任务的recent_completed
task.recent_completed = [...recentCompleted];
const result = { artwork_id: artworkId, success: true };
results.push(result);
return result;
}
} else {
// 下载失败
task.failed_files++;
const result = { artwork_id: artworkId, success: false, error: downloadResult.error };
results.push(result);
return result;
}
} catch (error) {
task.failed++;
// 异常情况
task.failed_files++;
const result = { artwork_id: artworkId, success: false, error: error.message };
results.push(result);
return result;
@@ -142,22 +201,38 @@ class DownloadExecutor {
});
await Promise.all(batchPromises);
task.progress = Math.round((task.completed / task.total) * 100);
// 更新进度并通知
task.progress = Math.round((task.completed_files / task.total_files) * 100);
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
// 添加延迟避免请求过于频繁
if (i + concurrent < task.filtered_ids.length) {
if (i + concurrent < artworkIds.length) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
}
// 更新任务状态
task.status = task.failed === 0 ? 'completed' : 'partial';
task.status = task.failed_files === 0 ? 'completed' : 'partial';
task.end_time = new Date();
task.results = results;
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
// 添加到历史记录
const historyItem = {
id: task.id,
type: 'batch',
total_files: task.total_files,
completed_files: task.completed_files,
failed_files: task.failed_files,
start_time: task.start_time,
end_time: task.end_time instanceof Date ? task.end_time.toISOString() : task.end_time,
status: task.status,
};
await this.historyManager.addHistoryItem(historyItem);
} catch (error) {
task.status = 'failed';
task.error = error.message;
@@ -175,6 +250,7 @@ class DownloadExecutor {
try {
const results = [];
const recentCompleted = []; // 最近完成的作品列表
// 分批下载作品
for (let i = 0; i < newArtworks.length; i += maxConcurrent) {
@@ -185,13 +261,55 @@ class DownloadExecutor {
const batch = newArtworks.slice(i, i + maxConcurrent);
const batchPromises = batch.map(async artwork => {
try {
// 这里需要调用主下载服务的方法，暂时返回模拟结果
task.completed++;
const result = { artwork_id: artwork.id, success: true };
results.push(result);
return result;
// 使用专门的批量下载方法，避免创建重复任务
const downloadResult = await this.downloadService.downloadSingleArtworkForBatch(artwork.id, {
size,
quality,
format,
skipExisting: true
});
if (downloadResult.success) {
// 检查是否跳过下载
if (downloadResult.skipped) {
// 跳过下载，不计入失败，但也不计入完成
const result = { artwork_id: artwork.id, success: true, skipped: true };
results.push(result);
return result;
} else {
// 真正下载成功
task.completed_files++;
// 添加到最近完成列表
const completedItem = {
artwork_id: artwork.id,
artwork_title: downloadResult.artwork_title || artwork.title || `作品 ${artwork.id}`,
artist_name: downloadResult.artist_name || artwork.user?.name || '未知作者'
};
recentCompleted.unshift(completedItem);
// 只保留最近5个
if (recentCompleted.length > 5) {
recentCompleted.pop();
}
// 更新任务的recent_completed
task.recent_completed = [...recentCompleted];
const result = { artwork_id: artwork.id, success: true };
results.push(result);
return result;
}
} else {
// 下载失败
task.failed_files++;
const result = { artwork_id: artwork.id, success: false, error: downloadResult.error };
results.push(result);
return result;
}
} catch (error) {
task.failed++;
// 异常情况
task.failed_files++;
const result = { artwork_id: artwork.id, success: false, error: error.message };
results.push(result);
return result;
@@ -199,7 +317,9 @@ class DownloadExecutor {
});
await Promise.all(batchPromises);
task.progress = Math.round((task.completed / task.total) * 100);
// 更新进度并通知
task.progress = Math.round((task.completed_files / task.total_files) * 100);
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
@@ -210,11 +330,26 @@ class DownloadExecutor {
}
// 更新任务状态
task.status = task.failed === 0 ? 'completed' : 'partial';
task.status = task.failed_files === 0 ? 'completed' : 'partial';
task.end_time = new Date();
task.results = results;
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
// 添加到历史记录
const historyItem = {
id: task.id,
type: 'artist',
artist_name: task.artist_name,
total_files: task.total_files,
completed_files: task.completed_files,
failed_files: task.failed_files,
start_time: task.start_time,
end_time: task.end_time instanceof Date ? task.end_time.toISOString() : task.end_time,
status: task.status,
};
await this.historyManager.addHistoryItem(historyItem);
} catch (error) {
task.status = 'failed';
task.error = error.message;
@@ -232,6 +367,7 @@ class DownloadExecutor {
try {
const results = [];
const recentCompleted = []; // 最近完成的作品列表
// 分批下载作品
for (let i = 0; i < newArtworks.length; i += maxConcurrent) {
@@ -242,13 +378,55 @@ class DownloadExecutor {
const batch = newArtworks.slice(i, i + maxConcurrent);
const batchPromises = batch.map(async artwork => {
try {
// 这里需要调用主下载服务的方法，暂时返回模拟结果
task.completed++;
const result = { artwork_id: artwork.id, success: true };
results.push(result);
return result;
// 使用专门的批量下载方法，避免创建重复任务
const downloadResult = await this.downloadService.downloadSingleArtworkForBatch(artwork.id, {
size,
quality,
format,
skipExisting: true
});
if (downloadResult.success) {
// 检查是否跳过下载
if (downloadResult.skipped) {
// 跳过下载，不计入失败，但也不计入完成
const result = { artwork_id: artwork.id, success: true, skipped: true };
results.push(result);
return result;
} else {
// 真正下载成功
task.completed_files++;
// 添加到最近完成列表
const completedItem = {
artwork_id: artwork.id,
artwork_title: downloadResult.artwork_title || artwork.title || `作品 ${artwork.id}`,
artist_name: downloadResult.artist_name || artwork.user?.name || '未知作者'
};
recentCompleted.unshift(completedItem);
// 只保留最近5个
if (recentCompleted.length > 5) {
recentCompleted.pop();
}
// 更新任务的recent_completed
task.recent_completed = [...recentCompleted];
const result = { artwork_id: artwork.id, success: true };
results.push(result);
return result;
}
} else {
// 下载失败
task.failed_files++;
const result = { artwork_id: artwork.id, success: false, error: downloadResult.error };
results.push(result);
return result;
}
} catch (error) {
task.failed++;
// 异常情况
task.failed_files++;
const result = { artwork_id: artwork.id, success: false, error: error.message };
results.push(result);
return result;
@@ -256,7 +434,9 @@ class DownloadExecutor {
});
await Promise.all(batchPromises);
task.progress = Math.round((task.completed / task.total) * 100);
// 更新进度并通知
task.progress = Math.round((task.completed_files / task.total_files) * 100);
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
@@ -267,11 +447,25 @@ class DownloadExecutor {
}
// 更新任务状态
task.status = task.failed === 0 ? 'completed' : 'partial';
task.status = task.failed_files === 0 ? 'completed' : 'partial';
task.end_time = new Date();
task.results = results;
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
// 添加到历史记录
const historyItem = {
id: task.id,
type: 'ranking',
total_files: task.total_files,
completed_files: task.completed_files,
failed_files: task.failed_files,
start_time: task.start_time,
end_time: task.end_time instanceof Date ? task.end_time.toISOString() : task.end_time,
status: task.status,
};
await this.historyManager.addHistoryItem(historyItem);
} catch (error) {
task.status = 'failed';
task.error = error.message;
@@ -280,6 +474,19 @@ class DownloadExecutor {
this.progressManager.notifyProgressUpdate(task.id, task);
}
}
/**
* 获取文件扩展名
*/
getFileExtension(url) {
if (typeof url !== 'string') {
console.warn('URL不是字符串，使用默认扩展名:', url);
return 'jpg';
}
const match = url.match(/\.([a-zA-Z0-9]+)(?:\?|$)/);
return match ? match[1] : 'jpg';
}
}
module.exports = DownloadExecutor;
backend/services/download.js
+236 -16
View File
@@ -6,6 +6,7 @@ const FileManager = require('./file-manager');
const ProgressManager = require('./progress-manager');
const HistoryManager = require('./history-manager');
const DownloadExecutor = require('./download-executor');
const fs = require('fs-extra'); // Added for fs-extra
/**
* 下载服务 - 主服务类，协调各个管理器
@@ -32,7 +33,8 @@ class DownloadService {
this.taskManager = new TaskManager(this.dataPath);
this.progressManager = new ProgressManager();
this.historyManager = new HistoryManager(this.dataPath);
this.downloadExecutor = new DownloadExecutor(this.fileManager, this.taskManager, this.progressManager, this.historyManager);
// 先创建下载执行器，稍后在init方法中设置downloadService引用
this.downloadExecutor = new DownloadExecutor(this.fileManager, this.taskManager, this.progressManager, this.historyManager, this);
this.initialized = false;
}
@@ -148,6 +150,63 @@ class DownloadService {
};
}
/**
* 清理历史记录
*/
async cleanupHistory(keepCount = 500) {
try {
const result = await this.historyManager.cleanupHistoryManually(keepCount);
return {
success: true,
data: result
};
} catch (error) {
return {
success: false,
error: error.message
};
}
}
/**
* 清理已完成的任务
*/
async cleanupTasks(keepActive = true, keepCompleted = 100) {
try {
const result = await this.taskManager.cleanupTasksManually(keepActive, keepCompleted);
return {
success: true,
data: { cleaned: result }
};
} catch (error) {
return {
success: false,
error: error.message
};
}
}
/**
* 获取系统统计信息
*/
async getSystemStats() {
try {
const taskStats = this.taskManager.getTaskStats();
const historyStats = this.historyManager.getHistoryStats();
return {
tasks: taskStats,
history: historyStats,
activeConnections: this.progressManager.getTotalListenerCount()
};
} catch (error) {
return {
success: false,
error: error.message
};
}
}
// 代理方法 - 文件管理
async getDownloadedFiles() {
try {
@@ -409,9 +468,9 @@ class DownloadService {
const task = this.taskManager.createTask('batch', {
artwork_ids: artworkIds,
filtered_ids: filteredIds,
total: filteredIds.length,
completed: 0,
failed: 0,
total_files: filteredIds.length,
completed_files: 0,
failed_files: 0,
skipped: skippedCount,
results: [],
});
@@ -439,7 +498,7 @@ class DownloadService {
}
// 异步执行批量下载
this.downloadExecutor.executeBatchDownload(task, artworkIds, options);
this.downloadExecutor.executeBatchDownload(task, filteredIds, options);
return {
success: true,
@@ -460,6 +519,153 @@ class DownloadService {
}
}
/**
* 内部方法：下载单个作品（用于批量下载，不创建新任务）
*/
async downloadSingleArtworkForBatch(artworkId, options = {}) {
const { size = 'original', quality = 'high', format = 'auto', skipExisting = true } = options;
try {
// 检查是否已下载
if (skipExisting && (await this.isArtworkDownloaded(artworkId))) {
return {
success: true,
skipped: true,
message: '作品已存在且完整，跳过下载',
};
}
// 获取作品信息
const artworkResult = await this.artworkService.getArtworkDetail(artworkId);
if (!artworkResult.success) {
throw new Error(`获取作品信息失败: ${artworkResult.error}`);
}
const artwork = artworkResult.data;
// 确保作品信息完整
if (!artwork || !artwork.user || !artwork.title) {
throw new Error('作品信息不完整');
}
const artistName = this.fileManager.createSafeDirectoryName(artwork.user.name || 'Unknown Artist');
const artworkTitle = this.fileManager.createSafeDirectoryName(artwork.title || 'Untitled');
// 创建作品目录
const downloadPath = await this.fileManager.getDownloadPath();
const artistDir = path.join(downloadPath, artistName);
const artworkDirName = `${artworkId}_${artworkTitle}`;
const artworkDir = path.join(artistDir, artworkDirName);
// 如果是重新下载，先删除现有目录
if (!skipExisting && (await this.fileManager.directoryExists(artworkDir))) {
console.log(`删除现有作品目录: ${artworkDir}`);
await this.fileManager.removeDirectory(artworkDir);
}
await this.fileManager.ensureDirectory(artworkDir);
// 获取图片URL
const imagesResult = await this.artworkService.getArtworkImages(artworkId, size);
if (!imagesResult.success) {
throw new Error(`获取图片URL失败: ${imagesResult.error}`);
}
const images = imagesResult.data.images;
// 直接下载，不创建新任务
const results = [];
for (let index = 0; index < images.length; index++) {
// 从图片对象中获取指定尺寸的URL
const imageObj = images[index];
let imageUrl;
// 根据size参数选择对应的URL
switch (size) {
case 'original':
imageUrl = imageObj.original;
break;
case 'large':
imageUrl = imageObj.large;
break;
case 'medium':
imageUrl = imageObj.medium;
break;
case 'square_medium':
imageUrl = imageObj.square_medium;
break;
default:
imageUrl = imageObj.original || imageObj.large || imageObj.medium;
}
// 确保imageUrl是字符串
if (typeof imageUrl !== 'string') {
console.error(`图片URL不是字符串:`, imageUrl);
results.push({ success: false, error: '图片URL格式错误' });
continue;
}
const fileName = `image_${index + 1}.${this.getFileExtension(imageUrl)}`;
const filePath = path.join(artworkDir, fileName);
// 检查文件是否已存在
if (await this.fileManager.fileExists(filePath)) {
results.push({ success: true, file: fileName, skipped: true });
continue;
}
try {
// 确保目录存在
await this.fileManager.ensureDirectory(path.dirname(filePath));
await this.fileManager.downloadFile(imageUrl, filePath);
results.push({ success: true, file: fileName });
} catch (error) {
console.error(`下载图片失败 ${index + 1}: ${error.message}`);
results.push({ success: false, error: error.message });
}
}
// 保存作品信息
const infoPath = path.join(artworkDir, 'artwork_info.json');
await fs.writeJson(infoPath, artwork, { spaces: 2 });
// 检查下载结果
const failedCount = results.filter(r => !r.success).length;
const successCount = results.filter(r => r.success && !r.skipped).length;
return {
success: failedCount === 0,
artwork_id: artworkId,
artist_name: artistName,
artwork_title: artworkTitle,
total_files: images.length,
completed_files: successCount,
failed_files: failedCount,
results: results,
};
} catch (error) {
console.error(`下载作品 ${artworkId} 失败:`, error);
return {
success: false,
error: error.message,
artwork_id: artworkId,
};
}
}
/**
* 获取文件扩展名
*/
getFileExtension(url) {
if (typeof url !== 'string') {
console.warn('URL不是字符串，使用默认扩展名:', url);
return 'jpg';
}
const match = url.match(/\.([a-zA-Z0-9]+)(?:\?|$)/);
return match ? match[1] : 'jpg';
}
/**
* 下载作者作品
*/
@@ -467,12 +673,24 @@ class DownloadService {
const { type = 'art', limit = 50, size = 'original', quality = 'high', format = 'auto', skipExisting = true, maxConcurrent = 3, pageSize = 30 } = options;
try {
// 先获取作者信息
let artistName = '未知作者';
try {
const artistResult = await this.artistService.getArtistInfo(artistId);
if (artistResult.success && artistResult.data) {
artistName = artistResult.data.name || `作者 ${artistId}`;
}
} catch (err) {
console.warn(`获取作者 ${artistId} 信息失败:`, err.message);
}
// 创建任务记录
const task = this.taskManager.createTask('artist', {
artist_id: artistId,
total: 0,
completed: 0,
failed: 0,
artist_name: artistName,
total_files: 0,
completed_files: 0,
failed_files: 0,
skipped: 0,
results: [],
});
@@ -521,7 +739,7 @@ class DownloadService {
await this.taskManager.updateTask(task.id, {
skipped: skippedCount,
total: newArtworks.length,
total_files: newArtworks.length,
});
// 作者作品下载统计
@@ -538,6 +756,7 @@ class DownloadService {
data: {
task_id: task.id,
artist_id: artistId,
artist_name: artistName,
total_artworks: allArtworks.length,
completed_artworks: 0,
failed_artworks: 0,
@@ -555,9 +774,10 @@ class DownloadService {
data: {
task_id: task.id,
artist_id: artistId,
total_artworks: task.total,
completed_artworks: task.completed,
failed_artworks: task.failed,
artist_name: artistName,
total_artworks: task.total_files,
completed_artworks: task.completed_files,
failed_artworks: task.failed_files,
message: '作者作品下载任务已创建，正在后台执行',
},
};
@@ -581,9 +801,9 @@ class DownloadService {
const task = this.taskManager.createTask('ranking', {
mode: mode,
type: type,
total: 0,
completed: 0,
failed: 0,
total_files: 0,
completed_files: 0,
failed_files: 0,
skipped: 0,
results: [],
});
@@ -631,7 +851,7 @@ class DownloadService {
await this.taskManager.updateTask(task.id, {
skipped: skippedCount,
total: newArtworks.length,
total_files: newArtworks.length,
});
// 排行榜作品下载统计
backend/services/history-manager.js
+64 -5
View File
@@ -10,6 +10,10 @@ class HistoryManager {
this.historyFile = path.join(dataPath, 'download_history.json');
this.history = [];
this.initialized = false;
// 配置
this.maxHistoryItems = 500; // 最多保存500条历史记录
this.cleanupThreshold = 600; // 超过600条时开始清理
}
/**
@@ -19,8 +23,12 @@ class HistoryManager {
try {
await fs.ensureDir(this.dataPath);
await this.loadHistory();
// 初始化时清理历史记录
await this.cleanupHistory();
this.initialized = true;
// 历史记录管理器初始化完成
console.log('历史记录管理器初始化完成');
} catch (error) {
console.error('历史记录管理器初始化失败:', error);
this.initialized = false;
@@ -53,10 +61,47 @@ class HistoryManager {
}
/**
* 添加历史记录
* 添加历史记录（简化版本）
*/
async addHistoryItem(item) {
this.history.unshift(item);
// 简化历史记录信息
const simplifiedItem = {
id: item.id,
type: item.type,
status: item.status,
total_files: item.total_files,
completed_files: item.completed_files,
failed_files: item.failed_files,
start_time: item.start_time,
end_time: item.end_time,
// 只保存关键信息
artwork_id: item.artwork_id,
artist_name: item.artist_name,
artwork_title: item.artwork_title
};
this.history.unshift(simplifiedItem);
// 检查是否需要清理
if (this.history.length > this.cleanupThreshold) {
await this.cleanupHistory();
} else {
await this.saveHistory();
}
}
/**
* 清理历史记录
*/
async cleanupHistory() {
if (this.history.length <= this.maxHistoryItems) {
return;
}
console.log(`清理历史记录: ${this.history.length} -> ${this.maxHistoryItems}`);
// 保留最新的记录
this.history = this.history.slice(0, this.maxHistoryItems);
await this.saveHistory();
}
@@ -120,8 +165,7 @@ class HistoryManager {
completed: 0,
failed: 0,
partial: 0,
totalFiles: 0,
totalSize: 0
totalFiles: 0
};
for (const item of this.history) {
@@ -153,6 +197,21 @@ class HistoryManager {
(item.artist_name && item.artist_name.toLowerCase().includes(lowerQuery))
);
}
/**
* 手动清理历史记录（保留指定数量）
*/
async cleanupHistoryManually(keepCount = this.maxHistoryItems) {
if (this.history.length <= keepCount) {
return { cleaned: 0, remaining: this.history.length };
}
const cleanedCount = this.history.length - keepCount;
this.history = this.history.slice(0, keepCount);
await this.saveHistory();
return { cleaned: cleanedCount, remaining: this.history.length };
}
}
module.exports = HistoryManager;
backend/services/progress-manager.js
+11
View File
@@ -58,6 +58,17 @@ class ProgressManager {
: 0;
}
/**
* 获取总连接数
*/
getTotalListenerCount() {
let total = 0;
for (const listeners of this.progressListeners.values()) {
total += listeners.length;
}
return total;
}
/**
* 清理所有监听器
*/
backend/services/task-manager.js
+100 -8
View File
@@ -11,6 +11,10 @@ class TaskManager {
this.tasksFile = path.join(dataPath, 'download_tasks.json');
this.tasks = new Map(); // 内存中的任务状态
this.initialized = false;
// 配置
this.maxCompletedTasks = 100; // 最多保留100个已完成的任务
this.cleanupThreshold = 150; // 超过150个时开始清理
}
/**
@@ -20,8 +24,12 @@ class TaskManager {
try {
await fs.ensureDir(this.dataPath);
await this.loadTasks();
// 初始化时清理已完成的任务
await this.cleanupCompletedTasks();
this.initialized = true;
// 任务管理器初始化完成
console.log('任务管理器初始化完成');
} catch (error) {
console.error('任务管理器初始化失败:', error);
this.initialized = false;
@@ -99,7 +107,14 @@ class TaskManager {
}
Object.assign(task, updates);
await this.saveTasks();
// 如果任务完成，检查是否需要清理
if (['completed', 'failed', 'cancelled', 'partial'].includes(updates.status)) {
await this.checkAndCleanupTasks();
} else {
await this.saveTasks();
}
return true;
}
@@ -121,6 +136,24 @@ class TaskManager {
return Array.from(this.tasks.values());
}
/**
* 获取活跃任务（下载中或暂停）
*/
getActiveTasks() {
return Array.from(this.tasks.values()).filter(task =>
['downloading', 'paused'].includes(task.status)
);
}
/**
* 获取已完成的任务
*/
getCompletedTasks() {
return Array.from(this.tasks.values()).filter(task =>
['completed', 'failed', 'cancelled', 'partial'].includes(task.status)
);
}
/**
* 获取指定状态的任务
*/
@@ -128,23 +161,45 @@ class TaskManager {
return Array.from(this.tasks.values()).filter(task => task.status === status);
}
/**
* 检查并清理任务
*/
async checkAndCleanupTasks() {
const completedTasks = this.getCompletedTasks();
if (completedTasks.length > this.cleanupThreshold) {
await this.cleanupCompletedTasks();
} else {
await this.saveTasks();
}
}
/**
* 清理已完成的任务
*/
async cleanupCompletedTasks() {
const completedStatuses = ['completed', 'failed', 'cancelled', 'partial'];
const completedTasks = Array.from(this.tasks.entries())
.filter(([_, task]) => completedStatuses.includes(task.status))
.sort((a, b) => new Date(b[1].end_time) - new Date(a[1].end_time)); // 按完成时间排序
if (completedTasks.length <= this.maxCompletedTasks) {
await this.saveTasks();
return 0;
}
// 删除超出限制的已完成任务
const tasksToDelete = completedTasks.slice(this.maxCompletedTasks);
let cleanedCount = 0;
for (const [taskId, task] of this.tasks) {
if (completedStatuses.includes(task.status)) {
this.tasks.delete(taskId);
cleanedCount++;
}
for (const [taskId, _] of tasksToDelete) {
this.tasks.delete(taskId);
cleanedCount++;
}
if (cleanedCount > 0) {
await this.saveTasks();
// 清理了已完成的任务
console.log(`清理已完成任务: ${cleanedCount} 个`);
}
return cleanedCount;
@@ -172,6 +227,43 @@ class TaskManager {
return stats;
}
/**
* 手动清理任务
*/
async cleanupTasksManually(keepActive = true, keepCompleted = this.maxCompletedTasks) {
let cleanedCount = 0;
if (keepActive) {
// 保留活跃任务
const activeTasks = this.getActiveTasks();
const completedTasks = this.getCompletedTasks()
.sort((a, b) => new Date(b.end_time) - new Date(a.end_time))
.slice(0, keepCompleted);
// 重建任务Map
const newTasks = new Map();
// 添加活跃任务
for (const task of activeTasks) {
newTasks.set(task.id, task);
}
// 添加要保留的已完成任务
for (const task of completedTasks) {
newTasks.set(task.id, task);
}
cleanedCount = this.tasks.size - newTasks.size;
this.tasks = newTasks;
} else {
// 清理所有已完成任务
cleanedCount = await this.cleanupCompletedTasks();
}
await this.saveTasks();
return cleanedCount;
}
}
module.exports = TaskManager;