kjqwdw/pixiv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

整合下载逻辑，修复下载历史记录和显示问题

Browse Source
This commit is contained in:
kjqwer
2025-09-29 12:15:02 +08:00
parent 9f25573bf8
commit 423d35d42d
10 changed files with 299 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/artist.js
+2 -3
View File
@@ -80,12 +80,11 @@ class ArtistService {
const response = await this.makeRequest('GET', `/v1/user/illusts?${stringify(params)}`);
// 获取作者作品列表
// 获取作者作品列表 - 不在这里处理 limit，让调用方处理
return {
success: true,
data: {
artworks: response.illusts,
artworks: response.illusts || [],
next_url: response.next_url,
},
};
backend/services/artwork.js
+9 -2
View File
@@ -264,15 +264,22 @@ class ArtworkService {
content,
filter,
offset,
limit,
};
const response = await this.makeRequest('GET', `/v1/illust/ranking?${stringify(params)}`);
// 获取排行榜作品列表
let artworks = response.illusts || [];
// 如果指定了 limit，则截取相应数量的作品
if (limit && limit < artworks.length) {
artworks = artworks.slice(0, limit);
}
return {
success: true,
data: {
artworks: response.illusts,
artworks: artworks,
next_url: response.next_url,
mode,
date: response.date,
backend/services/download-executor.js
+35 -247
View File
@@ -167,19 +167,31 @@ class DownloadExecutor {
}
/**
* 执行批量下载
* 执行批量下载 - 统一的批量下载方法
* @param {Object} task - 任务对象
* @param {Array} items - 要下载的项目列表（可以是作品ID数组或作品对象数组）
* @param {Object} options - 下载选项
*/
async executeBatchDownload(task, artworkIds, options) {
async executeBatchDownload(task, items, options = {}) {
// 获取动态并发配置
const concurrentConfig = await this.downloadService.getConcurrentConfig();
const { concurrent = concurrentConfig.concurrentDownloads, size = 'original', quality = 'high', format = 'auto' } = options;
const {
concurrent = concurrentConfig.concurrentDownloads,
maxConcurrent = concurrentConfig.maxConcurrentFiles,
size = 'original',
quality = 'high',
format = 'auto'
} = options;
// 使用合适的并发数
const batchSize = concurrent || maxConcurrent;
try {
const results = [];
const recentCompleted = []; // 最近完成的作品列表
// 分批下载
for (let i = 0; i < artworkIds.length; i += concurrent) {
for (let i = 0; i < items.length; i += batchSize) {
if (task.status === 'cancelled') {
break;
}
@@ -190,9 +202,12 @@ class DownloadExecutor {
break;
}
const batch = artworkIds.slice(i, i + concurrent);
const batchPromises = batch.map(async artworkId => {
const batch = items.slice(i, i + batchSize);
const batchPromises = batch.map(async item => {
try {
// 获取作品ID - 支持直接传入ID或作品对象
const artworkId = typeof item === 'object' ? item.id : item;
// 使用专门的批量下载方法，避免创建重复任务
const downloadResult = await this.downloadService.downloadSingleArtworkForBatch(artworkId, {
size,
@@ -215,8 +230,12 @@ class DownloadExecutor {
// 添加到最近完成列表
const completedItem = {
artwork_id: artworkId,
artwork_title: downloadResult.artwork_title || `作品 ${artworkId}`,
artist_name: downloadResult.artist_name || '未知作者'
artwork_title: downloadResult.artwork_title ||
(typeof item === 'object' ? item.title : null) ||
`作品 ${artworkId}`,
artist_name: downloadResult.artist_name ||
(typeof item === 'object' ? item.user?.name : null) ||
'未知作者'
};
recentCompleted.unshift(completedItem);
@@ -241,6 +260,7 @@ class DownloadExecutor {
}
} catch (error) {
// 异常情况
const artworkId = typeof item === 'object' ? item.id : item;
task.failed_files++;
const result = { artwork_id: artworkId, success: false, error: error.message };
results.push(result);
@@ -256,7 +276,7 @@ class DownloadExecutor {
this.progressManager.notifyProgressUpdate(task.id, task);
// 添加延迟避免请求过于频繁
if (i + concurrent < artworkIds.length) {
if (i + batchSize < items.length) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
}
@@ -271,244 +291,12 @@ class DownloadExecutor {
// 添加到历史记录
const historyItem = {
id: task.id,
type: 'batch',
total_files: task.total_files,
completed_files: task.completed_files,
failed_files: task.failed_files,
start_time: task.start_time,
end_time: task.end_time instanceof Date ? task.end_time.toISOString() : task.end_time,
status: task.status,
};
await this.historyManager.addHistoryItem(historyItem);
} catch (error) {
task.status = 'failed';
task.error = error.message;
task.end_time = new Date();
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
}
}
/**
* 执行作者作品下载
*/
async executeArtistDownload(task, newArtworks, options) {
// 获取动态并发配置
const concurrentConfig = await this.downloadService.getConcurrentConfig();
const { maxConcurrent = concurrentConfig.maxConcurrentFiles, size = 'original', quality = 'high', format = 'auto' } = options;
try {
const results = [];
const recentCompleted = []; // 最近完成的作品列表
// 分批下载作品
for (let i = 0; i < newArtworks.length; i += maxConcurrent) {
if (task.status === 'cancelled') {
break;
}
const batch = newArtworks.slice(i, i + maxConcurrent);
const batchPromises = batch.map(async artwork => {
try {
// 使用专门的批量下载方法，避免创建重复任务
const downloadResult = await this.downloadService.downloadSingleArtworkForBatch(artwork.id, {
size,
quality,
format,
skipExisting: true
});
if (downloadResult.success) {
// 检查是否跳过下载
if (downloadResult.skipped) {
// 跳过下载，不计入失败，但也不计入完成
const result = { artwork_id: artwork.id, success: true, skipped: true };
results.push(result);
return result;
} else {
// 真正下载成功
task.completed_files++;
// 添加到最近完成列表
const completedItem = {
artwork_id: artwork.id,
artwork_title: downloadResult.artwork_title || artwork.title || `作品 ${artwork.id}`,
artist_name: downloadResult.artist_name || artwork.user?.name || '未知作者'
};
recentCompleted.unshift(completedItem);
// 只保留最近5个
if (recentCompleted.length > 5) {
recentCompleted.pop();
}
// 更新任务的recent_completed
task.recent_completed = [...recentCompleted];
const result = { artwork_id: artwork.id, success: true };
results.push(result);
return result;
}
} else {
// 下载失败
task.failed_files++;
const result = { artwork_id: artwork.id, success: false, error: downloadResult.error };
results.push(result);
return result;
}
} catch (error) {
// 异常情况
task.failed_files++;
const result = { artwork_id: artwork.id, success: false, error: error.message };
results.push(result);
return result;
}
});
await Promise.all(batchPromises);
// 更新进度并通知
task.progress = Math.round((task.completed_files / task.total_files) * 100);
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
// 添加延迟避免请求过于频繁
if (i + maxConcurrent < newArtworks.length) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
}
// 更新任务状态
task.status = task.failed_files === 0 ? 'completed' : 'partial';
task.end_time = new Date();
task.results = results;
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
// 添加到历史记录
const historyItem = {
id: task.id,
type: 'artist',
artist_name: task.artist_name,
total_files: task.total_files,
completed_files: task.completed_files,
failed_files: task.failed_files,
start_time: task.start_time,
end_time: task.end_time instanceof Date ? task.end_time.toISOString() : task.end_time,
status: task.status,
};
await this.historyManager.addHistoryItem(historyItem);
} catch (error) {
task.status = 'failed';
task.error = error.message;
task.end_time = new Date();
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
}
}
/**
* 执行排行榜作品下载
*/
async executeRankingDownload(task, newArtworks, options) {
// 获取动态并发配置
const concurrentConfig = await this.downloadService.getConcurrentConfig();
const { maxConcurrent = concurrentConfig.maxConcurrentFiles, size = 'original', quality = 'high', format = 'auto' } = options;
try {
const results = [];
const recentCompleted = []; // 最近完成的作品列表
// 分批下载作品
for (let i = 0; i < newArtworks.length; i += maxConcurrent) {
if (task.status === 'cancelled') {
break;
}
const batch = newArtworks.slice(i, i + maxConcurrent);
const batchPromises = batch.map(async artwork => {
try {
// 使用专门的批量下载方法，避免创建重复任务
const downloadResult = await this.downloadService.downloadSingleArtworkForBatch(artwork.id, {
size,
quality,
format,
skipExisting: true
});
if (downloadResult.success) {
// 检查是否跳过下载
if (downloadResult.skipped) {
// 跳过下载，不计入失败，但也不计入完成
const result = { artwork_id: artwork.id, success: true, skipped: true };
results.push(result);
return result;
} else {
// 真正下载成功
task.completed_files++;
// 添加到最近完成列表
const completedItem = {
artwork_id: artwork.id,
artwork_title: downloadResult.artwork_title || artwork.title || `作品 ${artwork.id}`,
artist_name: downloadResult.artist_name || artwork.user?.name || '未知作者'
};
recentCompleted.unshift(completedItem);
// 只保留最近5个
if (recentCompleted.length > 5) {
recentCompleted.pop();
}
// 更新任务的recent_completed
task.recent_completed = [...recentCompleted];
const result = { artwork_id: artwork.id, success: true };
results.push(result);
return result;
}
} else {
// 下载失败
task.failed_files++;
const result = { artwork_id: artwork.id, success: false, error: downloadResult.error };
results.push(result);
return result;
}
} catch (error) {
// 异常情况
task.failed_files++;
const result = { artwork_id: artwork.id, success: false, error: error.message };
results.push(result);
return result;
}
});
await Promise.all(batchPromises);
// 更新进度并通知
task.progress = Math.round((task.completed_files / task.total_files) * 100);
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
// 添加延迟避免请求过于频繁
if (i + maxConcurrent < newArtworks.length) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
}
// 更新任务状态
task.status = task.failed_files === 0 ? 'completed' : 'partial';
task.end_time = new Date();
task.results = results;
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
// 添加到历史记录
const historyItem = {
id: task.id,
type: 'ranking',
type: task.type, // 使用任务的原始类型
artist_name: task.artist_name, // 如果是作者下载任务会有这个字段
artist_id: task.artist_id, // 作者ID
mode: task.mode, // 如果是排行榜下载任务会有这个字段
ranking_type: task.ranking_type, // 排行榜类型
task_description: task.task_description, // 任务描述
total_files: task.total_files,
completed_files: task.completed_files,
failed_files: task.failed_files,
backend/services/download.js
+165 -204
View File
@@ -626,72 +626,12 @@ class DownloadService {
* 批量下载作品
*/
async downloadMultipleArtworks(artworkIds, options = {}) {
// 获取动态并发配置
const concurrentConfig = await this.getConcurrentConfig();
const { concurrent = concurrentConfig.concurrentDownloads, size = 'original', quality = 'high', format = 'auto', skipExisting = true } = options;
try {
// 检查重复下载
let filteredIds = artworkIds;
let skippedCount = 0;
if (skipExisting) {
const downloadedIds = await this.getDownloadedArtworkIds();
const downloadedSet = new Set(downloadedIds);
filteredIds = artworkIds.filter(id => !downloadedSet.has(id));
skippedCount = artworkIds.length - filteredIds.length;
}
// 创建任务记录
const task = this.taskManager.createTask('batch', {
artwork_ids: artworkIds,
filtered_ids: filteredIds,
total_files: filteredIds.length,
completed_files: 0,
failed_files: 0,
skipped: skippedCount,
results: [],
// 使用统一的批量下载方法
return await this.downloadBatchArtworks(artworkIds, {
...options,
taskType: 'batch',
});
await this.taskManager.saveTasks();
// 立即发送初始状态更新，让前端能立即看到进度条
this.progressManager.notifyProgressUpdate(task.id, task);
// 如果没有需要下载的作品，直接返回
if (filteredIds.length === 0) {
await this.taskManager.updateTask(task.id, {
status: 'completed',
end_time: new Date(),
});
return {
success: true,
data: {
task_id: task.id,
total_artworks: artworkIds.length,
completed_artworks: 0,
failed_artworks: 0,
skipped_artworks: skippedCount,
message: '所有作品都已下载完成',
},
};
}
// 异步执行批量下载
this.downloadExecutor.executeBatchDownload(task, filteredIds, options);
return {
success: true,
data: {
task_id: task.id,
total_artworks: task.total,
completed_artworks: task.completed,
failed_artworks: task.failed,
message: '批量下载任务已创建，正在后台执行',
},
};
} catch (error) {
logger.error('批量下载失败:', error);
return {
@@ -848,11 +788,131 @@ class DownloadService {
return match ? match[1] : 'jpg';
}
/**
* 下载批量作品 - 统一的批量下载方法
* @param {Array} items - 要下载的项目列表（可以是作品ID数组或作品对象数组）
* @param {Object} options - 下载选项
*/
async downloadBatchArtworks(items, options = {}) {
const { size = 'original', quality = 'high', format = 'auto', skipExisting = true, maxConcurrent = 3, taskType = 'batch' } = options;
try {
// 生成任务描述
let taskDescription = '';
let taskTitle = '';
if (taskType === 'ranking') {
const modeMap = {
'day': '日榜',
'week': '周榜',
'month': '月榜',
'rookie': '新人榜'
};
const typeMap = {
'art': '插画',
'manga': '漫画',
'novel': '小说'
};
const modeText = modeMap[options.mode] || options.mode;
const typeText = typeMap[options.type] || options.type;
taskDescription = `${modeText}${typeText}排行榜`;
taskTitle = `排行榜下载 - ${taskDescription}`;
} else if (taskType === 'artist') {
const artistName = options.artist_name || `作者ID: ${options.artist_id}`;
taskDescription = `作者作品 - ${artistName}`;
taskTitle = `作者作品下载 - ${artistName}`;
} else {
taskDescription = '批量下载';
taskTitle = '批量下载';
}
// 创建任务记录
const task = this.taskManager.createTask(taskType, {
total_files: 0,
completed_files: 0,
failed_files: 0,
skipped: 0,
results: [],
task_description: taskDescription,
task_title: taskTitle,
// 保留原有的任务特定字段
...(options.artist_id && { artist_id: options.artist_id }),
...(options.artist_name && { artist_name: options.artist_name }),
...(options.mode && { mode: options.mode }),
...(options.type && { type: options.type }),
});
await this.taskManager.saveTasks();
// 获取已下载的作品ID
const downloadedIds = skipExisting ? await this.getDownloadedArtworkIds() : [];
const downloadedSet = new Set(downloadedIds);
// 过滤已下载的作品
let newItems;
if (skipExisting) {
newItems = items.filter(item => {
const artworkId = typeof item === 'object' ? item.id : item;
return !downloadedSet.has(artworkId);
});
} else {
newItems = items;
}
const skippedCount = items.length - newItems.length;
await this.taskManager.updateTask(task.id, {
skipped: skippedCount,
total_files: newItems.length,
});
// 如果没有需要下载的作品，直接返回
if (newItems.length === 0) {
await this.taskManager.updateTask(task.id, {
status: 'completed',
end_time: new Date(),
});
return {
success: true,
data: {
task_id: task.id,
total_artworks: items.length,
completed_artworks: 0,
failed_artworks: 0,
skipped_artworks: skippedCount,
message: '所有作品都已下载完成',
},
};
}
// 异步执行批量下载
this.downloadExecutor.executeBatchDownload(task, newItems, options);
return {
success: true,
data: {
task_id: task.id,
total_artworks: task.total_files,
completed_artworks: task.completed_files,
failed_artworks: task.failed_files,
message: '批量下载任务已创建，正在后台执行',
},
};
} catch (error) {
logger.error('批量下载失败:', error);
return {
success: false,
error: error.message,
};
}
}
/**
* 下载作者作品
*/
async downloadArtistArtworks(artistId, options = {}) {
const { type = 'art', limit = 50, size = 'original', quality = 'high', format = 'auto', skipExisting = true, maxConcurrent = 3, pageSize = 30 } = options;
const { type = 'art', limit = 50, pageSize = 30 } = options;
try {
// 先获取作者信息
@@ -866,23 +926,6 @@ class DownloadService {
logger.warn(`获取作者 ${artistId} 信息失败:`, err.message);
}
// 创建任务记录
const task = this.taskManager.createTask('artist', {
artist_id: artistId,
artist_name: artistName,
total_files: 0,
completed_files: 0,
failed_files: 0,
skipped: 0,
results: [],
});
await this.taskManager.saveTasks();
// 获取已下载的作品ID
const downloadedIds = skipExisting ? await this.getDownloadedArtworkIds() : [];
const downloadedSet = new Set(downloadedIds);
// 分页获取作者作品列表
let allArtworks = [];
let offset = 0;
@@ -892,7 +935,6 @@ class DownloadService {
const artworksResult = await this.artistService.getArtistArtworks(artistId, {
type,
offset: offset,
limit: Math.min(pageSize, limit - allArtworks.length),
});
if (!artworksResult.success) {
@@ -903,66 +945,28 @@ class DownloadService {
if (artworks.length === 0) {
hasMore = false;
} else {
allArtworks.push(...artworks);
// 确保不超过指定的 limit
const remainingSlots = limit - allArtworks.length;
const artworksToAdd = artworks.slice(0, remainingSlots);
allArtworks.push(...artworksToAdd);
offset += artworks.length;
// 基于 next_url 判断是否还有更多页面
hasMore = !!artworksResult.data.next_url;
hasMore = !!artworksResult.data.next_url && allArtworks.length < limit;
// 添加延迟避免请求过于频繁
await new Promise(resolve => setTimeout(resolve, 500));
}
}
// 过滤已下载的作品
const newArtworks = skipExisting ? allArtworks.filter(artwork => !downloadedSet.has(artwork.id)) : allArtworks;
const skippedCount = allArtworks.length - newArtworks.length;
await this.taskManager.updateTask(task.id, {
skipped: skippedCount,
total_files: newArtworks.length,
// 使用统一的批量下载方法
return await this.downloadBatchArtworks(allArtworks, {
...options,
taskType: 'artist',
artist_id: artistId,
artist_name: artistName,
});
// 作者作品下载统计
// 如果没有需要下载的作品，直接返回
if (newArtworks.length === 0) {
await this.taskManager.updateTask(task.id, {
status: 'completed',
end_time: new Date(),
});
return {
success: true,
data: {
task_id: task.id,
artist_id: artistId,
artist_name: artistName,
total_artworks: allArtworks.length,
completed_artworks: 0,
failed_artworks: 0,
skipped_artworks: skippedCount,
message: '所有作品都已下载完成',
},
};
}
// 异步执行作者作品下载
this.downloadExecutor.executeArtistDownload(task, newArtworks, options);
return {
success: true,
data: {
task_id: task.id,
artist_id: artistId,
artist_name: artistName,
total_artworks: task.total_files,
completed_artworks: task.completed_files,
failed_artworks: task.failed_files,
message: '作者作品下载任务已创建，正在后台执行',
},
};
} catch (error) {
logger.error('作者作品下载失败:', error);
return {
@@ -976,35 +980,21 @@ class DownloadService {
* 下载排行榜作品
*/
async downloadRankingArtworks(options = {}) {
const { mode = 'day', type = 'art', limit = 50, size = 'original', quality = 'high', format = 'auto', skipExisting = true, maxConcurrent = 3, pageSize = 30 } = options;
const { mode = 'day', type = 'art', limit = 50, pageSize = 30 } = options;
try {
// 创建任务记录
const task = this.taskManager.createTask('ranking', {
mode: mode,
type: type,
total_files: 0,
completed_files: 0,
failed_files: 0,
skipped: 0,
results: [],
});
await this.taskManager.saveTasks();
// 获取已下载的作品ID
const downloadedIds = skipExisting ? await this.getDownloadedArtworkIds() : [];
const downloadedSet = new Set(downloadedIds);
// 分页获取排行榜作品列表
let allArtworks = [];
let offset = 0;
let hasMore = true;
while (hasMore && allArtworks.length < limit) {
const remainingLimit = limit - allArtworks.length;
const requestLimit = Math.min(pageSize, remainingLimit);
const rankingResult = await this.getRankingArtworks(mode, type, {
offset: offset,
limit: Math.min(pageSize, limit - allArtworks.length),
limit: requestLimit,
});
if (!rankingResult.success) {
@@ -1015,66 +1005,28 @@ class DownloadService {
if (artworks.length === 0) {
hasMore = false;
} else {
allArtworks.push(...artworks);
// 确保不超过指定的 limit
const remainingSlots = limit - allArtworks.length;
const artworksToAdd = artworks.slice(0, remainingSlots);
allArtworks.push(...artworksToAdd);
offset += artworks.length;
// 基于 next_url 判断是否还有更多页面
hasMore = !!rankingResult.data.next_url;
hasMore = !!rankingResult.data.next_url && allArtworks.length < limit;
// 添加延迟避免请求过于频繁
await new Promise(resolve => setTimeout(resolve, 500));
}
}
// 过滤已下载的作品
const newArtworks = skipExisting ? allArtworks.filter(artwork => !downloadedSet.has(artwork.id)) : allArtworks;
const skippedCount = allArtworks.length - newArtworks.length;
await this.taskManager.updateTask(task.id, {
skipped: skippedCount,
total_files: newArtworks.length,
// 使用统一的批量下载方法
return await this.downloadBatchArtworks(allArtworks, {
...options,
taskType: 'ranking',
mode: mode,
type: type,
});
// 排行榜作品下载统计
// 如果没有需要下载的作品，直接返回
if (newArtworks.length === 0) {
await this.taskManager.updateTask(task.id, {
status: 'completed',
end_time: new Date(),
});
return {
success: true,
data: {
task_id: task.id,
mode: mode,
type: type,
total_artworks: allArtworks.length,
completed_artworks: 0,
failed_artworks: 0,
skipped_artworks: skippedCount,
message: '所有作品都已下载完成',
},
};
}
// 异步执行排行榜作品下载
this.downloadExecutor.executeRankingDownload(task, newArtworks, options);
return {
success: true,
data: {
task_id: task.id,
mode: mode,
type: type,
total_artworks: task.total,
completed_artworks: task.completed,
failed_artworks: task.failed,
message: '排行榜作品下载任务已创建，正在后台执行',
},
};
} catch (error) {
logger.error('排行榜作品下载失败:', error);
return {
@@ -1101,11 +1053,20 @@ class DownloadService {
limit,
});
// 检查 ArtworkService 返回的结果
if (!result.success) {
throw new Error(result.error || '获取排行榜数据失败');
}
// 确保数据结构正确
const artworks = result.data?.artworks || [];
const nextUrl = result.data?.next_url || null;
return {
success: true,
data: {
artworks: result.artworks,
next_url: result.next_url || null,
artworks: artworks,
next_url: nextUrl,
},
};
} catch (error) {
backend/services/history-manager.js
+6 -2
View File
@@ -82,7 +82,11 @@ class HistoryManager {
// 只保存关键信息
artwork_id: item.artwork_id,
artist_name: item.artist_name,
artwork_title: item.artwork_title
artist_id: item.artist_id,
artwork_title: item.artwork_title,
mode: item.mode,
ranking_type: item.ranking_type,
task_description: item.task_description
};
this.history.unshift(simplifiedItem);
@@ -219,4 +223,4 @@ class HistoryManager {
}
}
module.exports = HistoryManager;
module.exports = HistoryManager;