kjqwdw/pixiv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加作品完整性检查

Browse Source
This commit is contained in:
kjqwer
2025-08-26 08:43:55 +08:00
parent 5ce5543461
commit 7206d803eb
5 changed files with 331 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/download-executor.js
+23 -7
View File
@@ -59,21 +59,37 @@ class DownloadExecutor {
const fileName = `image_${index + 1}.${this.getFileExtension(imageUrl)}`;
const filePath = path.join(artworkDir, fileName);
// 检查文件是否已存在
// 检查文件是否已存在且完整
if (await this.fileManager.fileExists(filePath)) {
task.completed_files++;
task.progress = Math.round((task.completed_files / task.total_files) * 100);
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
continue;
// 验证文件完整性
const integrity = await this.fileManager.checkFileIntegrity(filePath);
if (integrity.valid) {
task.completed_files++;
task.progress = Math.round((task.completed_files / task.total_files) * 100);
await this.taskManager.saveTasks();
this.progressManager.notifyProgressUpdate(task.id, task);
results.push({ success: true, file: fileName, skipped: true });
continue;
} else {
// 文件不完整，删除重新下载
console.log(`文件不完整，重新下载: ${filePath}`);
await this.fileManager.safeDeleteFile(filePath);
}
}
try {
// 确保目录存在
await this.fileManager.ensureDirectory(path.dirname(filePath));
// 下载文件并等待完成
await this.fileManager.downloadFile(imageUrl, filePath);
// 验证下载的文件完整性
const integrity = await this.fileManager.checkFileIntegrity(filePath);
if (!integrity.valid) {
throw new Error(`文件下载不完整: ${integrity.reason}`);
}
task.completed_files++;
task.progress = Math.round((task.completed_files / task.total_files) * 100);
await this.taskManager.saveTasks();
@@ -92,7 +108,7 @@ class DownloadExecutor {
const infoPath = path.join(artworkDir, 'artwork_info.json');
await fs.writeJson(infoPath, artwork, { spaces: 2 });
// 更新任务状态
// 更新任务状态 - 确保所有文件都处理完成后再更新
task.status = task.failed_files === 0 ? 'completed' : 'partial';
task.end_time = new Date();
task.progress = 100;
backend/services/download.js
+125 -3
View File
@@ -320,6 +320,7 @@ class DownloadService {
// 检查作品信息文件
const infoPath = path.join(artworkPath, 'artwork_info.json');
if (!(await this.fileManager.fileExists(infoPath))) {
console.log(`作品 ${artworkId} 缺少信息文件`);
return false;
}
@@ -328,19 +329,30 @@ class DownloadService {
const imageFiles = files.filter(file => /\.(jpg|jpeg|png|gif|webp)$/i.test(file) && file !== 'artwork_info.json');
if (imageFiles.length === 0) {
console.log(`作品 ${artworkId} 没有图片文件`);
return false;
}
// 检查每个图片文件的完整性
let validFiles = 0;
for (const imageFile of imageFiles) {
const imagePath = path.join(artworkPath, imageFile);
const integrity = await this.fileManager.checkFileIntegrity(imagePath);
if (!integrity.valid) {
return false;
if (integrity.valid) {
validFiles++;
} else {
console.log(`作品 ${artworkId} 的图片文件 ${imageFile} 不完整: ${integrity.reason}`);
}
}
return true;
// 只有当所有图片文件都完整时，才认为作品已下载
if (validFiles === imageFiles.length) {
console.log(`作品 ${artworkId} 已完整下载，共 ${validFiles} 个文件`);
return true;
} else {
console.log(`作品 ${artworkId} 下载不完整，${validFiles}/${imageFiles.length} 个文件有效`);
return false;
}
}
}
}
@@ -352,6 +364,116 @@ class DownloadService {
}
}
/**
* 强制重新检查作品下载状态，包括清理不完整的文件
*/
async forceCheckArtworkDownloaded(artworkId) {
try {
const downloadPath = await this.fileManager.getDownloadPath();
let cleanedFiles = 0;
// 扫描所有作者目录
const artistEntries = await this.fileManager.listDirectory(downloadPath);
for (const artistEntry of artistEntries) {
const artistPath = path.join(downloadPath, artistEntry);
const artistStat = await this.fileManager.getFileInfo(artistPath);
if (!artistStat.exists || !artistStat.isDirectory) continue;
// 扫描作者下的作品目录
const artworkEntries = await this.fileManager.listDirectory(artistPath);
for (const artworkEntry of artworkEntries) {
// 检查是否是目标作品目录（包含数字ID）
const artworkMatch = artworkEntry.match(/^(\d+)_(.+)$/);
if (artworkMatch && artworkMatch[1] === artworkId.toString()) {
const artworkPath = path.join(artistPath, artworkEntry);
// 检查作品信息文件
const infoPath = path.join(artworkPath, 'artwork_info.json');
if (!(await this.fileManager.fileExists(infoPath))) {
console.log(`作品 ${artworkId} 缺少信息文件，清理目录`);
await this.fileManager.removeDirectory(artworkPath);
return {
is_downloaded: false,
cleaned_files: 1,
message: '作品目录不完整，已清理'
};
}
// 检查图片文件
const files = await this.fileManager.listDirectory(artworkPath);
const imageFiles = files.filter(file => /\.(jpg|jpeg|png|gif|webp)$/i.test(file) && file !== 'artwork_info.json');
if (imageFiles.length === 0) {
console.log(`作品 ${artworkId} 没有图片文件，清理目录`);
await this.fileManager.removeDirectory(artworkPath);
return {
is_downloaded: false,
cleaned_files: 1,
message: '作品目录没有图片文件，已清理'
};
}
// 检查每个图片文件的完整性，清理不完整的文件
let validFiles = 0;
for (const imageFile of imageFiles) {
const imagePath = path.join(artworkPath, imageFile);
const integrity = await this.fileManager.checkFileIntegrity(imagePath);
if (integrity.valid) {
validFiles++;
} else {
console.log(`作品 ${artworkId} 的图片文件 ${imageFile} 不完整，删除: ${integrity.reason}`);
await this.fileManager.safeDeleteFile(imagePath);
cleanedFiles++;
}
}
// 如果所有文件都被清理了，删除整个目录
if (validFiles === 0) {
console.log(`作品 ${artworkId} 所有图片文件都不完整，清理目录`);
await this.fileManager.removeDirectory(artworkPath);
return {
is_downloaded: false,
cleaned_files: cleanedFiles + 1,
message: '所有图片文件都不完整，已清理整个目录'
};
}
// 只有当所有图片文件都完整时，才认为作品已下载
if (validFiles === imageFiles.length) {
return {
is_downloaded: true,
cleaned_files: cleanedFiles,
message: `作品已完整下载，共 ${validFiles} 个文件`
};
} else {
return {
is_downloaded: false,
cleaned_files: cleanedFiles,
message: `作品下载不完整，${validFiles}/${imageFiles.length} 个文件有效，已清理 ${cleanedFiles} 个不完整文件`
};
}
}
}
}
return {
is_downloaded: false,
cleaned_files: cleanedFiles,
message: '作品未找到'
};
} catch (error) {
console.error('强制检查作品下载状态失败:', error);
return {
is_downloaded: false,
cleaned_files: 0,
message: `检查失败: ${error.message}`
};
}
}
/**
* 下载单个作品
*/
backend/services/file-manager.js
+53
View File
@@ -83,6 +83,59 @@ class FileManager {
return { valid: false, reason: '文件为空' };
}
// 检查文件是否过小（可能下载不完整）
if (stats.size < 1024) { // 小于1KB的文件可能是损坏的
return { valid: false, reason: '文件过小，可能下载不完整', size: stats.size };
}
// 检查文件头，验证是否为有效的图片文件
try {
const fileHandle = await fs.open(filePath, 'r');
const buffer = Buffer.alloc(12);
await fileHandle.read(buffer, 0, 12, 0);
await fileHandle.close();
// 检查常见图片格式的文件头
const isJPEG = buffer[0] === 0xFF && buffer[1] === 0xD8 && buffer[2] === 0xFF;
const isPNG = buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4E && buffer[3] === 0x47;
const isGIF = (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) ||
(buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x38);
const isWebP = buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46;
if (!isJPEG && !isPNG && !isGIF && !isWebP) {
return { valid: false, reason: '文件格式无效或损坏', size: stats.size };
}
// 对于JPEG文件，检查文件尾
if (isJPEG) {
const endBuffer = Buffer.alloc(2);
const endHandle = await fs.open(filePath, 'r');
await endHandle.read(endBuffer, 0, 2, stats.size - 2);
await endHandle.close();
if (endBuffer[0] !== 0xFF || endBuffer[1] !== 0xD9) {
return { valid: false, reason: 'JPEG文件不完整（缺少结束标记）', size: stats.size };
}
}
// 对于PNG文件，检查文件尾
if (isPNG) {
const endBuffer = Buffer.alloc(8);
const endHandle = await fs.open(filePath, 'r');
await endHandle.read(endBuffer, 0, 8, stats.size - 8);
await endHandle.close();
const pngEnd = Buffer.from([0x49, 0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82]);
if (!endBuffer.equals(pngEnd)) {
return { valid: false, reason: 'PNG文件不完整（缺少结束标记）', size: stats.size };
}
}
} catch (headerError) {
console.warn('文件头检查失败，但继续验证:', headerError.message);
// 如果文件头检查失败，但文件大小正常，仍然认为是有效的
}
return { valid: true, size: stats.size };
} catch (error) {
return { valid: false, reason: '检查文件失败', error: error.message };