kjqwdw/pixiv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

注册表新增数据库存储同步

Browse Source
This commit is contained in:
kjqwer
2025-10-11 11:56:19 +08:00
parent 4e34063373
commit 47c68cadf3
23 changed files with 4396 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/download-registry.js
+201 -110
View File
@@ -2,26 +2,35 @@ const path = require('path');
const fs = require('fs-extra');
const { defaultLogger } = require('../utils/logger');
const ConfigManager = require('../config/config-manager');
const CacheConfigManager = require('../config/cache-config');
const RegistryDatabase = require('../database/registry-database');
const artworkUtils = require('../utils/artwork-utils');
// 创建logger实例
const logger = defaultLogger.child('DownloadRegistry');
/**
* 下载记录管理器 - 维护已下载作品的JSON记录
* 下载记录管理器 - 维护已下载作品的记录
* 支持JSON文件和数据库两种存储模式，根据配置自动选择
* 用于快速检测作品是否已下载，支持导入导出和多设备同步
*/
class DownloadRegistry {
constructor(dataPath) {
constructor(dataPath, databaseManager = null) {
this.dataPath = dataPath;
this.registryPath = path.join(dataPath, 'download-registry.json');
this.registry = {
version: '1.0.5',
artists: {},
lastUpdated: null
lastUpdated: null,
};
this.loaded = false;
this.configManager = new ConfigManager();
this.cacheConfigManager = new CacheConfigManager();
// 数据库相关
this.databaseManager = databaseManager;
this.registryDatabase = null;
this.storageMode = 'json'; // 默认使用JSON存储
}
/**
@@ -31,17 +40,61 @@ class DownloadRegistry {
try {
// 确保数据目录存在
await fs.ensureDir(this.dataPath);
// 加载现有注册表
await this.loadRegistry();
logger.info(`下载记录注册表初始化完成，总共包含${Object.keys(this.registry.artists).length}个作者，${this.getTotalArtworkCount()}个作品`);
// 获取存储模式配置
await this.loadStorageMode();
// 根据存储模式初始化相应的存储系统
if (this.storageMode === 'database' && this.databaseManager) {
await this.initDatabaseStorage();
} else {
await this.initJsonStorage();
}
const stats = await this.getStats();
logger.info(`下载记录注册表初始化完成（${this.storageMode}模式），总共包含${stats.artistCount}个作者，${stats.artworkCount}个作品`);
} catch (error) {
logger.error('下载记录注册表初始化失败:', error);
throw error;
}
}
/**
* 加载存储模式配置
*/
async loadStorageMode() {
try {
const cacheConfig = await this.cacheConfigManager.loadConfig();
this.storageMode = cacheConfig.download?.storageMode || 'json';
logger.debug(`存储模式配置: ${this.storageMode}`);
} catch (error) {
logger.warn('加载存储模式配置失败，使用默认JSON模式:', error.message);
this.storageMode = 'json';
}
}
/**
* 初始化数据库存储
*/
async initDatabaseStorage() {
if (!this.databaseManager) {
throw new Error('数据库管理器未提供，无法使用数据库存储模式');
}
this.registryDatabase = new RegistryDatabase(this.databaseManager);
await this.registryDatabase.init();
this.loaded = true;
logger.info('数据库存储模式初始化完成');
}
/**
* 初始化JSON存储
*/
async initJsonStorage() {
await this.loadRegistry();
logger.info('JSON存储模式初始化完成');
}
/**
* 加载注册表文件
*/
@@ -49,14 +102,14 @@ class DownloadRegistry {
try {
if (await fs.pathExists(this.registryPath)) {
const data = await fs.readJson(this.registryPath);
// 验证数据格式
if (data && typeof data === 'object' && data.artists) {
this.registry = {
version: data.version || '1.0.0',
created_at: data.created_at || new Date().toISOString(),
updated_at: data.updated_at || new Date().toISOString(),
artists: data.artists || {}
artists: data.artists || {},
};
} else {
logger.warn('注册表文件格式不正确，使用默认格式');
@@ -64,7 +117,7 @@ class DownloadRegistry {
} else {
logger.info('注册表文件不存在，将创建新的注册表');
}
this.loaded = true;
} catch (error) {
logger.error('加载注册表文件失败:', error);
@@ -94,25 +147,32 @@ class DownloadRegistry {
*/
async addArtwork(artistName, artworkId) {
if (!this.loaded) {
await this.loadRegistry();
await this.init();
}
const normalizedArtistName = this.normalizeArtistName(artistName);
const normalizedArtworkId = parseInt(artworkId);
if (!this.registry.artists[normalizedArtistName]) {
this.registry.artists[normalizedArtistName] = {
artworks: []
};
}
if (this.storageMode === 'database' && this.registryDatabase) {
// 使用数据库存储
await this.registryDatabase.addArtwork(normalizedArtistName, normalizedArtworkId);
logger.debug('添加作品记录到数据库', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
} else {
// 使用JSON存储
if (!this.registry.artists[normalizedArtistName]) {
this.registry.artists[normalizedArtistName] = {
artworks: [],
};
}
// 检查是否已存在
if (!this.registry.artists[normalizedArtistName].artworks.includes(normalizedArtworkId)) {
this.registry.artists[normalizedArtistName].artworks.push(normalizedArtworkId);
this.registry.artists[normalizedArtistName].artworks.sort((a, b) => b - a); // 按ID倒序排列
await this.saveRegistry();
logger.debug('添加作品记录', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
// 检查是否已存在
if (!this.registry.artists[normalizedArtistName].artworks.includes(normalizedArtworkId)) {
this.registry.artists[normalizedArtistName].artworks.push(normalizedArtworkId);
this.registry.artists[normalizedArtistName].artworks.sort((a, b) => b - a); // 按ID倒序排列
await this.saveRegistry();
logger.debug('添加作品记录到JSON', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
}
}
}
@@ -123,45 +183,52 @@ class DownloadRegistry {
*/
async removeArtwork(artistName, artworkId) {
if (!this.loaded) {
await this.loadRegistry();
await this.init();
}
const normalizedArtistName = this.normalizeArtistName(artistName);
const normalizedArtworkId = parseInt(artworkId);
logger.debug('开始移除作品记录', {
originalArtistName: artistName,
normalizedArtistName: normalizedArtistName,
artworkId: normalizedArtworkId
logger.debug('开始移除作品记录', {
originalArtistName: artistName,
normalizedArtistName: normalizedArtistName,
artworkId: normalizedArtworkId,
});
if (this.registry.artists[normalizedArtistName]) {
const artworks = this.registry.artists[normalizedArtistName].artworks;
const index = artworks.indexOf(normalizedArtworkId);
logger.debug('查找作品在注册表中的位置', {
artistName: normalizedArtistName,
artworkId: normalizedArtworkId,
index: index,
artworks: artworks
});
if (index !== -1) {
artworks.splice(index, 1);
// 如果作者下没有作品了，删除作者记录
if (artworks.length === 0) {
delete this.registry.artists[normalizedArtistName];
logger.info('作者下无作品，删除作者记录', { artistName: normalizedArtistName });
}
await this.saveRegistry();
logger.debug('成功移除作品记录', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
} else {
logger.warn('作品在注册表中未找到', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
}
if (this.storageMode === 'database' && this.registryDatabase) {
// 使用数据库存储
await this.registryDatabase.removeArtwork(normalizedArtistName, normalizedArtworkId);
logger.debug('从数据库移除作品记录', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
} else {
logger.warn('作者在注册表中未找到', { artistName: normalizedArtistName });
// 使用JSON存储
if (this.registry.artists[normalizedArtistName]) {
const artworks = this.registry.artists[normalizedArtistName].artworks;
const index = artworks.indexOf(normalizedArtworkId);
logger.debug('查找作品在注册表中的位置', {
artistName: normalizedArtistName,
artworkId: normalizedArtworkId,
index: index,
artworks: artworks,
});
if (index !== -1) {
artworks.splice(index, 1);
// 如果作者下没有作品了，删除作者记录
if (artworks.length === 0) {
delete this.registry.artists[normalizedArtistName];
logger.info('作者下无作品，删除作者记录', { artistName: normalizedArtistName });
}
await this.saveRegistry();
logger.debug('成功移除作品记录', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
} else {
logger.warn('作品在注册表中未找到', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
}
} else {
logger.warn('作者在注册表中未找到', { artistName: normalizedArtistName });
}
}
}
@@ -172,19 +239,25 @@ class DownloadRegistry {
*/
async isArtworkDownloaded(artworkId) {
if (!this.loaded) {
await this.loadRegistry();
await this.init();
}
const normalizedArtworkId = parseInt(artworkId);
// 遍历所有作者查找作品
for (const artistName in this.registry.artists) {
if (this.registry.artists[artistName].artworks.includes(normalizedArtworkId)) {
return true;
if (this.storageMode === 'database' && this.registryDatabase) {
// 使用数据库存储
return await this.registryDatabase.isArtworkDownloaded(normalizedArtworkId);
} else {
// 使用JSON存储
// 遍历所有作者查找作品
for (const artistName in this.registry.artists) {
if (this.registry.artists[artistName].artworks.includes(normalizedArtworkId)) {
return true;
}
}
return false;
}
return false;
}
/**
@@ -235,13 +308,13 @@ class DownloadRegistry {
}
const artworkIds = new Set();
for (const artistName in this.registry.artists) {
for (const artworkId of this.registry.artists[artistName].artworks) {
artworkIds.add(artworkId);
}
}
return Array.from(artworkIds).sort((a, b) => b - a);
}
@@ -252,16 +325,22 @@ class DownloadRegistry {
*/
async getArtistArtworks(artistName) {
if (!this.loaded) {
await this.loadRegistry();
await this.init();
}
const normalizedArtistName = this.normalizeArtistName(artistName);
if (this.registry.artists[normalizedArtistName]) {
return [...this.registry.artists[normalizedArtistName].artworks];
if (this.storageMode === 'database' && this.registryDatabase) {
// 使用数据库存储
return await this.registryDatabase.getArtistArtworks(normalizedArtistName);
} else {
// 使用JSON存储
if (this.registry.artists[normalizedArtistName]) {
return [...this.registry.artists[normalizedArtistName].artworks];
}
return [];
}
return [];
}
/**
@@ -270,10 +349,16 @@ class DownloadRegistry {
*/
async getDownloadedArtists() {
if (!this.loaded) {
await this.loadRegistry();
await this.init();
}
return Object.keys(this.registry.artists);
if (this.storageMode === 'database' && this.registryDatabase) {
// 使用数据库存储
return await this.registryDatabase.getDownloadedArtists();
} else {
// 使用JSON存储
return Object.keys(this.registry.artists);
}
}
/**
@@ -281,19 +366,25 @@ class DownloadRegistry {
*/
async getStats() {
if (!this.loaded) {
await this.loadRegistry();
await this.init();
}
const artists = Object.keys(this.registry.artists);
const totalArtworks = this.getTotalArtworkCount();
return {
artistCount: artists.length,
artworkCount: totalArtworks,
version: this.registry.version,
created_at: this.registry.created_at,
updated_at: this.registry.updated_at
};
if (this.storageMode === 'database' && this.registryDatabase) {
// 使用数据库存储
return await this.registryDatabase.getStats();
} else {
// 使用JSON存储
const artists = Object.keys(this.registry.artists);
const totalArtworks = this.getTotalArtworkCount();
return {
artistCount: artists.length,
artworkCount: totalArtworks,
version: this.registry.version,
created_at: this.registry.created_at,
updated_at: this.registry.updated_at,
};
}
}
/**
@@ -307,7 +398,7 @@ class DownloadRegistry {
return {
...this.registry,
exported_at: new Date().toISOString()
exported_at: new Date().toISOString(),
};
}
@@ -332,14 +423,14 @@ class DownloadRegistry {
for (const artistName in importData.artists) {
const normalizedArtistName = this.normalizeArtistName(artistName);
const importArtworks = importData.artists[artistName].artworks || [];
if (!this.registry.artists[normalizedArtistName]) {
this.registry.artists[normalizedArtistName] = { artworks: [] };
addedArtists++;
}
const existingArtworks = new Set(this.registry.artists[normalizedArtistName].artworks);
for (const artworkId of importArtworks) {
const normalizedArtworkId = parseInt(artworkId);
if (!existingArtworks.has(normalizedArtworkId)) {
@@ -349,7 +440,7 @@ class DownloadRegistry {
skippedArtworks++;
}
}
// 排序
this.registry.artists[normalizedArtistName].artworks.sort((a, b) => b - a);
}
@@ -361,27 +452,29 @@ class DownloadRegistry {
addedArtworks,
skippedArtworks,
totalArtists: Object.keys(this.registry.artists).length,
totalArtworks: this.getTotalArtworkCount()
totalArtworks: this.getTotalArtworkCount(),
};
logger.info(`注册表导入完成，导入了 ${result.addedArtists} 个作者，${result.addedArtworks} 个作品，跳过了 ${result.skippedArtworks} 个重复作品。当前总计：${result.totalArtists} 个作者，${result.totalArtworks} 个作品`);
logger.info(
`注册表导入完成，导入了 ${result.addedArtists} 个作者，${result.addedArtworks} 个作品，跳过了 ${result.skippedArtworks} 个重复作品。当前总计：${result.totalArtists} 个作者，${result.totalArtworks} 个作品`
);
return result;
}
/**
* 从文件系统重建注册表
* @param {FileManager} fileManager
* @param {FileManager} fileManager
* @param {string} taskId - 任务ID，用于更新进度
* @returns {Promise<{scannedArtists: number, scannedArtworks: number, addedArtworks: number, skippedArtworks: number}>}
*/
async rebuildFromFileSystem(fileManager, taskId = null) {
logger.info('开始从文件系统重建下载注册表...');
const stats = {
scannedArtists: 0,
scannedArtworks: 0,
addedArtworks: 0,
skippedArtworks: 0
skippedArtworks: 0,
};
// 获取所有艺术家目录
@@ -397,8 +490,8 @@ class DownloadRegistry {
...task,
progress: {
...stats,
currentArtist
}
currentArtist,
},
});
}
// 检查是否被取消
@@ -412,20 +505,20 @@ class DownloadRegistry {
try {
stats.scannedArtists++;
updateProgress(artistDir);
logger.info(`扫描艺术家目录: ${artistDir}`);
// 获取艺术家目录下的所有作品目录
const artworkDirs = await fileManager.getArtworkDirectories(artistDir);
for (const artworkDir of artworkDirs) {
try {
stats.scannedArtworks++;
updateProgress(artistDir);
// 检查作品是否已在注册表中
const isRegistered = await this.isArtworkRegistered(artistDir, artworkDir);
if (!isRegistered) {
// 从作品目录名中提取作品ID并添加到注册表
const artworkId = await this.extractArtworkIdFromDir(artworkDir);
@@ -437,17 +530,15 @@ class DownloadRegistry {
} else {
stats.skippedArtworks++;
}
// 每处理10个作品更新一次进度
if (stats.scannedArtworks % 10 === 0) {
updateProgress(artistDir);
}
} catch (error) {
logger.warn(`处理作品目录失败 ${artistDir}/${artworkDir}:`, error.message);
}
}
} catch (error) {
logger.warn(`处理艺术家目录失败 ${artistDir}:`, error.message);
}
@@ -455,7 +546,7 @@ class DownloadRegistry {
// 最终更新进度
updateProgress(null);
logger.info('从文件系统重建下载注册表完成', stats);
return stats;
}
@@ -472,7 +563,7 @@ class DownloadRegistry {
}
logger.info('开始清理注册表...');
let removedArtists = 0;
let removedArtworks = 0;
const downloadPath = await fileManager.getDownloadPath();
@@ -484,18 +575,18 @@ class DownloadRegistry {
for (const artworkId of artworks) {
// 检查作品目录是否存在
let found = false;
try {
const artistPath = path.join(downloadPath, artistName);
if (await fileManager.directoryExists(artistPath)) {
const artworkEntries = await fileManager.listDirectory(artistPath);
for (const entry of artworkEntries) {
const extractedArtworkId = await artworkUtils.extractArtworkIdFromDir(entry);
if (extractedArtworkId && extractedArtworkId === artworkId) {
const artworkPath = path.join(artistPath, entry);
const infoPath = path.join(artworkPath, 'artwork_info.json');
// 检查信息文件是否存在
if (await fs.pathExists(infoPath)) {
found = true;
@@ -531,7 +622,7 @@ class DownloadRegistry {
removedArtists,
removedArtworks,
remainingArtists: Object.keys(this.registry.artists).length,
remainingArtworks: this.getTotalArtworkCount()
remainingArtworks: this.getTotalArtworkCount(),
};
logger.info('注册表清理完成', result);
@@ -571,4 +662,4 @@ class DownloadRegistry {
}
}
module.exports = DownloadRegistry;
module.exports = DownloadRegistry;
backend/services/download.js
+3 -2
View File
@@ -20,11 +20,12 @@ const logger = defaultLogger.child('DownloadService');
* 下载服务 - 主服务类，协调各个管理器
*/
class DownloadService {
constructor(auth) {
constructor(auth, databaseManager = null) {
this.auth = auth;
this.artworkService = new ArtworkService(auth);
this.artistService = new ArtistService(auth);
this.cacheConfigManager = new CacheConfigManager();
this.databaseManager = databaseManager;
// 检测是否在pkg打包环境中运行
const isPkg = process.pkg !== undefined;
@@ -42,7 +43,7 @@ class DownloadService {
this.taskManager = new TaskManager(this.dataPath);
this.progressManager = new ProgressManager();
this.historyManager = new HistoryManager(this.dataPath);
this.downloadRegistry = new DownloadRegistry(this.dataPath);
this.downloadRegistry = new DownloadRegistry(this.dataPath, this.databaseManager);
// 先创建下载执行器，稍后在init方法中设置downloadService引用
this.downloadExecutor = new DownloadExecutor(this.fileManager, this.taskManager, this.progressManager, this.historyManager, this);
backend/services/registry-migration.js
+461
View File
@@ -0,0 +1,461 @@
const { defaultLogger } = require('../utils/logger');
const fs = require('fs-extra');
const path = require('path');
const logger = defaultLogger.child('RegistryMigration');
/**
* 注册表数据迁移服务
* 处理JSON和数据库之间的数据转换
*/
class RegistryMigration {
constructor(jsonRegistry, databaseRegistry, fileManager) {
this.jsonRegistry = jsonRegistry;
this.databaseRegistry = databaseRegistry;
this.fileManager = fileManager;
}
/**
* 从JSON迁移到数据库
* @param {boolean} overwrite 是否覆盖现有数据
* @returns {Object} 迁移结果
*/
async migrateJsonToDatabase(overwrite = true) {
logger.info('开始从JSON迁移数据到数据库...');
try {
// 确保JSON注册表已加载
if (!this.jsonRegistry.loaded) {
await this.jsonRegistry.init();
}
// 初始化数据库注册表
await this.databaseRegistry.init();
// 如果需要覆盖，先清空数据库
if (overwrite) {
logger.info('清空数据库中的现有数据...');
await this.clearDatabaseRegistry();
}
// 获取JSON数据
const jsonData = await this.jsonRegistry.exportRegistry();
if (!jsonData || !jsonData.artists) {
throw new Error('JSON注册表数据为空或格式不正确');
}
// 导入到数据库
const importResult = await this.databaseRegistry.importRegistry(jsonData);
const result = {
success: true,
direction: 'json-to-db',
recordsProcessed: importResult.addedArtworks,
addedArtists: importResult.addedArtists,
addedArtworks: importResult.addedArtworks,
skippedArtworks: importResult.skippedArtworks,
totalArtists: importResult.totalArtists,
totalArtworks: importResult.totalArtworks,
message: `成功从JSON迁移到数据库，处理了${importResult.addedArtists}个作者，${importResult.addedArtworks}个作品`
};
logger.info('JSON到数据库迁移完成', result);
return result;
} catch (error) {
logger.error('JSON到数据库迁移失败:', error);
throw error;
}
}
/**
* 从数据库迁移到JSON
* @param {boolean} overwrite 是否覆盖现有数据
* @returns {Object} 迁移结果
*/
async migrateDatabaseToJson(overwrite = true) {
logger.info('开始从数据库迁移数据到JSON...');
try {
// 确保数据库注册表已初始化
await this.databaseRegistry.init();
// 确保JSON注册表已加载
if (!this.jsonRegistry.loaded) {
await this.jsonRegistry.init();
}
// 获取数据库数据
const dbData = await this.databaseRegistry.exportRegistry();
if (!dbData || !dbData.artists) {
throw new Error('数据库注册表数据为空或格式不正确');
}
// 如果需要覆盖，先清空JSON注册表
if (overwrite) {
logger.info('清空JSON注册表中的现有数据...');
this.jsonRegistry.artists = {};
this.jsonRegistry.version = dbData.version || '1.0.5';
this.jsonRegistry.created_at = dbData.created_at || new Date().toISOString();
}
// 统计信息
let addedArtists = 0;
let addedArtworks = 0;
let skippedArtworks = 0;
// 导入数据到JSON注册表
for (const artistName in dbData.artists) {
const artistData = dbData.artists[artistName];
const artworks = artistData.artworks || [];
// 检查艺术家是否已存在
const isNewArtist = !this.jsonRegistry.artists[artistName];
if (isNewArtist) {
this.jsonRegistry.artists[artistName] = { artworks: [] };
addedArtists++;
}
// 获取现有作品ID
const existingArtworkIds = new Set(this.jsonRegistry.artists[artistName].artworks);
// 添加新作品
for (const artworkId of artworks) {
if (!existingArtworkIds.has(artworkId)) {
this.jsonRegistry.artists[artistName].artworks.push(artworkId);
addedArtworks++;
} else {
skippedArtworks++;
}
}
// 排序作品ID
this.jsonRegistry.artists[artistName].artworks.sort((a, b) => b - a);
}
// 更新时间戳
this.jsonRegistry.updated_at = new Date().toISOString();
// 保存JSON注册表
await this.jsonRegistry.save();
// 获取最终统计
const finalStats = await this.jsonRegistry.getStats();
const result = {
success: true,
direction: 'db-to-json',
recordsProcessed: addedArtworks,
addedArtists,
addedArtworks,
skippedArtworks,
totalArtists: finalStats.totalArtists,
totalArtworks: finalStats.totalArtworks,
message: `成功从数据库迁移到JSON，处理了${addedArtists}个作者，${addedArtworks}个作品`
};
logger.info('数据库到JSON迁移完成', result);
return result;
} catch (error) {
logger.error('数据库到JSON迁移失败:', error);
throw error;
}
}
/**
* 清空数据库注册表
*/
async clearDatabaseRegistry() {
try {
const db = this.databaseRegistry.db;
// 删除所有作品记录
await db.query('DELETE FROM registry_artworks');
// 删除所有艺术家记录
await db.query('DELETE FROM registry_artists');
// 重置自增ID
await db.query('ALTER TABLE registry_artists AUTO_INCREMENT = 1');
await db.query('ALTER TABLE registry_artworks AUTO_INCREMENT = 1');
logger.info('数据库注册表已清空');
} catch (error) {
logger.error('清空数据库注册表失败:', error);
throw error;
}
}
/**
* 比较JSON和数据库注册表的差异
* @returns {Object} 差异报告
*/
async compareRegistries() {
logger.info('开始比较JSON和数据库注册表...');
try {
// 确保两个注册表都已初始化
if (!this.jsonRegistry.loaded) {
await this.jsonRegistry.init();
}
await this.databaseRegistry.init();
// 获取统计信息
const jsonStats = await this.jsonRegistry.getStats();
const dbStats = await this.databaseRegistry.getStats();
// 获取艺术家列表
const jsonArtists = await this.jsonRegistry.getDownloadedArtists();
const dbArtists = await this.databaseRegistry.getDownloadedArtists();
// 计算差异
const jsonArtistSet = new Set(jsonArtists);
const dbArtistSet = new Set(dbArtists);
const onlyInJson = jsonArtists.filter(artist => !dbArtistSet.has(artist));
const onlyInDb = dbArtists.filter(artist => !jsonArtistSet.has(artist));
const commonArtists = jsonArtists.filter(artist => dbArtistSet.has(artist));
// 比较共同艺术家的作品差异
let artworkDifferences = [];
for (const artist of commonArtists.slice(0, 10)) { // 限制比较数量以避免性能问题
const jsonArtworks = await this.jsonRegistry.getArtistArtworks(artist);
const dbArtworks = await this.databaseRegistry.getArtistArtworks(artist);
const jsonArtworkSet = new Set(jsonArtworks);
const dbArtworkSet = new Set(dbArtworks);
const onlyInJsonArtworks = jsonArtworks.filter(id => !dbArtworkSet.has(id));
const onlyInDbArtworks = dbArtworks.filter(id => !jsonArtworkSet.has(id));
if (onlyInJsonArtworks.length > 0 || onlyInDbArtworks.length > 0) {
artworkDifferences.push({
artist,
onlyInJson: onlyInJsonArtworks.length,
onlyInDb: onlyInDbArtworks.length,
jsonTotal: jsonArtworks.length,
dbTotal: dbArtworks.length
});
}
}
const comparison = {
json: {
artists: jsonStats.totalArtists,
artworks: jsonStats.totalArtworks,
version: jsonStats.version,
updated_at: jsonStats.updated_at
},
database: {
artists: dbStats.artistCount,
artworks: dbStats.artworkCount,
version: dbStats.version,
updated_at: dbStats.updated_at
},
differences: {
artistsOnlyInJson: onlyInJson.length,
artistsOnlyInDb: onlyInDb.length,
commonArtists: commonArtists.length,
artworkDifferences: artworkDifferences.length,
sampleArtworkDifferences: artworkDifferences.slice(0, 5)
},
recommendation: this.getRecommendation(jsonStats, dbStats, onlyInJson, onlyInDb)
};
logger.info('注册表比较完成', {
jsonArtists: jsonStats.totalArtists,
dbArtists: dbStats.artistCount,
differences: comparison.differences
});
return comparison;
} catch (error) {
logger.error('比较注册表失败:', error);
throw error;
}
}
/**
* 根据比较结果给出建议
*/
getRecommendation(jsonStats, dbStats, onlyInJson, onlyInDb) {
if (jsonStats.totalArtworks === 0 && dbStats.artworkCount === 0) {
return '两个注册表都为空，无需迁移';
}
if (jsonStats.totalArtworks === 0) {
return '建议从数据库迁移到JSON';
}
if (dbStats.artworkCount === 0) {
return '建议从JSON迁移到数据库';
}
if (jsonStats.totalArtworks > dbStats.artworkCount) {
return 'JSON注册表包含更多数据，建议从JSON迁移到数据库';
}
if (dbStats.artworkCount > jsonStats.totalArtworks) {
return '数据库注册表包含更多数据，建议从数据库迁移到JSON';
}
if (onlyInJson.length > onlyInDb.length) {
return 'JSON注册表有更多独有艺术家，建议从JSON迁移到数据库';
}
if (onlyInDb.length > onlyInJson.length) {
return '数据库注册表有更多独有艺术家，建议从数据库迁移到JSON';
}
return '两个注册表数据相似，可根据需要选择迁移方向';
}
/**
* 验证迁移结果
* @param {string} direction 迁移方向
* @returns {Object} 验证结果
*/
async validateMigration(direction) {
logger.info(`验证${direction}迁移结果...`);
try {
const comparison = await this.compareRegistries();
let isValid = false;
let message = '';
if (direction === 'json-to-db') {
// 验证数据库是否包含JSON的所有数据
isValid = comparison.database.artworks >= comparison.json.artworks &&
comparison.differences.artistsOnlyInJson === 0;
message = isValid ?
'迁移验证成功：数据库包含了JSON的所有数据' :
'迁移验证失败：数据库缺少部分JSON数据';
} else if (direction === 'db-to-json') {
// 验证JSON是否包含数据库的所有数据
isValid = comparison.json.artworks >= comparison.database.artworks &&
comparison.differences.artistsOnlyInDb === 0;
message = isValid ?
'迁移验证成功：JSON包含了数据库的所有数据' :
'迁移验证失败：JSON缺少部分数据库数据';
}
const result = {
valid: isValid,
message,
comparison
};
logger.info('迁移验证完成', { valid: isValid, message });
return result;
} catch (error) {
logger.error('验证迁移结果失败:', error);
throw error;
}
}
/**
* 创建数据备份
* @param {string} type 备份类型 ('json' | 'database')
* @returns {string} 备份文件路径
*/
async createBackup(type) {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
// 检测是否在pkg打包环境中运行
const isPkg = process.pkg !== undefined;
const backupDir = isPkg
? path.join(process.cwd(), 'data', 'backups', 'registry') // 打包环境：当前工作目录的data文件夹
: path.join(__dirname, '..', '..', 'backups', 'registry'); // 开发环境：项目根目录的backups文件夹
await fs.ensureDir(backupDir);
try {
if (type === 'json') {
// 备份JSON注册表
const jsonData = await this.jsonRegistry.exportRegistry();
const backupPath = path.join(backupDir, `registry-json-backup-${timestamp}.json`);
await fs.writeJson(backupPath, jsonData, { spaces: 2 });
logger.info(`JSON注册表备份已创建: ${backupPath}`);
return backupPath;
} else if (type === 'database') {
// 备份数据库注册表
const dbData = await this.databaseRegistry.exportRegistry();
const backupPath = path.join(backupDir, `registry-db-backup-${timestamp}.json`);
await fs.writeJson(backupPath, dbData, { spaces: 2 });
logger.info(`数据库注册表备份已创建: ${backupPath}`);
return backupPath;
}
throw new Error(`不支持的备份类型: ${type}`);
} catch (error) {
logger.error(`创建${type}备份失败:`, error);
throw error;
}
}
/**
* 执行完整的迁移流程（包含备份和验证）
* @param {string} direction 迁移方向
* @param {boolean} overwrite 是否覆盖
* @param {boolean} createBackup 是否创建备份
* @returns {Object} 迁移结果
*/
async performMigration(direction, overwrite = true, createBackup = true) {
logger.info(`开始执行完整迁移流程: ${direction}`);
const result = {
success: false,
direction,
backupPath: null,
migrationResult: null,
validationResult: null,
error: null
};
try {
// 创建备份
if (createBackup) {
const backupType = direction === 'json-to-db' ? 'database' : 'json';
result.backupPath = await this.createBackup(backupType);
}
// 执行迁移
if (direction === 'json-to-db') {
result.migrationResult = await this.migrateJsonToDatabase(overwrite);
} else if (direction === 'db-to-json') {
result.migrationResult = await this.migrateDatabaseToJson(overwrite);
} else {
throw new Error(`不支持的迁移方向: ${direction}`);
}
// 验证迁移结果
result.validationResult = await this.validateMigration(direction);
result.success = result.migrationResult.success && result.validationResult.valid;
logger.info('完整迁移流程完成', {
direction,
success: result.success,
recordsProcessed: result.migrationResult.recordsProcessed
});
return result;
} catch (error) {
result.error = error.message;
logger.error('完整迁移流程失败:', error);
throw error;
}
}
}
module.exports = RegistryMigration;