注册表新增数据库存储同步
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"version": "1.0.5",
|
||||
"created_at": "2025-10-11T02:10:36.567Z",
|
||||
"updated_at": "2025-10-11T02:48:22.684Z",
|
||||
"exported_at": "2025-10-11T02:48:22.684Z",
|
||||
"artists": {}
|
||||
}
|
||||
@@ -39,7 +39,7 @@ class CacheConfigManager {
|
||||
retryDelay: 1000,
|
||||
},
|
||||
allowedExtensions: ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp'],
|
||||
// 新增并发下载配置
|
||||
// 下载检测配置
|
||||
download: {
|
||||
concurrentDownloads: 3, // 同时下载任务数
|
||||
maxConcurrentFiles: 5, // 单个任务内最大并发文件数
|
||||
@@ -52,6 +52,8 @@ class CacheConfigManager {
|
||||
useRegistryCheck: true, // 是否使用注册表检测（默认启用）
|
||||
fallbackToScan: false, // 检测失败时是否回退到扫盘检测
|
||||
maxFileSize: 50 * 1024 * 1024, // 最大文件大小 50MB
|
||||
// 存储模式配置
|
||||
storageMode: 'json', // 存储模式：'json' 或 'database'
|
||||
},
|
||||
// 新增Windows特定配置
|
||||
windows: {
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"host": "sywb.top",
|
||||
"port": 3306,
|
||||
"user": "pixiv",
|
||||
"password": "yT6LYysxB4HPPkZc",
|
||||
"database": "pixiv",
|
||||
"connectionLimit": 10,
|
||||
"ssl": false
|
||||
}
|
||||
+48
-1
@@ -29,6 +29,7 @@ class PixivBackend {
|
||||
this.auth = null;
|
||||
this.isLoggedIn = false;
|
||||
this.downloadService = null;
|
||||
this.databaseManager = null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -41,6 +42,9 @@ class PixivBackend {
|
||||
this.initConfig();
|
||||
this.config = this.readConfig();
|
||||
|
||||
// 自动加载数据库配置
|
||||
await this.initDatabaseConfig();
|
||||
|
||||
// 创建认证实例，传入代理配置
|
||||
this.auth = new PixivAuth(this.config.proxy);
|
||||
|
||||
@@ -63,7 +67,7 @@ class PixivBackend {
|
||||
}
|
||||
|
||||
// 创建下载服务实例
|
||||
this.downloadService = new DownloadService(this.auth);
|
||||
this.downloadService = new DownloadService(this.auth, this.databaseManager);
|
||||
await this.downloadService.init();
|
||||
|
||||
// 检查登录状态
|
||||
@@ -335,6 +339,49 @@ class PixivBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化数据库配置
|
||||
*/
|
||||
async initDatabaseConfig() {
|
||||
try {
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
|
||||
// 检测是否在pkg打包环境中运行
|
||||
const isPkg = process.pkg !== undefined;
|
||||
|
||||
const configPath = isPkg
|
||||
? path.join(process.cwd(), 'data', 'database.json') // 打包环境：当前工作目录的data文件夹
|
||||
: path.join(__dirname, '..', 'data', 'database.json'); // 开发环境：项目根目录的data文件夹
|
||||
|
||||
if (await fs.pathExists(configPath)) {
|
||||
const config = await fs.readJson(configPath);
|
||||
logger.info('检测到数据库配置文件，正在初始化数据库连接...');
|
||||
|
||||
// 动态导入数据库管理器
|
||||
const DatabaseManager = require('./database/database-manager');
|
||||
const RegistryDatabase = require('./database/registry-database');
|
||||
|
||||
// 创建并初始化数据库管理器
|
||||
this.databaseManager = new DatabaseManager();
|
||||
await this.databaseManager.init(config);
|
||||
|
||||
// 初始化注册表数据库
|
||||
const registryDatabase = new RegistryDatabase(this.databaseManager);
|
||||
|
||||
// 将实例设置到数据库路由模块中
|
||||
const databaseRoute = require('./routes/database');
|
||||
databaseRoute.setDatabaseInstances(this.databaseManager, registryDatabase);
|
||||
|
||||
logger.info('数据库连接已自动初始化');
|
||||
} else {
|
||||
logger.info('未检测到数据库配置文件，跳过数据库初始化');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('初始化数据库配置时出错:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取下载服务实例
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
const mysql = require('mysql2/promise');
|
||||
const { defaultLogger } = require('../utils/logger');
|
||||
|
||||
const logger = defaultLogger.child('DatabaseManager');
|
||||
|
||||
/**
|
||||
* 数据库连接管理器
|
||||
* 提供MySQL连接管理和基础CRUD操作
|
||||
*/
|
||||
class DatabaseManager {
|
||||
constructor() {
|
||||
this.pool = null;
|
||||
this.config = null;
|
||||
this.isConnected = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化数据库连接
|
||||
* @param {Object} config 数据库配置
|
||||
* @param {string} config.host 主机地址
|
||||
* @param {number} config.port 端口号
|
||||
* @param {string} config.user 用户名
|
||||
* @param {string} config.password 密码
|
||||
* @param {string} config.database 数据库名
|
||||
*/
|
||||
async init(config) {
|
||||
try {
|
||||
this.config = {
|
||||
host: config.host || 'localhost',
|
||||
port: config.port || 3306,
|
||||
user: config.user,
|
||||
password: config.password,
|
||||
database: config.database,
|
||||
waitForConnections: true,
|
||||
connectionLimit: 10,
|
||||
queueLimit: 0,
|
||||
charset: 'utf8mb4'
|
||||
};
|
||||
|
||||
// 创建连接池
|
||||
this.pool = mysql.createPool(this.config);
|
||||
|
||||
// 测试连接
|
||||
await this.testConnection();
|
||||
|
||||
this.isConnected = true;
|
||||
logger.info('数据库连接初始化成功');
|
||||
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('数据库连接初始化失败:', error);
|
||||
this.isConnected = false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试数据库连接
|
||||
*/
|
||||
async testConnection() {
|
||||
if (!this.pool) {
|
||||
throw new Error('数据库连接池未初始化');
|
||||
}
|
||||
|
||||
try {
|
||||
const connection = await this.pool.getConnection();
|
||||
await connection.ping();
|
||||
connection.release();
|
||||
logger.info('数据库连接测试成功');
|
||||
return { success: true, message: '数据库连接正常' };
|
||||
} catch (error) {
|
||||
logger.error('数据库连接测试失败:', error);
|
||||
throw new Error(`数据库连接失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行SQL查询
|
||||
* @param {string} sql SQL语句
|
||||
* @param {Array} params 参数
|
||||
*/
|
||||
async query(sql, params = []) {
|
||||
if (!this.pool) {
|
||||
throw new Error('数据库连接池未初始化');
|
||||
}
|
||||
|
||||
try {
|
||||
const [rows, fields] = await this.pool.execute(sql, params);
|
||||
return { success: true, data: rows, fields };
|
||||
} catch (error) {
|
||||
logger.error('SQL查询执行失败:', { sql, params, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行事务
|
||||
* @param {Function} callback 事务回调函数
|
||||
*/
|
||||
async transaction(callback) {
|
||||
if (!this.pool) {
|
||||
throw new Error('数据库连接池未初始化');
|
||||
}
|
||||
|
||||
const connection = await this.pool.getConnection();
|
||||
|
||||
try {
|
||||
await connection.beginTransaction();
|
||||
|
||||
const result = await callback(connection);
|
||||
|
||||
await connection.commit();
|
||||
connection.release();
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
await connection.rollback();
|
||||
connection.release();
|
||||
logger.error('事务执行失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 插入数据
|
||||
* @param {string} table 表名
|
||||
* @param {Object} data 数据对象
|
||||
*/
|
||||
async insert(table, data) {
|
||||
const keys = Object.keys(data);
|
||||
const values = Object.values(data);
|
||||
const placeholders = keys.map(() => '?').join(', ');
|
||||
|
||||
const sql = `INSERT INTO ${table} (${keys.join(', ')}) VALUES (${placeholders})`;
|
||||
|
||||
try {
|
||||
const result = await this.query(sql, values);
|
||||
return {
|
||||
success: true,
|
||||
insertId: result.data.insertId,
|
||||
affectedRows: result.data.affectedRows
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('插入数据失败:', { table, data, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新数据
|
||||
* @param {string} table 表名
|
||||
* @param {Object} data 更新数据
|
||||
* @param {Object} where 条件
|
||||
*/
|
||||
async update(table, data, where) {
|
||||
const setClause = Object.keys(data).map(key => `${key} = ?`).join(', ');
|
||||
const whereClause = Object.keys(where).map(key => `${key} = ?`).join(' AND ');
|
||||
|
||||
const sql = `UPDATE ${table} SET ${setClause} WHERE ${whereClause}`;
|
||||
const params = [...Object.values(data), ...Object.values(where)];
|
||||
|
||||
try {
|
||||
const result = await this.query(sql, params);
|
||||
return {
|
||||
success: true,
|
||||
affectedRows: result.data.affectedRows,
|
||||
changedRows: result.data.changedRows
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('更新数据失败:', { table, data, where, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除数据
|
||||
* @param {string} table 表名
|
||||
* @param {Object} where 条件
|
||||
*/
|
||||
async delete(table, where) {
|
||||
const whereClause = Object.keys(where).map(key => `${key} = ?`).join(' AND ');
|
||||
const sql = `DELETE FROM ${table} WHERE ${whereClause}`;
|
||||
const params = Object.values(where);
|
||||
|
||||
try {
|
||||
const result = await this.query(sql, params);
|
||||
return {
|
||||
success: true,
|
||||
affectedRows: result.data.affectedRows
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('删除数据失败:', { table, where, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询数据
|
||||
* @param {string} table 表名
|
||||
* @param {Object} where 条件
|
||||
* @param {Object} options 选项
|
||||
*/
|
||||
async select(table, where = {}, options = {}) {
|
||||
let sql = `SELECT * FROM ${table}`;
|
||||
let params = [];
|
||||
|
||||
if (Object.keys(where).length > 0) {
|
||||
const whereClause = Object.keys(where).map(key => `${key} = ?`).join(' AND ');
|
||||
sql += ` WHERE ${whereClause}`;
|
||||
params = Object.values(where);
|
||||
}
|
||||
|
||||
if (options.orderBy) {
|
||||
sql += ` ORDER BY ${options.orderBy}`;
|
||||
}
|
||||
|
||||
if (options.limit) {
|
||||
sql += ` LIMIT ${options.limit}`;
|
||||
}
|
||||
|
||||
if (options.offset) {
|
||||
sql += ` OFFSET ${options.offset}`;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.query(sql, params);
|
||||
return {
|
||||
success: true,
|
||||
data: result.data
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('查询数据失败:', { table, where, options, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查表是否存在
|
||||
* @param {string} tableName 表名
|
||||
*/
|
||||
async tableExists(tableName) {
|
||||
try {
|
||||
const sql = `
|
||||
SELECT COUNT(*) as count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = ? AND table_name = ?
|
||||
`;
|
||||
const result = await this.query(sql, [this.config.database, tableName]);
|
||||
return result.data[0].count > 0;
|
||||
} catch (error) {
|
||||
logger.error('检查表是否存在失败:', { tableName, error: error.message });
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建表
|
||||
* @param {string} tableName 表名
|
||||
* @param {string} createSQL 创建表的SQL语句
|
||||
*/
|
||||
async createTable(tableName, createSQL) {
|
||||
try {
|
||||
await this.query(createSQL);
|
||||
logger.info(`表 ${tableName} 创建成功`);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error(`创建表 ${tableName} 失败:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量插入数据
|
||||
* @param {string} table 表名
|
||||
* @param {Array} dataArray 数据数组
|
||||
*/
|
||||
async batchInsert(table, dataArray) {
|
||||
if (!dataArray || dataArray.length === 0) {
|
||||
return { success: true, affectedRows: 0 };
|
||||
}
|
||||
|
||||
const keys = Object.keys(dataArray[0]);
|
||||
const placeholders = keys.map(() => '?').join(', ');
|
||||
const sql = `INSERT INTO ${table} (${keys.join(', ')}) VALUES (${placeholders})`;
|
||||
|
||||
try {
|
||||
return await this.transaction(async (connection) => {
|
||||
let totalAffectedRows = 0;
|
||||
|
||||
for (const data of dataArray) {
|
||||
const values = keys.map(key => data[key]);
|
||||
const [result] = await connection.execute(sql, values);
|
||||
totalAffectedRows += result.affectedRows;
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
affectedRows: totalAffectedRows
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('批量插入数据失败:', { table, count: dataArray.length, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭数据库连接
|
||||
*/
|
||||
async close() {
|
||||
if (this.pool) {
|
||||
try {
|
||||
await this.pool.end();
|
||||
this.pool = null;
|
||||
this.isConnected = false;
|
||||
logger.info('数据库连接已关闭');
|
||||
} catch (error) {
|
||||
logger.error('关闭数据库连接失败:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取连接状态
|
||||
*/
|
||||
getConnectionStatus() {
|
||||
return {
|
||||
isConnected: this.isConnected,
|
||||
config: this.config ? {
|
||||
host: this.config.host,
|
||||
port: this.config.port,
|
||||
user: this.config.user,
|
||||
database: this.config.database
|
||||
} : null
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = DatabaseManager;
|
||||
@@ -0,0 +1,667 @@
|
||||
const { defaultLogger } = require('../utils/logger');
|
||||
const RegistrySchema = require('./registry-schema');
|
||||
|
||||
const logger = defaultLogger.child('RegistryDatabase');
|
||||
|
||||
/**
|
||||
* 数据库版本的下载注册表
|
||||
* 提供与JSON版本相同的接口，但使用MySQL数据库存储
|
||||
*/
|
||||
class RegistryDatabase {
|
||||
constructor(databaseManager) {
|
||||
this.db = databaseManager;
|
||||
this.schema = new RegistrySchema(databaseManager);
|
||||
this.loaded = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化数据库注册表
|
||||
*/
|
||||
async init() {
|
||||
try {
|
||||
// 初始化数据库表结构
|
||||
await this.schema.initializeTables();
|
||||
this.loaded = true;
|
||||
|
||||
const stats = await this.getStats();
|
||||
logger.info(`数据库注册表初始化完成，总共包含${stats.artistCount}个作者，${stats.artworkCount}个作品`);
|
||||
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('数据库注册表初始化失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 标准化作者名称
|
||||
*/
|
||||
normalizeArtistName(artistName) {
|
||||
if (!artistName || typeof artistName !== 'string') {
|
||||
return 'Unknown Artist';
|
||||
}
|
||||
return artistName.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取或创建艺术家记录
|
||||
* @param {string} artistName 艺术家名称
|
||||
* @returns {number} 艺术家ID
|
||||
*/
|
||||
async getOrCreateArtist(artistName) {
|
||||
const normalizedName = this.normalizeArtistName(artistName);
|
||||
|
||||
try {
|
||||
// 先尝试查找现有艺术家
|
||||
const existingResult = await this.db.select('registry_artists', {
|
||||
normalized_name: normalizedName
|
||||
});
|
||||
|
||||
if (existingResult.data.length > 0) {
|
||||
return existingResult.data[0].id;
|
||||
}
|
||||
|
||||
// 创建新艺术家
|
||||
const insertResult = await this.db.insert('registry_artists', {
|
||||
artist_name: artistName,
|
||||
normalized_name: normalizedName,
|
||||
artwork_count: 0
|
||||
});
|
||||
|
||||
return insertResult.insertId;
|
||||
} catch (error) {
|
||||
logger.error('获取或创建艺术家失败:', { artistName, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加作品到注册表
|
||||
* @param {string} artistName 艺术家名称
|
||||
* @param {number} artworkId 作品ID
|
||||
* @param {string} filePath 文件路径（可选）
|
||||
*/
|
||||
async addArtwork(artistName, artworkId, filePath = null) {
|
||||
try {
|
||||
const normalizedArtworkId = parseInt(artworkId);
|
||||
if (!normalizedArtworkId) {
|
||||
throw new Error('无效的作品ID');
|
||||
}
|
||||
|
||||
// 获取或创建艺术家
|
||||
const artistId = await this.getOrCreateArtist(artistName);
|
||||
|
||||
// 检查作品是否已存在
|
||||
const existingResult = await this.db.select('registry_artworks', {
|
||||
artist_id: artistId,
|
||||
artwork_id: normalizedArtworkId
|
||||
});
|
||||
|
||||
if (existingResult.data.length > 0) {
|
||||
logger.debug('作品已存在于注册表中', { artistName, artworkId: normalizedArtworkId });
|
||||
return;
|
||||
}
|
||||
|
||||
// 添加作品记录
|
||||
await this.db.insert('registry_artworks', {
|
||||
artist_id: artistId,
|
||||
artwork_id: normalizedArtworkId,
|
||||
artist_name: artistName,
|
||||
file_path: filePath,
|
||||
download_date: new Date()
|
||||
});
|
||||
|
||||
// 更新艺术家作品数量
|
||||
await this.updateArtistArtworkCount(artistId);
|
||||
|
||||
logger.debug('成功添加作品到注册表', { artistName, artworkId: normalizedArtworkId });
|
||||
} catch (error) {
|
||||
logger.error('添加作品到注册表失败:', { artistName, artworkId, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从注册表移除作品
|
||||
* @param {string} artistName 艺术家名称
|
||||
* @param {number} artworkId 作品ID
|
||||
*/
|
||||
async removeArtwork(artistName, artworkId) {
|
||||
try {
|
||||
const normalizedArtworkId = parseInt(artworkId);
|
||||
const normalizedArtistName = this.normalizeArtistName(artistName);
|
||||
|
||||
// 查找艺术家
|
||||
const artistResult = await this.db.select('registry_artists', {
|
||||
normalized_name: normalizedArtistName
|
||||
});
|
||||
|
||||
if (artistResult.data.length === 0) {
|
||||
logger.warn('艺术家在注册表中未找到', { artistName: normalizedArtistName });
|
||||
return;
|
||||
}
|
||||
|
||||
const artistId = artistResult.data[0].id;
|
||||
|
||||
// 删除作品记录
|
||||
const deleteResult = await this.db.delete('registry_artworks', {
|
||||
artist_id: artistId,
|
||||
artwork_id: normalizedArtworkId
|
||||
});
|
||||
|
||||
if (deleteResult.affectedRows > 0) {
|
||||
// 更新艺术家作品数量
|
||||
await this.updateArtistArtworkCount(artistId);
|
||||
logger.debug('成功移除作品记录', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
} else {
|
||||
logger.warn('作品在注册表中未找到', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('移除作品记录失败:', { artistName, artworkId, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新艺术家作品数量
|
||||
* @param {number} artistId 艺术家ID
|
||||
*/
|
||||
async updateArtistArtworkCount(artistId) {
|
||||
try {
|
||||
const countResult = await this.db.query(
|
||||
'SELECT COUNT(*) as count FROM registry_artworks WHERE artist_id = ?',
|
||||
[artistId]
|
||||
);
|
||||
|
||||
const count = countResult.data[0].count;
|
||||
|
||||
await this.db.update('registry_artists',
|
||||
{ artwork_count: count },
|
||||
{ id: artistId }
|
||||
);
|
||||
} catch (error) {
|
||||
logger.error('更新艺术家作品数量失败:', { artistId, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查作品是否已下载
|
||||
* @param {number|string} artworkId - 作品ID
|
||||
* @returns {boolean} 是否已下载
|
||||
*/
|
||||
async isArtworkDownloaded(artworkId) {
|
||||
try {
|
||||
const normalizedArtworkId = parseInt(artworkId);
|
||||
if (!normalizedArtworkId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const result = await this.db.query(`
|
||||
SELECT COUNT(*) as count
|
||||
FROM registry_artworks
|
||||
WHERE artwork_id = ?
|
||||
`, [normalizedArtworkId]);
|
||||
|
||||
return result.data[0].count > 0;
|
||||
} catch (error) {
|
||||
logger.error('检查作品下载状态失败:', { artworkId, error: error.message });
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查作品是否已在注册表中注册
|
||||
* @param {string} artistName 艺术家名称
|
||||
* @param {string} artworkDir 作品目录名或作品ID
|
||||
* @returns {boolean} 是否已注册
|
||||
*/
|
||||
async isArtworkRegistered(artistName, artworkDir) {
|
||||
try {
|
||||
// 从目录名提取作品ID
|
||||
let artworkId;
|
||||
if (typeof artworkDir === 'string' && isNaN(artworkDir)) {
|
||||
// 如果是目录名，需要提取ID
|
||||
const artworkUtils = require('../utils/artwork-utils');
|
||||
artworkId = await artworkUtils.extractArtworkIdFromDir(artworkDir);
|
||||
} else {
|
||||
artworkId = parseInt(artworkDir);
|
||||
}
|
||||
|
||||
if (!artworkId) {
|
||||
logger.warn(`无法从作品目录名中提取作品ID: ${artworkDir}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
const normalizedArtistName = this.normalizeArtistName(artistName);
|
||||
|
||||
// 查询数据库
|
||||
const result = await this.db.query(`
|
||||
SELECT COUNT(*) as count
|
||||
FROM registry_artworks ra
|
||||
JOIN registry_artists rt ON ra.artist_id = rt.id
|
||||
WHERE rt.normalized_name = ? AND ra.artwork_id = ?
|
||||
`, [normalizedArtistName, artworkId]);
|
||||
|
||||
return result.data[0].count > 0;
|
||||
} catch (error) {
|
||||
logger.error('检查作品注册状态失败:', { artistName, artworkDir, error: error.message });
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取已下载的作品ID列表
|
||||
* @returns {number[]} 作品ID数组
|
||||
*/
|
||||
async getDownloadedArtworkIds() {
|
||||
try {
|
||||
const result = await this.db.query(`
|
||||
SELECT DISTINCT artwork_id
|
||||
FROM registry_artworks
|
||||
ORDER BY artwork_id DESC
|
||||
`);
|
||||
|
||||
return result.data.map(row => row.artwork_id);
|
||||
} catch (error) {
|
||||
logger.error('获取已下载作品ID列表失败:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取指定作者的已下载作品
|
||||
* @param {string} artistName 作者名称
|
||||
* @returns {number[]} 作品ID数组
|
||||
*/
|
||||
async getArtistArtworks(artistName) {
|
||||
try {
|
||||
const normalizedArtistName = this.normalizeArtistName(artistName);
|
||||
|
||||
const result = await this.db.query(`
|
||||
SELECT ra.artwork_id
|
||||
FROM registry_artworks ra
|
||||
JOIN registry_artists rt ON ra.artist_id = rt.id
|
||||
WHERE rt.normalized_name = ?
|
||||
ORDER BY ra.artwork_id DESC
|
||||
`, [normalizedArtistName]);
|
||||
|
||||
return result.data.map(row => row.artwork_id);
|
||||
} catch (error) {
|
||||
logger.error('获取艺术家作品列表失败:', { artistName, error: error.message });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有已下载的作者列表
|
||||
* @returns {string[]} 作者名称数组
|
||||
*/
|
||||
async getDownloadedArtists() {
|
||||
try {
|
||||
const result = await this.db.query(`
|
||||
SELECT DISTINCT artist_name
|
||||
FROM registry_artists
|
||||
WHERE artwork_count > 0
|
||||
ORDER BY artist_name
|
||||
`);
|
||||
|
||||
return result.data.map(row => row.artist_name);
|
||||
} catch (error) {
|
||||
logger.error('获取已下载作者列表失败:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取统计信息
|
||||
*/
|
||||
async getStats() {
|
||||
try {
|
||||
// 获取艺术家数量
|
||||
const artistCountResult = await this.db.query(`
|
||||
SELECT COUNT(*) as count FROM registry_artists WHERE artwork_count > 0
|
||||
`);
|
||||
|
||||
// 获取作品数量
|
||||
const artworkCountResult = await this.db.query(`
|
||||
SELECT COUNT(*) as count FROM registry_artworks
|
||||
`);
|
||||
|
||||
// 获取版本信息
|
||||
const versionResult = await this.db.select('registry_meta', { meta_key: 'version' });
|
||||
const createdAtResult = await this.db.select('registry_meta', { meta_key: 'created_at' });
|
||||
|
||||
// 获取最后更新时间
|
||||
const lastUpdatedResult = await this.db.query(`
|
||||
SELECT MAX(updated_at) as last_updated FROM registry_artworks
|
||||
`);
|
||||
|
||||
return {
|
||||
artistCount: artistCountResult.data[0].count,
|
||||
artworkCount: artworkCountResult.data[0].count,
|
||||
version: versionResult.data[0]?.meta_value || '1.0.5',
|
||||
created_at: createdAtResult.data[0]?.meta_value || new Date().toISOString(),
|
||||
updated_at: lastUpdatedResult.data[0]?.last_updated || new Date().toISOString()
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('获取统计信息失败:', error);
|
||||
return {
|
||||
artistCount: 0,
|
||||
artworkCount: 0,
|
||||
version: '1.0.5',
|
||||
created_at: new Date().toISOString(),
|
||||
updated_at: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 导出注册表数据（转换为JSON格式）
|
||||
*/
|
||||
async exportRegistry() {
|
||||
try {
|
||||
const result = await this.db.query(`
|
||||
SELECT
|
||||
rt.artist_name,
|
||||
rt.normalized_name,
|
||||
GROUP_CONCAT(ra.artwork_id ORDER BY ra.artwork_id DESC) as artwork_ids
|
||||
FROM registry_artists rt
|
||||
LEFT JOIN registry_artworks ra ON rt.id = ra.artist_id
|
||||
WHERE rt.artwork_count > 0
|
||||
GROUP BY rt.id, rt.artist_name, rt.normalized_name
|
||||
`);
|
||||
|
||||
const artists = {};
|
||||
|
||||
for (const row of result.data) {
|
||||
const artworkIds = row.artwork_ids ?
|
||||
row.artwork_ids.split(',').map(id => parseInt(id)) : [];
|
||||
|
||||
artists[row.artist_name] = {
|
||||
artworks: artworkIds
|
||||
};
|
||||
}
|
||||
|
||||
const stats = await this.getStats();
|
||||
|
||||
return {
|
||||
version: stats.version,
|
||||
created_at: stats.created_at,
|
||||
updated_at: stats.updated_at,
|
||||
exported_at: new Date().toISOString(),
|
||||
artists: artists
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('导出注册表数据失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 导入注册表数据（从JSON格式）
|
||||
* @param {Object} importData 要导入的数据
|
||||
*/
|
||||
async importRegistry(importData) {
|
||||
if (!importData || !importData.artists) {
|
||||
throw new Error('导入数据格式不正确');
|
||||
}
|
||||
|
||||
let addedArtists = 0;
|
||||
let addedArtworks = 0;
|
||||
let skippedArtworks = 0;
|
||||
|
||||
try {
|
||||
await this.db.transaction(async (connection) => {
|
||||
for (const artistName in importData.artists) {
|
||||
const importArtworks = importData.artists[artistName].artworks || [];
|
||||
|
||||
// 获取或创建艺术家
|
||||
const artistId = await this.getOrCreateArtist(artistName);
|
||||
|
||||
// 检查是否是新艺术家
|
||||
const existingArtworkCount = await this.db.query(
|
||||
'SELECT COUNT(*) as count FROM registry_artworks WHERE artist_id = ?',
|
||||
[artistId]
|
||||
);
|
||||
|
||||
if (existingArtworkCount.data[0].count === 0) {
|
||||
addedArtists++;
|
||||
}
|
||||
|
||||
// 获取现有作品ID
|
||||
const existingArtworksResult = await this.db.select('registry_artworks', {
|
||||
artist_id: artistId
|
||||
});
|
||||
const existingArtworkIds = new Set(
|
||||
existingArtworksResult.data.map(row => row.artwork_id)
|
||||
);
|
||||
|
||||
// 添加新作品
|
||||
for (const artworkId of importArtworks) {
|
||||
const normalizedArtworkId = parseInt(artworkId);
|
||||
|
||||
if (!existingArtworkIds.has(normalizedArtworkId)) {
|
||||
await this.db.insert('registry_artworks', {
|
||||
artist_id: artistId,
|
||||
artwork_id: normalizedArtworkId,
|
||||
artist_name: artistName,
|
||||
download_date: new Date()
|
||||
});
|
||||
addedArtworks++;
|
||||
} else {
|
||||
skippedArtworks++;
|
||||
}
|
||||
}
|
||||
|
||||
// 更新艺术家作品数量
|
||||
await this.updateArtistArtworkCount(artistId);
|
||||
}
|
||||
});
|
||||
|
||||
const stats = await this.getStats();
|
||||
const result = {
|
||||
addedArtists,
|
||||
addedArtworks,
|
||||
skippedArtworks,
|
||||
totalArtists: stats.artistCount,
|
||||
totalArtworks: stats.artworkCount
|
||||
};
|
||||
|
||||
logger.info(`注册表导入完成，导入了 ${result.addedArtists} 个作者，${result.addedArtworks} 个作品，跳过了 ${result.skippedArtworks} 个重复作品。当前总计：${result.totalArtists} 个作者，${result.totalArtworks} 个作品`);
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error('导入注册表数据失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从文件系统重建注册表
|
||||
* @param {FileManager} fileManager
|
||||
* @param {string} taskId 任务ID
|
||||
*/
|
||||
async rebuildFromFileSystem(fileManager, taskId = null) {
|
||||
logger.info('开始从文件系统重建数据库注册表...');
|
||||
|
||||
const stats = {
|
||||
scannedArtists: 0,
|
||||
scannedArtworks: 0,
|
||||
addedArtworks: 0,
|
||||
skippedArtworks: 0
|
||||
};
|
||||
|
||||
// 进度更新函数
|
||||
const updateProgress = (currentArtist) => {
|
||||
if (taskId) {
|
||||
const progressManager = require('../services/progress-manager');
|
||||
progressManager.updateProgress(taskId, {
|
||||
...stats,
|
||||
currentArtist: currentArtist || ''
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
// 获取所有艺术家目录
|
||||
const artistDirs = await fileManager.getArtistDirectories();
|
||||
logger.info(`发现 ${artistDirs.length} 个艺术家目录`);
|
||||
|
||||
for (const artistDir of artistDirs) {
|
||||
try {
|
||||
stats.scannedArtists++;
|
||||
updateProgress(artistDir);
|
||||
|
||||
logger.info(`扫描艺术家目录: ${artistDir}`);
|
||||
|
||||
// 获取艺术家目录下的所有作品目录
|
||||
const artworkDirs = await fileManager.getArtworkDirectories(artistDir);
|
||||
|
||||
for (const artworkDir of artworkDirs) {
|
||||
try {
|
||||
stats.scannedArtworks++;
|
||||
updateProgress(artistDir);
|
||||
|
||||
// 检查作品是否已在注册表中
|
||||
const isRegistered = await this.isArtworkRegistered(artistDir, artworkDir);
|
||||
|
||||
if (!isRegistered) {
|
||||
// 从作品目录名中提取作品ID并添加到注册表
|
||||
const artworkUtils = require('../utils/artwork-utils');
|
||||
const artworkId = await artworkUtils.extractArtworkIdFromDir(artworkDir);
|
||||
if (artworkId) {
|
||||
await this.addArtwork(artistDir, artworkId);
|
||||
stats.addedArtworks++;
|
||||
logger.debug(`添加作品到注册表: ${artistDir}/${artworkDir}`);
|
||||
}
|
||||
} else {
|
||||
stats.skippedArtworks++;
|
||||
}
|
||||
|
||||
// 每处理10个作品更新一次进度
|
||||
if (stats.scannedArtworks % 10 === 0) {
|
||||
updateProgress(artistDir);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.warn(`处理作品目录失败 ${artistDir}/${artworkDir}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.warn(`处理艺术家目录失败 ${artistDir}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
// 最终更新进度
|
||||
updateProgress(null);
|
||||
|
||||
logger.info('从文件系统重建数据库注册表完成', stats);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
logger.error('从文件系统重建数据库注册表失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理注册表（移除不存在的记录）
|
||||
* @param {FileManager} fileManager 文件管理器实例
|
||||
*/
|
||||
async cleanupRegistry(fileManager) {
|
||||
try {
|
||||
logger.info('开始清理数据库注册表...');
|
||||
|
||||
let removedArtists = 0;
|
||||
let removedArtworks = 0;
|
||||
const downloadPath = await fileManager.getDownloadPath();
|
||||
|
||||
// 获取所有作品记录
|
||||
const artworksResult = await this.db.query(`
|
||||
SELECT ra.id, ra.artist_id, ra.artwork_id, rt.artist_name, rt.normalized_name
|
||||
FROM registry_artworks ra
|
||||
JOIN registry_artists rt ON ra.artist_id = rt.id
|
||||
`);
|
||||
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
const artworkUtils = require('../utils/artwork-utils');
|
||||
|
||||
for (const artwork of artworksResult.data) {
|
||||
let found = false;
|
||||
|
||||
try {
|
||||
const artistPath = path.join(downloadPath, artwork.artist_name);
|
||||
if (await fileManager.directoryExists(artistPath)) {
|
||||
const artworkEntries = await fileManager.listDirectory(artistPath);
|
||||
|
||||
for (const entry of artworkEntries) {
|
||||
const extractedArtworkId = await artworkUtils.extractArtworkIdFromDir(entry);
|
||||
if (extractedArtworkId && extractedArtworkId === artwork.artwork_id) {
|
||||
const artworkPath = path.join(artistPath, entry);
|
||||
const infoPath = path.join(artworkPath, 'artwork_info.json');
|
||||
|
||||
// 检查信息文件是否存在
|
||||
if (await fs.pathExists(infoPath)) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.debug(`检查作品 ${artwork.artwork_id} 时出错:`, error);
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
// 删除作品记录
|
||||
await this.db.delete('registry_artworks', { id: artwork.id });
|
||||
removedArtworks++;
|
||||
logger.debug(`移除无效作品记录: ${artwork.artist_name} - ${artwork.artwork_id}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新所有艺术家的作品数量
|
||||
const artistsResult = await this.db.select('registry_artists');
|
||||
for (const artist of artistsResult.data) {
|
||||
await this.updateArtistArtworkCount(artist.id);
|
||||
}
|
||||
|
||||
// 删除没有作品的艺术家
|
||||
const emptyArtistsResult = await this.db.select('registry_artists', { artwork_count: 0 });
|
||||
for (const artist of emptyArtistsResult.data) {
|
||||
await this.db.delete('registry_artists', { id: artist.id });
|
||||
removedArtists++;
|
||||
logger.debug(`移除空作者记录: ${artist.artist_name}`);
|
||||
}
|
||||
|
||||
const stats = await this.getStats();
|
||||
const result = {
|
||||
removedArtists,
|
||||
removedArtworks,
|
||||
remainingArtists: stats.artistCount,
|
||||
remainingArtworks: stats.artworkCount
|
||||
};
|
||||
|
||||
logger.info('数据库注册表清理完成', result);
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error('数据库注册表清理失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取总作品数量
|
||||
*/
|
||||
async getTotalArtworkCount() {
|
||||
try {
|
||||
const result = await this.db.query('SELECT COUNT(*) as count FROM registry_artworks');
|
||||
return result.data[0].count;
|
||||
} catch (error) {
|
||||
logger.error('获取总作品数量失败:', error);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RegistryDatabase;
|
||||
@@ -0,0 +1,241 @@
|
||||
const { defaultLogger } = require('../utils/logger');
|
||||
|
||||
const logger = defaultLogger.child('RegistrySchema');
|
||||
|
||||
/**
|
||||
* 注册表数据库表结构管理
|
||||
*/
|
||||
class RegistrySchema {
|
||||
constructor(databaseManager) {
|
||||
this.db = databaseManager;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化所有注册表相关的数据库表
|
||||
*/
|
||||
async initializeTables() {
|
||||
try {
|
||||
await this.createArtistsTable();
|
||||
await this.createArtworksTable();
|
||||
await this.createRegistryMetaTable();
|
||||
logger.info('注册表数据库表初始化完成');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('初始化注册表数据库表失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建艺术家表
|
||||
*/
|
||||
async createArtistsTable() {
|
||||
const tableName = 'registry_artists';
|
||||
|
||||
if (await this.db.tableExists(tableName)) {
|
||||
logger.debug(`表 ${tableName} 已存在，跳过创建`);
|
||||
return;
|
||||
}
|
||||
|
||||
const createSQL = `
|
||||
CREATE TABLE ${tableName} (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
artist_name VARCHAR(255) NOT NULL UNIQUE COMMENT '艺术家名称',
|
||||
normalized_name VARCHAR(255) NOT NULL COMMENT '标准化后的艺术家名称',
|
||||
artwork_count INT DEFAULT 0 COMMENT '作品数量',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
INDEX idx_artist_name (artist_name),
|
||||
INDEX idx_normalized_name (normalized_name)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='注册表艺术家表';
|
||||
`;
|
||||
|
||||
await this.db.createTable(tableName, createSQL);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建作品表
|
||||
*/
|
||||
async createArtworksTable() {
|
||||
const tableName = 'registry_artworks';
|
||||
|
||||
if (await this.db.tableExists(tableName)) {
|
||||
logger.debug(`表 ${tableName} 已存在，跳过创建`);
|
||||
return;
|
||||
}
|
||||
|
||||
const createSQL = `
|
||||
CREATE TABLE ${tableName} (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
artist_id INT NOT NULL COMMENT '艺术家ID',
|
||||
artwork_id BIGINT NOT NULL COMMENT '作品ID',
|
||||
artist_name VARCHAR(255) NOT NULL COMMENT '艺术家名称（冗余字段，便于查询）',
|
||||
file_path TEXT COMMENT '文件路径（可选）',
|
||||
download_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '下载时间',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
UNIQUE KEY uk_artist_artwork (artist_id, artwork_id),
|
||||
INDEX idx_artwork_id (artwork_id),
|
||||
INDEX idx_artist_id (artist_id),
|
||||
INDEX idx_artist_name (artist_name),
|
||||
INDEX idx_download_date (download_date),
|
||||
FOREIGN KEY (artist_id) REFERENCES registry_artists(id) ON DELETE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='注册表作品表';
|
||||
`;
|
||||
|
||||
await this.db.createTable(tableName, createSQL);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建注册表元数据表
|
||||
*/
|
||||
async createRegistryMetaTable() {
|
||||
const tableName = 'registry_meta';
|
||||
|
||||
if (await this.db.tableExists(tableName)) {
|
||||
logger.debug(`表 ${tableName} 已存在，跳过创建`);
|
||||
return;
|
||||
}
|
||||
|
||||
const createSQL = `
|
||||
CREATE TABLE ${tableName} (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
meta_key VARCHAR(100) NOT NULL UNIQUE COMMENT '元数据键',
|
||||
meta_value TEXT COMMENT '元数据值',
|
||||
description TEXT COMMENT '描述',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
INDEX idx_meta_key (meta_key)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='注册表元数据表';
|
||||
`;
|
||||
|
||||
await this.db.createTable(tableName, createSQL);
|
||||
|
||||
// 插入初始元数据
|
||||
await this.initializeMetaData();
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化元数据
|
||||
*/
|
||||
async initializeMetaData() {
|
||||
const metaData = [
|
||||
{
|
||||
meta_key: 'version',
|
||||
meta_value: '1.0.5',
|
||||
description: '注册表版本号'
|
||||
},
|
||||
{
|
||||
meta_key: 'storage_type',
|
||||
meta_value: 'database',
|
||||
description: '存储类型：database 或 json'
|
||||
},
|
||||
{
|
||||
meta_key: 'created_at',
|
||||
meta_value: new Date().toISOString(),
|
||||
description: '注册表创建时间'
|
||||
},
|
||||
{
|
||||
meta_key: 'last_migration',
|
||||
meta_value: new Date().toISOString(),
|
||||
description: '最后一次迁移时间'
|
||||
}
|
||||
];
|
||||
|
||||
for (const meta of metaData) {
|
||||
try {
|
||||
await this.db.insert('registry_meta', meta);
|
||||
} catch (error) {
|
||||
// 如果键已存在，忽略错误
|
||||
if (!error.message.includes('Duplicate entry')) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取表结构信息
|
||||
*/
|
||||
getTableInfo() {
|
||||
return {
|
||||
artists: {
|
||||
tableName: 'registry_artists',
|
||||
description: '存储艺术家信息',
|
||||
fields: {
|
||||
id: '主键ID',
|
||||
artist_name: '艺术家名称',
|
||||
normalized_name: '标准化后的艺术家名称',
|
||||
artwork_count: '作品数量',
|
||||
created_at: '创建时间',
|
||||
updated_at: '更新时间'
|
||||
}
|
||||
},
|
||||
artworks: {
|
||||
tableName: 'registry_artworks',
|
||||
description: '存储作品信息',
|
||||
fields: {
|
||||
id: '主键ID',
|
||||
artist_id: '艺术家ID（外键）',
|
||||
artwork_id: '作品ID',
|
||||
artist_name: '艺术家名称（冗余字段）',
|
||||
file_path: '文件路径',
|
||||
download_date: '下载时间',
|
||||
created_at: '创建时间',
|
||||
updated_at: '更新时间'
|
||||
}
|
||||
},
|
||||
meta: {
|
||||
tableName: 'registry_meta',
|
||||
description: '存储注册表元数据',
|
||||
fields: {
|
||||
id: '主键ID',
|
||||
meta_key: '元数据键',
|
||||
meta_value: '元数据值',
|
||||
description: '描述',
|
||||
created_at: '创建时间',
|
||||
updated_at: '更新时间'
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查所有表是否存在
|
||||
*/
|
||||
async checkTablesExist() {
|
||||
const tables = ['registry_artists', 'registry_artworks', 'registry_meta'];
|
||||
const results = {};
|
||||
|
||||
for (const table of tables) {
|
||||
results[table] = await this.db.tableExists(table);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除所有注册表相关的表（谨慎使用）
|
||||
*/
|
||||
async dropAllTables() {
|
||||
const tables = ['registry_artworks', 'registry_artists', 'registry_meta'];
|
||||
|
||||
try {
|
||||
// 按照外键依赖顺序删除表
|
||||
for (const table of tables) {
|
||||
if (await this.db.tableExists(table)) {
|
||||
await this.db.query(`DROP TABLE ${table}`);
|
||||
logger.info(`表 ${table} 已删除`);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('所有注册表相关的表已删除');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('删除注册表表失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RegistrySchema;
|
||||
@@ -0,0 +1,670 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const DatabaseManager = require('../database/database-manager');
|
||||
const RegistryDatabase = require('../database/registry-database');
|
||||
const RegistryMigration = require('../services/registry-migration');
|
||||
const { defaultLogger } = require('../utils/logger');
|
||||
|
||||
// 创建logger实例
|
||||
const logger = defaultLogger.child('DatabaseRouter');
|
||||
|
||||
// 全局数据库管理器实例
|
||||
let databaseManager = null;
|
||||
let registryDatabase = null;
|
||||
|
||||
/**
|
||||
* 设置数据库实例
|
||||
*/
|
||||
function setDatabaseInstances(dbManager, regDatabase) {
|
||||
databaseManager = dbManager;
|
||||
registryDatabase = regDatabase;
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试数据库连接
|
||||
* POST /api/database/test-connection
|
||||
*/
|
||||
router.post('/test-connection', async (req, res) => {
|
||||
try {
|
||||
const { host, port, user, password, database, ssl } = req.body;
|
||||
|
||||
if (!host || !port || !user || !database) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '缺少必要的连接参数'
|
||||
});
|
||||
}
|
||||
|
||||
// 创建临时数据库管理器进行测试
|
||||
const testManager = new DatabaseManager();
|
||||
|
||||
// 初始化连接池
|
||||
await testManager.init({
|
||||
host,
|
||||
port: parseInt(port),
|
||||
user,
|
||||
password,
|
||||
database,
|
||||
ssl: ssl || false
|
||||
});
|
||||
|
||||
const result = await testManager.testConnection();
|
||||
|
||||
if (result.success) {
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: '数据库连接成功',
|
||||
serverVersion: result.serverVersion,
|
||||
connectionTime: result.connectionTime
|
||||
}
|
||||
});
|
||||
} else {
|
||||
res.status(400).json({
|
||||
success: false,
|
||||
error: result.error
|
||||
});
|
||||
}
|
||||
|
||||
// 关闭测试连接
|
||||
await testManager.close();
|
||||
|
||||
} catch (error) {
|
||||
logger.error('测试数据库连接失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 保存数据库配置
|
||||
* POST /api/database/config
|
||||
*/
|
||||
router.post('/config', async (req, res) => {
|
||||
try {
|
||||
const { host, port, user, password, database, connectionLimit, ssl } = req.body;
|
||||
|
||||
if (!host || !port || !user || !database) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '缺少必要的连接参数'
|
||||
});
|
||||
}
|
||||
|
||||
const config = {
|
||||
host,
|
||||
port: parseInt(port),
|
||||
user,
|
||||
password,
|
||||
database,
|
||||
connectionLimit: parseInt(connectionLimit) || 10,
|
||||
ssl: ssl || false
|
||||
};
|
||||
|
||||
// 先测试连接
|
||||
const testManager = new DatabaseManager();
|
||||
await testManager.init(config);
|
||||
const testResult = await testManager.testConnection();
|
||||
|
||||
if (!testResult.success) {
|
||||
await testManager.close();
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: `连接测试失败: ${testResult.error}`
|
||||
});
|
||||
}
|
||||
|
||||
await testManager.close();
|
||||
|
||||
// 保存配置到文件
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
|
||||
// 检测是否在pkg打包环境中运行
|
||||
const isPkg = process.pkg !== undefined;
|
||||
|
||||
const configPath = isPkg
|
||||
? path.join(process.cwd(), 'data', 'database.json') // 打包环境：当前工作目录的data文件夹
|
||||
: path.join(__dirname, '..', '..', 'data', 'database.json'); // 开发环境：项目根目录的data文件夹
|
||||
|
||||
await fs.ensureDir(path.dirname(configPath));
|
||||
await fs.writeJson(configPath, config, { spaces: 2 });
|
||||
|
||||
// 重新初始化全局数据库管理器
|
||||
if (databaseManager) {
|
||||
await databaseManager.close();
|
||||
}
|
||||
|
||||
databaseManager = new DatabaseManager();
|
||||
await databaseManager.init(config);
|
||||
|
||||
// 初始化注册表数据库
|
||||
registryDatabase = new RegistryDatabase(databaseManager);
|
||||
await registryDatabase.init();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: '数据库配置已保存并连接成功',
|
||||
config: {
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
user: config.user,
|
||||
database: config.database,
|
||||
connectionLimit: config.connectionLimit,
|
||||
ssl: config.ssl
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('保存数据库配置失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 获取数据库配置
|
||||
* GET /api/database/config
|
||||
*/
|
||||
router.get('/config', async (req, res) => {
|
||||
try {
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
|
||||
// 检测是否在pkg打包环境中运行
|
||||
const isPkg = process.pkg !== undefined;
|
||||
|
||||
const configPath = isPkg
|
||||
? path.join(process.cwd(), 'data', 'database.json') // 打包环境：当前工作目录的data文件夹
|
||||
: path.join(__dirname, '..', '..', 'data', 'database.json'); // 开发环境：项目根目录的data文件夹
|
||||
|
||||
if (await fs.pathExists(configPath)) {
|
||||
const config = await fs.readJson(configPath);
|
||||
|
||||
// 不返回密码
|
||||
const safeConfig = {
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
user: config.user,
|
||||
database: config.database,
|
||||
connectionLimit: config.connectionLimit,
|
||||
ssl: config.ssl,
|
||||
hasPassword: !!config.password
|
||||
};
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: safeConfig
|
||||
});
|
||||
} else {
|
||||
res.json({
|
||||
success: true,
|
||||
data: null
|
||||
});
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error('获取数据库配置失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 获取数据库连接状态
|
||||
* GET /api/database/status
|
||||
*/
|
||||
router.get('/status', async (req, res) => {
|
||||
try {
|
||||
if (!databaseManager) {
|
||||
return res.json({
|
||||
success: true,
|
||||
data: {
|
||||
connected: false,
|
||||
message: '数据库未配置'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const isConnected = databaseManager.isConnected;
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
connected: isConnected,
|
||||
message: isConnected ? '数据库已连接' : '数据库连接断开'
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('获取数据库状态失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 初始化数据库表
|
||||
* POST /api/database/init-tables
|
||||
*/
|
||||
router.post('/init-tables', async (req, res) => {
|
||||
try {
|
||||
if (!databaseManager) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '数据库未配置'
|
||||
});
|
||||
}
|
||||
|
||||
if (!registryDatabase) {
|
||||
registryDatabase = new RegistryDatabase(databaseManager);
|
||||
}
|
||||
|
||||
await registryDatabase.init();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: '数据库表初始化成功'
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('初始化数据库表失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 数据迁移：JSON到数据库
|
||||
* POST /api/database/migrate/json-to-db
|
||||
*/
|
||||
router.post('/migrate/json-to-db', async (req, res) => {
|
||||
try {
|
||||
const { overwrite = true, createBackup = true } = req.body;
|
||||
|
||||
if (!databaseManager || !registryDatabase) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '数据库未配置或未初始化'
|
||||
});
|
||||
}
|
||||
|
||||
const downloadService = req.backend.getDownloadService();
|
||||
const jsonRegistry = downloadService.downloadRegistry;
|
||||
const fileManager = downloadService.fileManager;
|
||||
|
||||
const migration = new RegistryMigration(jsonRegistry, registryDatabase, fileManager);
|
||||
const result = await migration.performMigration('json-to-db', overwrite, createBackup);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('JSON到数据库迁移失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 数据迁移：数据库到JSON
|
||||
* POST /api/database/migrate/db-to-json
|
||||
*/
|
||||
router.post('/migrate/db-to-json', async (req, res) => {
|
||||
try {
|
||||
const { overwrite = true, createBackup = true } = req.body;
|
||||
|
||||
if (!databaseManager || !registryDatabase) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '数据库未配置或未初始化'
|
||||
});
|
||||
}
|
||||
|
||||
const downloadService = req.backend.getDownloadService();
|
||||
const jsonRegistry = downloadService.downloadRegistry;
|
||||
const fileManager = downloadService.fileManager;
|
||||
|
||||
const migration = new RegistryMigration(jsonRegistry, registryDatabase, fileManager);
|
||||
const result = await migration.performMigration('db-to-json', overwrite, createBackup);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('数据库到JSON迁移失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 比较注册表差异
|
||||
* GET /api/database/compare-registries
|
||||
*/
|
||||
router.get('/compare-registries', async (req, res) => {
|
||||
try {
|
||||
if (!databaseManager || !registryDatabase) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '数据库未配置或未初始化'
|
||||
});
|
||||
}
|
||||
|
||||
const downloadService = req.backend.getDownloadService();
|
||||
const jsonRegistry = downloadService.downloadRegistry;
|
||||
const fileManager = downloadService.fileManager;
|
||||
|
||||
const migration = new RegistryMigration(jsonRegistry, registryDatabase, fileManager);
|
||||
const comparison = await migration.compareRegistries();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: comparison
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('比较注册表失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 断开数据库连接
|
||||
* POST /api/database/disconnect
|
||||
*/
|
||||
router.post('/disconnect', async (req, res) => {
|
||||
try {
|
||||
if (databaseManager) {
|
||||
await databaseManager.disconnect();
|
||||
databaseManager = null;
|
||||
registryDatabase = null;
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: '数据库连接已断开'
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('断开数据库连接失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// 数据库注册表相关API
|
||||
|
||||
/**
|
||||
* 获取数据库注册表统计信息
|
||||
* GET /api/database/registry/stats
|
||||
*/
|
||||
router.get('/registry/stats', async (req, res) => {
|
||||
try {
|
||||
if (!registryDatabase) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '数据库注册表未初始化'
|
||||
});
|
||||
}
|
||||
|
||||
const stats = await registryDatabase.getStats();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: stats
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('获取数据库注册表统计信息失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 导出数据库注册表
|
||||
* GET /api/database/registry/export
|
||||
*/
|
||||
router.get('/registry/export', async (req, res) => {
|
||||
try {
|
||||
if (!registryDatabase) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '数据库注册表未初始化'
|
||||
});
|
||||
}
|
||||
|
||||
const registryData = await registryDatabase.exportRegistry();
|
||||
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.setHeader('Content-Disposition', 'attachment; filename="database-registry.json"');
|
||||
res.json(registryData);
|
||||
|
||||
} catch (error) {
|
||||
logger.error('导出数据库注册表失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 导入数据到数据库注册表
|
||||
* POST /api/database/registry/import
|
||||
*/
|
||||
router.post('/registry/import', async (req, res) => {
|
||||
try {
|
||||
const { registryData } = req.body;
|
||||
|
||||
if (!registryData) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '缺少注册表数据'
|
||||
});
|
||||
}
|
||||
|
||||
if (!registryDatabase) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '数据库注册表未初始化'
|
||||
});
|
||||
}
|
||||
|
||||
const result = await registryDatabase.importRegistry(registryData);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('导入数据库注册表失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 从文件系统重建数据库注册表
|
||||
* POST /api/database/registry/rebuild
|
||||
*/
|
||||
router.post('/registry/rebuild', async (req, res) => {
|
||||
try {
|
||||
if (!registryDatabase) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '数据库注册表未初始化'
|
||||
});
|
||||
}
|
||||
|
||||
const downloadService = req.backend.getDownloadService();
|
||||
const fileManager = downloadService.fileManager;
|
||||
|
||||
// 生成任务ID
|
||||
const taskId = `db-registry-rebuild-${Date.now()}`;
|
||||
|
||||
// 立即返回任务ID，不等待完成
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
taskId,
|
||||
status: 'started',
|
||||
message: '数据库注册表重建任务已启动'
|
||||
}
|
||||
});
|
||||
|
||||
// 异步执行重建任务
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
// 设置任务状态为进行中
|
||||
global.dbRegistryRebuildTasks = global.dbRegistryRebuildTasks || new Map();
|
||||
global.dbRegistryRebuildTasks.set(taskId, {
|
||||
status: 'running',
|
||||
startTime: Date.now(),
|
||||
progress: {
|
||||
scannedArtists: 0,
|
||||
scannedArtworks: 0,
|
||||
addedArtworks: 0,
|
||||
skippedArtworks: 0,
|
||||
currentArtist: null
|
||||
}
|
||||
});
|
||||
|
||||
const result = await registryDatabase.rebuildFromFileSystem(fileManager, taskId);
|
||||
|
||||
// 更新任务状态为完成
|
||||
global.dbRegistryRebuildTasks.set(taskId, {
|
||||
status: 'completed',
|
||||
startTime: global.dbRegistryRebuildTasks.get(taskId).startTime,
|
||||
endTime: Date.now(),
|
||||
result: result
|
||||
});
|
||||
|
||||
logger.info(`数据库注册表重建任务完成: ${taskId}`, result);
|
||||
} catch (error) {
|
||||
logger.error(`数据库注册表重建任务失败: ${taskId}`, error);
|
||||
|
||||
// 更新任务状态为失败
|
||||
global.dbRegistryRebuildTasks.set(taskId, {
|
||||
status: 'failed',
|
||||
startTime: global.dbRegistryRebuildTasks.get(taskId).startTime,
|
||||
endTime: Date.now(),
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('启动数据库注册表重建任务失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 获取数据库注册表重建任务状态
|
||||
* GET /api/database/registry/rebuild/status/:taskId
|
||||
*/
|
||||
router.get('/registry/rebuild/status/:taskId', async (req, res) => {
|
||||
try {
|
||||
const { taskId } = req.params;
|
||||
|
||||
global.dbRegistryRebuildTasks = global.dbRegistryRebuildTasks || new Map();
|
||||
const task = global.dbRegistryRebuildTasks.get(taskId);
|
||||
|
||||
if (!task) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: '任务不存在'
|
||||
});
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: task
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('获取数据库注册表重建任务状态失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 清理数据库注册表
|
||||
* POST /api/database/registry/cleanup
|
||||
*/
|
||||
router.post('/registry/cleanup', async (req, res) => {
|
||||
try {
|
||||
if (!registryDatabase) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '数据库注册表未初始化'
|
||||
});
|
||||
}
|
||||
|
||||
const downloadService = req.backend.getDownloadService();
|
||||
const fileManager = downloadService.fileManager;
|
||||
|
||||
const result = await registryDatabase.cleanupRegistry(fileManager);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('清理数据库注册表失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// 导出数据库管理器实例，供其他模块使用
|
||||
router.getDatabaseManager = () => databaseManager;
|
||||
router.getRegistryDatabase = () => registryDatabase;
|
||||
router.setDatabaseInstances = setDatabaseInstances;
|
||||
|
||||
module.exports = router;
|
||||
@@ -1203,7 +1203,8 @@ router.get('/registry/config', async (req, res) => {
|
||||
// 提取下载相关的配置
|
||||
const downloadConfig = {
|
||||
useRegistryCheck: config.download?.useRegistryCheck !== false, // 默认启用
|
||||
fallbackToScan: config.download?.fallbackToScan === true // 默认不启用
|
||||
fallbackToScan: config.download?.fallbackToScan === true, // 默认不启用
|
||||
storageMode: config.download?.storageMode || 'json' // 默认JSON存储
|
||||
};
|
||||
|
||||
res.json({
|
||||
@@ -1225,8 +1226,9 @@ router.get('/registry/config', async (req, res) => {
|
||||
*/
|
||||
router.put('/registry/config', async (req, res) => {
|
||||
try {
|
||||
const { useRegistryCheck, fallbackToScan } = req.body;
|
||||
const { useRegistryCheck, fallbackToScan, storageMode } = req.body;
|
||||
|
||||
// 验证必需的布尔值参数
|
||||
if (typeof useRegistryCheck !== 'boolean' || typeof fallbackToScan !== 'boolean') {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
@@ -1234,21 +1236,38 @@ router.put('/registry/config', async (req, res) => {
|
||||
});
|
||||
}
|
||||
|
||||
// 验证存储模式参数（可选）
|
||||
if (storageMode && !['json', 'database'].includes(storageMode)) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '存储模式必须是 json 或 database'
|
||||
});
|
||||
}
|
||||
|
||||
const downloadService = req.backend.getDownloadService();
|
||||
|
||||
// 更新配置
|
||||
const updatedConfig = await downloadService.cacheConfigManager.updateConfig({
|
||||
// 构建更新配置对象
|
||||
const updateData = {
|
||||
download: {
|
||||
useRegistryCheck,
|
||||
fallbackToScan
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// 如果提供了存储模式，添加到更新数据中
|
||||
if (storageMode) {
|
||||
updateData.download.storageMode = storageMode;
|
||||
}
|
||||
|
||||
// 更新配置
|
||||
const updatedConfig = await downloadService.cacheConfigManager.updateConfig(updateData);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
useRegistryCheck: updatedConfig.download?.useRegistryCheck !== false,
|
||||
fallbackToScan: updatedConfig.download?.fallbackToScan === true
|
||||
fallbackToScan: updatedConfig.download?.fallbackToScan === true,
|
||||
storageMode: updatedConfig.download?.storageMode || 'json'
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
@@ -1260,4 +1279,27 @@ router.put('/registry/config', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 清理下载注册表
|
||||
* POST /api/download/registry/cleanup
|
||||
*/
|
||||
router.post('/registry/cleanup', async (req, res) => {
|
||||
try {
|
||||
const downloadService = req.backend.getDownloadService();
|
||||
const result = await downloadService.downloadRegistry.cleanupRegistry(downloadService.fileManager);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('清理下载注册表失败:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -14,6 +14,7 @@ const rankingRoutes = require('./ranking');
|
||||
const watchlistRoutes = require('./watchlist');
|
||||
const updateRoutes = require('./update');
|
||||
const systemRoutes = require('./system');
|
||||
const databaseRoutes = require('./database');
|
||||
|
||||
// 导入认证中间件
|
||||
const { authMiddleware } = require('../middleware/auth');
|
||||
@@ -49,6 +50,7 @@ function setupRoutes(app, backend) {
|
||||
app.use('/api/watchlist', authMiddleware, watchlistRoutes); // 待看名单，需要认证
|
||||
app.use('/api/update', updateRoutes); // 更新检查，不需要认证
|
||||
app.use('/api/system', systemRoutes); // 系统管理，不需要认证
|
||||
app.use('/api/database', databaseRoutes); // 数据库管理，不需要认证
|
||||
|
||||
// 404 处理
|
||||
app.use((req, res) => {
|
||||
|
||||
@@ -2,26 +2,35 @@ const path = require('path');
|
||||
const fs = require('fs-extra');
|
||||
const { defaultLogger } = require('../utils/logger');
|
||||
const ConfigManager = require('../config/config-manager');
|
||||
const CacheConfigManager = require('../config/cache-config');
|
||||
const RegistryDatabase = require('../database/registry-database');
|
||||
const artworkUtils = require('../utils/artwork-utils');
|
||||
|
||||
// 创建logger实例
|
||||
const logger = defaultLogger.child('DownloadRegistry');
|
||||
|
||||
/**
|
||||
* 下载记录管理器 - 维护已下载作品的JSON记录
|
||||
* 下载记录管理器 - 维护已下载作品的记录
|
||||
* 支持JSON文件和数据库两种存储模式，根据配置自动选择
|
||||
* 用于快速检测作品是否已下载，支持导入导出和多设备同步
|
||||
*/
|
||||
class DownloadRegistry {
|
||||
constructor(dataPath) {
|
||||
constructor(dataPath, databaseManager = null) {
|
||||
this.dataPath = dataPath;
|
||||
this.registryPath = path.join(dataPath, 'download-registry.json');
|
||||
this.registry = {
|
||||
version: '1.0.5',
|
||||
artists: {},
|
||||
lastUpdated: null
|
||||
lastUpdated: null,
|
||||
};
|
||||
this.loaded = false;
|
||||
this.configManager = new ConfigManager();
|
||||
this.cacheConfigManager = new CacheConfigManager();
|
||||
|
||||
// 数据库相关
|
||||
this.databaseManager = databaseManager;
|
||||
this.registryDatabase = null;
|
||||
this.storageMode = 'json'; // 默认使用JSON存储
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -31,17 +40,61 @@ class DownloadRegistry {
|
||||
try {
|
||||
// 确保数据目录存在
|
||||
await fs.ensureDir(this.dataPath);
|
||||
|
||||
// 加载现有注册表
|
||||
await this.loadRegistry();
|
||||
|
||||
logger.info(`下载记录注册表初始化完成，总共包含${Object.keys(this.registry.artists).length}个作者，${this.getTotalArtworkCount()}个作品`);
|
||||
|
||||
// 获取存储模式配置
|
||||
await this.loadStorageMode();
|
||||
|
||||
// 根据存储模式初始化相应的存储系统
|
||||
if (this.storageMode === 'database' && this.databaseManager) {
|
||||
await this.initDatabaseStorage();
|
||||
} else {
|
||||
await this.initJsonStorage();
|
||||
}
|
||||
|
||||
const stats = await this.getStats();
|
||||
logger.info(`下载记录注册表初始化完成（${this.storageMode}模式），总共包含${stats.artistCount}个作者，${stats.artworkCount}个作品`);
|
||||
} catch (error) {
|
||||
logger.error('下载记录注册表初始化失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载存储模式配置
|
||||
*/
|
||||
async loadStorageMode() {
|
||||
try {
|
||||
const cacheConfig = await this.cacheConfigManager.loadConfig();
|
||||
this.storageMode = cacheConfig.download?.storageMode || 'json';
|
||||
logger.debug(`存储模式配置: ${this.storageMode}`);
|
||||
} catch (error) {
|
||||
logger.warn('加载存储模式配置失败，使用默认JSON模式:', error.message);
|
||||
this.storageMode = 'json';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化数据库存储
|
||||
*/
|
||||
async initDatabaseStorage() {
|
||||
if (!this.databaseManager) {
|
||||
throw new Error('数据库管理器未提供，无法使用数据库存储模式');
|
||||
}
|
||||
|
||||
this.registryDatabase = new RegistryDatabase(this.databaseManager);
|
||||
await this.registryDatabase.init();
|
||||
this.loaded = true;
|
||||
logger.info('数据库存储模式初始化完成');
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化JSON存储
|
||||
*/
|
||||
async initJsonStorage() {
|
||||
await this.loadRegistry();
|
||||
logger.info('JSON存储模式初始化完成');
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载注册表文件
|
||||
*/
|
||||
@@ -49,14 +102,14 @@ class DownloadRegistry {
|
||||
try {
|
||||
if (await fs.pathExists(this.registryPath)) {
|
||||
const data = await fs.readJson(this.registryPath);
|
||||
|
||||
|
||||
// 验证数据格式
|
||||
if (data && typeof data === 'object' && data.artists) {
|
||||
this.registry = {
|
||||
version: data.version || '1.0.0',
|
||||
created_at: data.created_at || new Date().toISOString(),
|
||||
updated_at: data.updated_at || new Date().toISOString(),
|
||||
artists: data.artists || {}
|
||||
artists: data.artists || {},
|
||||
};
|
||||
} else {
|
||||
logger.warn('注册表文件格式不正确，使用默认格式');
|
||||
@@ -64,7 +117,7 @@ class DownloadRegistry {
|
||||
} else {
|
||||
logger.info('注册表文件不存在，将创建新的注册表');
|
||||
}
|
||||
|
||||
|
||||
this.loaded = true;
|
||||
} catch (error) {
|
||||
logger.error('加载注册表文件失败:', error);
|
||||
@@ -94,25 +147,32 @@ class DownloadRegistry {
|
||||
*/
|
||||
async addArtwork(artistName, artworkId) {
|
||||
if (!this.loaded) {
|
||||
await this.loadRegistry();
|
||||
await this.init();
|
||||
}
|
||||
|
||||
const normalizedArtistName = this.normalizeArtistName(artistName);
|
||||
const normalizedArtworkId = parseInt(artworkId);
|
||||
|
||||
if (!this.registry.artists[normalizedArtistName]) {
|
||||
this.registry.artists[normalizedArtistName] = {
|
||||
artworks: []
|
||||
};
|
||||
}
|
||||
if (this.storageMode === 'database' && this.registryDatabase) {
|
||||
// 使用数据库存储
|
||||
await this.registryDatabase.addArtwork(normalizedArtistName, normalizedArtworkId);
|
||||
logger.debug('添加作品记录到数据库', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
} else {
|
||||
// 使用JSON存储
|
||||
if (!this.registry.artists[normalizedArtistName]) {
|
||||
this.registry.artists[normalizedArtistName] = {
|
||||
artworks: [],
|
||||
};
|
||||
}
|
||||
|
||||
// 检查是否已存在
|
||||
if (!this.registry.artists[normalizedArtistName].artworks.includes(normalizedArtworkId)) {
|
||||
this.registry.artists[normalizedArtistName].artworks.push(normalizedArtworkId);
|
||||
this.registry.artists[normalizedArtistName].artworks.sort((a, b) => b - a); // 按ID倒序排列
|
||||
|
||||
await this.saveRegistry();
|
||||
logger.debug('添加作品记录', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
// 检查是否已存在
|
||||
if (!this.registry.artists[normalizedArtistName].artworks.includes(normalizedArtworkId)) {
|
||||
this.registry.artists[normalizedArtistName].artworks.push(normalizedArtworkId);
|
||||
this.registry.artists[normalizedArtistName].artworks.sort((a, b) => b - a); // 按ID倒序排列
|
||||
|
||||
await this.saveRegistry();
|
||||
logger.debug('添加作品记录到JSON', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,45 +183,52 @@ class DownloadRegistry {
|
||||
*/
|
||||
async removeArtwork(artistName, artworkId) {
|
||||
if (!this.loaded) {
|
||||
await this.loadRegistry();
|
||||
await this.init();
|
||||
}
|
||||
|
||||
const normalizedArtistName = this.normalizeArtistName(artistName);
|
||||
const normalizedArtworkId = parseInt(artworkId);
|
||||
|
||||
logger.debug('开始移除作品记录', {
|
||||
originalArtistName: artistName,
|
||||
normalizedArtistName: normalizedArtistName,
|
||||
artworkId: normalizedArtworkId
|
||||
logger.debug('开始移除作品记录', {
|
||||
originalArtistName: artistName,
|
||||
normalizedArtistName: normalizedArtistName,
|
||||
artworkId: normalizedArtworkId,
|
||||
});
|
||||
|
||||
if (this.registry.artists[normalizedArtistName]) {
|
||||
const artworks = this.registry.artists[normalizedArtistName].artworks;
|
||||
const index = artworks.indexOf(normalizedArtworkId);
|
||||
|
||||
logger.debug('查找作品在注册表中的位置', {
|
||||
artistName: normalizedArtistName,
|
||||
artworkId: normalizedArtworkId,
|
||||
index: index,
|
||||
artworks: artworks
|
||||
});
|
||||
|
||||
if (index !== -1) {
|
||||
artworks.splice(index, 1);
|
||||
|
||||
// 如果作者下没有作品了，删除作者记录
|
||||
if (artworks.length === 0) {
|
||||
delete this.registry.artists[normalizedArtistName];
|
||||
logger.info('作者下无作品，删除作者记录', { artistName: normalizedArtistName });
|
||||
}
|
||||
|
||||
await this.saveRegistry();
|
||||
logger.debug('成功移除作品记录', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
} else {
|
||||
logger.warn('作品在注册表中未找到', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
}
|
||||
if (this.storageMode === 'database' && this.registryDatabase) {
|
||||
// 使用数据库存储
|
||||
await this.registryDatabase.removeArtwork(normalizedArtistName, normalizedArtworkId);
|
||||
logger.debug('从数据库移除作品记录', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
} else {
|
||||
logger.warn('作者在注册表中未找到', { artistName: normalizedArtistName });
|
||||
// 使用JSON存储
|
||||
if (this.registry.artists[normalizedArtistName]) {
|
||||
const artworks = this.registry.artists[normalizedArtistName].artworks;
|
||||
const index = artworks.indexOf(normalizedArtworkId);
|
||||
|
||||
logger.debug('查找作品在注册表中的位置', {
|
||||
artistName: normalizedArtistName,
|
||||
artworkId: normalizedArtworkId,
|
||||
index: index,
|
||||
artworks: artworks,
|
||||
});
|
||||
|
||||
if (index !== -1) {
|
||||
artworks.splice(index, 1);
|
||||
|
||||
// 如果作者下没有作品了，删除作者记录
|
||||
if (artworks.length === 0) {
|
||||
delete this.registry.artists[normalizedArtistName];
|
||||
logger.info('作者下无作品，删除作者记录', { artistName: normalizedArtistName });
|
||||
}
|
||||
|
||||
await this.saveRegistry();
|
||||
logger.debug('成功移除作品记录', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
} else {
|
||||
logger.warn('作品在注册表中未找到', { artistName: normalizedArtistName, artworkId: normalizedArtworkId });
|
||||
}
|
||||
} else {
|
||||
logger.warn('作者在注册表中未找到', { artistName: normalizedArtistName });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,19 +239,25 @@ class DownloadRegistry {
|
||||
*/
|
||||
async isArtworkDownloaded(artworkId) {
|
||||
if (!this.loaded) {
|
||||
await this.loadRegistry();
|
||||
await this.init();
|
||||
}
|
||||
|
||||
const normalizedArtworkId = parseInt(artworkId);
|
||||
|
||||
// 遍历所有作者查找作品
|
||||
for (const artistName in this.registry.artists) {
|
||||
if (this.registry.artists[artistName].artworks.includes(normalizedArtworkId)) {
|
||||
return true;
|
||||
|
||||
if (this.storageMode === 'database' && this.registryDatabase) {
|
||||
// 使用数据库存储
|
||||
return await this.registryDatabase.isArtworkDownloaded(normalizedArtworkId);
|
||||
} else {
|
||||
// 使用JSON存储
|
||||
// 遍历所有作者查找作品
|
||||
for (const artistName in this.registry.artists) {
|
||||
if (this.registry.artists[artistName].artworks.includes(normalizedArtworkId)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -235,13 +308,13 @@ class DownloadRegistry {
|
||||
}
|
||||
|
||||
const artworkIds = new Set();
|
||||
|
||||
|
||||
for (const artistName in this.registry.artists) {
|
||||
for (const artworkId of this.registry.artists[artistName].artworks) {
|
||||
artworkIds.add(artworkId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return Array.from(artworkIds).sort((a, b) => b - a);
|
||||
}
|
||||
|
||||
@@ -252,16 +325,22 @@ class DownloadRegistry {
|
||||
*/
|
||||
async getArtistArtworks(artistName) {
|
||||
if (!this.loaded) {
|
||||
await this.loadRegistry();
|
||||
await this.init();
|
||||
}
|
||||
|
||||
const normalizedArtistName = this.normalizeArtistName(artistName);
|
||||
|
||||
if (this.registry.artists[normalizedArtistName]) {
|
||||
return [...this.registry.artists[normalizedArtistName].artworks];
|
||||
|
||||
if (this.storageMode === 'database' && this.registryDatabase) {
|
||||
// 使用数据库存储
|
||||
return await this.registryDatabase.getArtistArtworks(normalizedArtistName);
|
||||
} else {
|
||||
// 使用JSON存储
|
||||
if (this.registry.artists[normalizedArtistName]) {
|
||||
return [...this.registry.artists[normalizedArtistName].artworks];
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -270,10 +349,16 @@ class DownloadRegistry {
|
||||
*/
|
||||
async getDownloadedArtists() {
|
||||
if (!this.loaded) {
|
||||
await this.loadRegistry();
|
||||
await this.init();
|
||||
}
|
||||
|
||||
return Object.keys(this.registry.artists);
|
||||
if (this.storageMode === 'database' && this.registryDatabase) {
|
||||
// 使用数据库存储
|
||||
return await this.registryDatabase.getDownloadedArtists();
|
||||
} else {
|
||||
// 使用JSON存储
|
||||
return Object.keys(this.registry.artists);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -281,19 +366,25 @@ class DownloadRegistry {
|
||||
*/
|
||||
async getStats() {
|
||||
if (!this.loaded) {
|
||||
await this.loadRegistry();
|
||||
await this.init();
|
||||
}
|
||||
|
||||
const artists = Object.keys(this.registry.artists);
|
||||
const totalArtworks = this.getTotalArtworkCount();
|
||||
|
||||
return {
|
||||
artistCount: artists.length,
|
||||
artworkCount: totalArtworks,
|
||||
version: this.registry.version,
|
||||
created_at: this.registry.created_at,
|
||||
updated_at: this.registry.updated_at
|
||||
};
|
||||
if (this.storageMode === 'database' && this.registryDatabase) {
|
||||
// 使用数据库存储
|
||||
return await this.registryDatabase.getStats();
|
||||
} else {
|
||||
// 使用JSON存储
|
||||
const artists = Object.keys(this.registry.artists);
|
||||
const totalArtworks = this.getTotalArtworkCount();
|
||||
|
||||
return {
|
||||
artistCount: artists.length,
|
||||
artworkCount: totalArtworks,
|
||||
version: this.registry.version,
|
||||
created_at: this.registry.created_at,
|
||||
updated_at: this.registry.updated_at,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -307,7 +398,7 @@ class DownloadRegistry {
|
||||
|
||||
return {
|
||||
...this.registry,
|
||||
exported_at: new Date().toISOString()
|
||||
exported_at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -332,14 +423,14 @@ class DownloadRegistry {
|
||||
for (const artistName in importData.artists) {
|
||||
const normalizedArtistName = this.normalizeArtistName(artistName);
|
||||
const importArtworks = importData.artists[artistName].artworks || [];
|
||||
|
||||
|
||||
if (!this.registry.artists[normalizedArtistName]) {
|
||||
this.registry.artists[normalizedArtistName] = { artworks: [] };
|
||||
addedArtists++;
|
||||
}
|
||||
|
||||
const existingArtworks = new Set(this.registry.artists[normalizedArtistName].artworks);
|
||||
|
||||
|
||||
for (const artworkId of importArtworks) {
|
||||
const normalizedArtworkId = parseInt(artworkId);
|
||||
if (!existingArtworks.has(normalizedArtworkId)) {
|
||||
@@ -349,7 +440,7 @@ class DownloadRegistry {
|
||||
skippedArtworks++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 排序
|
||||
this.registry.artists[normalizedArtistName].artworks.sort((a, b) => b - a);
|
||||
}
|
||||
@@ -361,27 +452,29 @@ class DownloadRegistry {
|
||||
addedArtworks,
|
||||
skippedArtworks,
|
||||
totalArtists: Object.keys(this.registry.artists).length,
|
||||
totalArtworks: this.getTotalArtworkCount()
|
||||
totalArtworks: this.getTotalArtworkCount(),
|
||||
};
|
||||
|
||||
logger.info(`注册表导入完成，导入了 ${result.addedArtists} 个作者，${result.addedArtworks} 个作品，跳过了 ${result.skippedArtworks} 个重复作品。当前总计：${result.totalArtists} 个作者，${result.totalArtworks} 个作品`);
|
||||
logger.info(
|
||||
`注册表导入完成，导入了 ${result.addedArtists} 个作者，${result.addedArtworks} 个作品，跳过了 ${result.skippedArtworks} 个重复作品。当前总计：${result.totalArtists} 个作者，${result.totalArtworks} 个作品`
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从文件系统重建注册表
|
||||
* @param {FileManager} fileManager
|
||||
* @param {FileManager} fileManager
|
||||
* @param {string} taskId - 任务ID，用于更新进度
|
||||
* @returns {Promise<{scannedArtists: number, scannedArtworks: number, addedArtworks: number, skippedArtworks: number}>}
|
||||
*/
|
||||
async rebuildFromFileSystem(fileManager, taskId = null) {
|
||||
logger.info('开始从文件系统重建下载注册表...');
|
||||
|
||||
|
||||
const stats = {
|
||||
scannedArtists: 0,
|
||||
scannedArtworks: 0,
|
||||
addedArtworks: 0,
|
||||
skippedArtworks: 0
|
||||
skippedArtworks: 0,
|
||||
};
|
||||
|
||||
// 获取所有艺术家目录
|
||||
@@ -397,8 +490,8 @@ class DownloadRegistry {
|
||||
...task,
|
||||
progress: {
|
||||
...stats,
|
||||
currentArtist
|
||||
}
|
||||
currentArtist,
|
||||
},
|
||||
});
|
||||
}
|
||||
// 检查是否被取消
|
||||
@@ -412,20 +505,20 @@ class DownloadRegistry {
|
||||
try {
|
||||
stats.scannedArtists++;
|
||||
updateProgress(artistDir);
|
||||
|
||||
|
||||
logger.info(`扫描艺术家目录: ${artistDir}`);
|
||||
|
||||
|
||||
// 获取艺术家目录下的所有作品目录
|
||||
const artworkDirs = await fileManager.getArtworkDirectories(artistDir);
|
||||
|
||||
|
||||
for (const artworkDir of artworkDirs) {
|
||||
try {
|
||||
stats.scannedArtworks++;
|
||||
updateProgress(artistDir);
|
||||
|
||||
|
||||
// 检查作品是否已在注册表中
|
||||
const isRegistered = await this.isArtworkRegistered(artistDir, artworkDir);
|
||||
|
||||
|
||||
if (!isRegistered) {
|
||||
// 从作品目录名中提取作品ID并添加到注册表
|
||||
const artworkId = await this.extractArtworkIdFromDir(artworkDir);
|
||||
@@ -437,17 +530,15 @@ class DownloadRegistry {
|
||||
} else {
|
||||
stats.skippedArtworks++;
|
||||
}
|
||||
|
||||
|
||||
// 每处理10个作品更新一次进度
|
||||
if (stats.scannedArtworks % 10 === 0) {
|
||||
updateProgress(artistDir);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.warn(`处理作品目录失败 ${artistDir}/${artworkDir}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.warn(`处理艺术家目录失败 ${artistDir}:`, error.message);
|
||||
}
|
||||
@@ -455,7 +546,7 @@ class DownloadRegistry {
|
||||
|
||||
// 最终更新进度
|
||||
updateProgress(null);
|
||||
|
||||
|
||||
logger.info('从文件系统重建下载注册表完成', stats);
|
||||
return stats;
|
||||
}
|
||||
@@ -472,7 +563,7 @@ class DownloadRegistry {
|
||||
}
|
||||
|
||||
logger.info('开始清理注册表...');
|
||||
|
||||
|
||||
let removedArtists = 0;
|
||||
let removedArtworks = 0;
|
||||
const downloadPath = await fileManager.getDownloadPath();
|
||||
@@ -484,18 +575,18 @@ class DownloadRegistry {
|
||||
for (const artworkId of artworks) {
|
||||
// 检查作品目录是否存在
|
||||
let found = false;
|
||||
|
||||
|
||||
try {
|
||||
const artistPath = path.join(downloadPath, artistName);
|
||||
if (await fileManager.directoryExists(artistPath)) {
|
||||
const artworkEntries = await fileManager.listDirectory(artistPath);
|
||||
|
||||
|
||||
for (const entry of artworkEntries) {
|
||||
const extractedArtworkId = await artworkUtils.extractArtworkIdFromDir(entry);
|
||||
if (extractedArtworkId && extractedArtworkId === artworkId) {
|
||||
const artworkPath = path.join(artistPath, entry);
|
||||
const infoPath = path.join(artworkPath, 'artwork_info.json');
|
||||
|
||||
|
||||
// 检查信息文件是否存在
|
||||
if (await fs.pathExists(infoPath)) {
|
||||
found = true;
|
||||
@@ -531,7 +622,7 @@ class DownloadRegistry {
|
||||
removedArtists,
|
||||
removedArtworks,
|
||||
remainingArtists: Object.keys(this.registry.artists).length,
|
||||
remainingArtworks: this.getTotalArtworkCount()
|
||||
remainingArtworks: this.getTotalArtworkCount(),
|
||||
};
|
||||
|
||||
logger.info('注册表清理完成', result);
|
||||
@@ -571,4 +662,4 @@ class DownloadRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = DownloadRegistry;
|
||||
module.exports = DownloadRegistry;
|
||||
|
||||
@@ -20,11 +20,12 @@ const logger = defaultLogger.child('DownloadService');
|
||||
* 下载服务 - 主服务类，协调各个管理器
|
||||
*/
|
||||
class DownloadService {
|
||||
constructor(auth) {
|
||||
constructor(auth, databaseManager = null) {
|
||||
this.auth = auth;
|
||||
this.artworkService = new ArtworkService(auth);
|
||||
this.artistService = new ArtistService(auth);
|
||||
this.cacheConfigManager = new CacheConfigManager();
|
||||
this.databaseManager = databaseManager;
|
||||
|
||||
// 检测是否在pkg打包环境中运行
|
||||
const isPkg = process.pkg !== undefined;
|
||||
@@ -42,7 +43,7 @@ class DownloadService {
|
||||
this.taskManager = new TaskManager(this.dataPath);
|
||||
this.progressManager = new ProgressManager();
|
||||
this.historyManager = new HistoryManager(this.dataPath);
|
||||
this.downloadRegistry = new DownloadRegistry(this.dataPath);
|
||||
this.downloadRegistry = new DownloadRegistry(this.dataPath, this.databaseManager);
|
||||
// 先创建下载执行器，稍后在init方法中设置downloadService引用
|
||||
this.downloadExecutor = new DownloadExecutor(this.fileManager, this.taskManager, this.progressManager, this.historyManager, this);
|
||||
|
||||
|
||||
@@ -0,0 +1,461 @@
|
||||
const { defaultLogger } = require('../utils/logger');
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
|
||||
const logger = defaultLogger.child('RegistryMigration');
|
||||
|
||||
/**
|
||||
* 注册表数据迁移服务
|
||||
* 处理JSON和数据库之间的数据转换
|
||||
*/
|
||||
class RegistryMigration {
|
||||
constructor(jsonRegistry, databaseRegistry, fileManager) {
|
||||
this.jsonRegistry = jsonRegistry;
|
||||
this.databaseRegistry = databaseRegistry;
|
||||
this.fileManager = fileManager;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从JSON迁移到数据库
|
||||
* @param {boolean} overwrite 是否覆盖现有数据
|
||||
* @returns {Object} 迁移结果
|
||||
*/
|
||||
async migrateJsonToDatabase(overwrite = true) {
|
||||
logger.info('开始从JSON迁移数据到数据库...');
|
||||
|
||||
try {
|
||||
// 确保JSON注册表已加载
|
||||
if (!this.jsonRegistry.loaded) {
|
||||
await this.jsonRegistry.init();
|
||||
}
|
||||
|
||||
// 初始化数据库注册表
|
||||
await this.databaseRegistry.init();
|
||||
|
||||
// 如果需要覆盖，先清空数据库
|
||||
if (overwrite) {
|
||||
logger.info('清空数据库中的现有数据...');
|
||||
await this.clearDatabaseRegistry();
|
||||
}
|
||||
|
||||
// 获取JSON数据
|
||||
const jsonData = await this.jsonRegistry.exportRegistry();
|
||||
if (!jsonData || !jsonData.artists) {
|
||||
throw new Error('JSON注册表数据为空或格式不正确');
|
||||
}
|
||||
|
||||
// 导入到数据库
|
||||
const importResult = await this.databaseRegistry.importRegistry(jsonData);
|
||||
|
||||
const result = {
|
||||
success: true,
|
||||
direction: 'json-to-db',
|
||||
recordsProcessed: importResult.addedArtworks,
|
||||
addedArtists: importResult.addedArtists,
|
||||
addedArtworks: importResult.addedArtworks,
|
||||
skippedArtworks: importResult.skippedArtworks,
|
||||
totalArtists: importResult.totalArtists,
|
||||
totalArtworks: importResult.totalArtworks,
|
||||
message: `成功从JSON迁移到数据库，处理了${importResult.addedArtists}个作者，${importResult.addedArtworks}个作品`
|
||||
};
|
||||
|
||||
logger.info('JSON到数据库迁移完成', result);
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
logger.error('JSON到数据库迁移失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从数据库迁移到JSON
|
||||
* @param {boolean} overwrite 是否覆盖现有数据
|
||||
* @returns {Object} 迁移结果
|
||||
*/
|
||||
async migrateDatabaseToJson(overwrite = true) {
|
||||
logger.info('开始从数据库迁移数据到JSON...');
|
||||
|
||||
try {
|
||||
// 确保数据库注册表已初始化
|
||||
await this.databaseRegistry.init();
|
||||
|
||||
// 确保JSON注册表已加载
|
||||
if (!this.jsonRegistry.loaded) {
|
||||
await this.jsonRegistry.init();
|
||||
}
|
||||
|
||||
// 获取数据库数据
|
||||
const dbData = await this.databaseRegistry.exportRegistry();
|
||||
if (!dbData || !dbData.artists) {
|
||||
throw new Error('数据库注册表数据为空或格式不正确');
|
||||
}
|
||||
|
||||
// 如果需要覆盖，先清空JSON注册表
|
||||
if (overwrite) {
|
||||
logger.info('清空JSON注册表中的现有数据...');
|
||||
this.jsonRegistry.artists = {};
|
||||
this.jsonRegistry.version = dbData.version || '1.0.5';
|
||||
this.jsonRegistry.created_at = dbData.created_at || new Date().toISOString();
|
||||
}
|
||||
|
||||
// 统计信息
|
||||
let addedArtists = 0;
|
||||
let addedArtworks = 0;
|
||||
let skippedArtworks = 0;
|
||||
|
||||
// 导入数据到JSON注册表
|
||||
for (const artistName in dbData.artists) {
|
||||
const artistData = dbData.artists[artistName];
|
||||
const artworks = artistData.artworks || [];
|
||||
|
||||
// 检查艺术家是否已存在
|
||||
const isNewArtist = !this.jsonRegistry.artists[artistName];
|
||||
if (isNewArtist) {
|
||||
this.jsonRegistry.artists[artistName] = { artworks: [] };
|
||||
addedArtists++;
|
||||
}
|
||||
|
||||
// 获取现有作品ID
|
||||
const existingArtworkIds = new Set(this.jsonRegistry.artists[artistName].artworks);
|
||||
|
||||
// 添加新作品
|
||||
for (const artworkId of artworks) {
|
||||
if (!existingArtworkIds.has(artworkId)) {
|
||||
this.jsonRegistry.artists[artistName].artworks.push(artworkId);
|
||||
addedArtworks++;
|
||||
} else {
|
||||
skippedArtworks++;
|
||||
}
|
||||
}
|
||||
|
||||
// 排序作品ID
|
||||
this.jsonRegistry.artists[artistName].artworks.sort((a, b) => b - a);
|
||||
}
|
||||
|
||||
// 更新时间戳
|
||||
this.jsonRegistry.updated_at = new Date().toISOString();
|
||||
|
||||
// 保存JSON注册表
|
||||
await this.jsonRegistry.save();
|
||||
|
||||
// 获取最终统计
|
||||
const finalStats = await this.jsonRegistry.getStats();
|
||||
|
||||
const result = {
|
||||
success: true,
|
||||
direction: 'db-to-json',
|
||||
recordsProcessed: addedArtworks,
|
||||
addedArtists,
|
||||
addedArtworks,
|
||||
skippedArtworks,
|
||||
totalArtists: finalStats.totalArtists,
|
||||
totalArtworks: finalStats.totalArtworks,
|
||||
message: `成功从数据库迁移到JSON，处理了${addedArtists}个作者，${addedArtworks}个作品`
|
||||
};
|
||||
|
||||
logger.info('数据库到JSON迁移完成', result);
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
logger.error('数据库到JSON迁移失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空数据库注册表
|
||||
*/
|
||||
async clearDatabaseRegistry() {
|
||||
try {
|
||||
const db = this.databaseRegistry.db;
|
||||
|
||||
// 删除所有作品记录
|
||||
await db.query('DELETE FROM registry_artworks');
|
||||
|
||||
// 删除所有艺术家记录
|
||||
await db.query('DELETE FROM registry_artists');
|
||||
|
||||
// 重置自增ID
|
||||
await db.query('ALTER TABLE registry_artists AUTO_INCREMENT = 1');
|
||||
await db.query('ALTER TABLE registry_artworks AUTO_INCREMENT = 1');
|
||||
|
||||
logger.info('数据库注册表已清空');
|
||||
} catch (error) {
|
||||
logger.error('清空数据库注册表失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 比较JSON和数据库注册表的差异
|
||||
* @returns {Object} 差异报告
|
||||
*/
|
||||
async compareRegistries() {
|
||||
logger.info('开始比较JSON和数据库注册表...');
|
||||
|
||||
try {
|
||||
// 确保两个注册表都已初始化
|
||||
if (!this.jsonRegistry.loaded) {
|
||||
await this.jsonRegistry.init();
|
||||
}
|
||||
await this.databaseRegistry.init();
|
||||
|
||||
// 获取统计信息
|
||||
const jsonStats = await this.jsonRegistry.getStats();
|
||||
const dbStats = await this.databaseRegistry.getStats();
|
||||
|
||||
// 获取艺术家列表
|
||||
const jsonArtists = await this.jsonRegistry.getDownloadedArtists();
|
||||
const dbArtists = await this.databaseRegistry.getDownloadedArtists();
|
||||
|
||||
// 计算差异
|
||||
const jsonArtistSet = new Set(jsonArtists);
|
||||
const dbArtistSet = new Set(dbArtists);
|
||||
|
||||
const onlyInJson = jsonArtists.filter(artist => !dbArtistSet.has(artist));
|
||||
const onlyInDb = dbArtists.filter(artist => !jsonArtistSet.has(artist));
|
||||
const commonArtists = jsonArtists.filter(artist => dbArtistSet.has(artist));
|
||||
|
||||
// 比较共同艺术家的作品差异
|
||||
let artworkDifferences = [];
|
||||
for (const artist of commonArtists.slice(0, 10)) { // 限制比较数量以避免性能问题
|
||||
const jsonArtworks = await this.jsonRegistry.getArtistArtworks(artist);
|
||||
const dbArtworks = await this.databaseRegistry.getArtistArtworks(artist);
|
||||
|
||||
const jsonArtworkSet = new Set(jsonArtworks);
|
||||
const dbArtworkSet = new Set(dbArtworks);
|
||||
|
||||
const onlyInJsonArtworks = jsonArtworks.filter(id => !dbArtworkSet.has(id));
|
||||
const onlyInDbArtworks = dbArtworks.filter(id => !jsonArtworkSet.has(id));
|
||||
|
||||
if (onlyInJsonArtworks.length > 0 || onlyInDbArtworks.length > 0) {
|
||||
artworkDifferences.push({
|
||||
artist,
|
||||
onlyInJson: onlyInJsonArtworks.length,
|
||||
onlyInDb: onlyInDbArtworks.length,
|
||||
jsonTotal: jsonArtworks.length,
|
||||
dbTotal: dbArtworks.length
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const comparison = {
|
||||
json: {
|
||||
artists: jsonStats.totalArtists,
|
||||
artworks: jsonStats.totalArtworks,
|
||||
version: jsonStats.version,
|
||||
updated_at: jsonStats.updated_at
|
||||
},
|
||||
database: {
|
||||
artists: dbStats.artistCount,
|
||||
artworks: dbStats.artworkCount,
|
||||
version: dbStats.version,
|
||||
updated_at: dbStats.updated_at
|
||||
},
|
||||
differences: {
|
||||
artistsOnlyInJson: onlyInJson.length,
|
||||
artistsOnlyInDb: onlyInDb.length,
|
||||
commonArtists: commonArtists.length,
|
||||
artworkDifferences: artworkDifferences.length,
|
||||
sampleArtworkDifferences: artworkDifferences.slice(0, 5)
|
||||
},
|
||||
recommendation: this.getRecommendation(jsonStats, dbStats, onlyInJson, onlyInDb)
|
||||
};
|
||||
|
||||
logger.info('注册表比较完成', {
|
||||
jsonArtists: jsonStats.totalArtists,
|
||||
dbArtists: dbStats.artistCount,
|
||||
differences: comparison.differences
|
||||
});
|
||||
|
||||
return comparison;
|
||||
|
||||
} catch (error) {
|
||||
logger.error('比较注册表失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据比较结果给出建议
|
||||
*/
|
||||
getRecommendation(jsonStats, dbStats, onlyInJson, onlyInDb) {
|
||||
if (jsonStats.totalArtworks === 0 && dbStats.artworkCount === 0) {
|
||||
return '两个注册表都为空，无需迁移';
|
||||
}
|
||||
|
||||
if (jsonStats.totalArtworks === 0) {
|
||||
return '建议从数据库迁移到JSON';
|
||||
}
|
||||
|
||||
if (dbStats.artworkCount === 0) {
|
||||
return '建议从JSON迁移到数据库';
|
||||
}
|
||||
|
||||
if (jsonStats.totalArtworks > dbStats.artworkCount) {
|
||||
return 'JSON注册表包含更多数据，建议从JSON迁移到数据库';
|
||||
}
|
||||
|
||||
if (dbStats.artworkCount > jsonStats.totalArtworks) {
|
||||
return '数据库注册表包含更多数据，建议从数据库迁移到JSON';
|
||||
}
|
||||
|
||||
if (onlyInJson.length > onlyInDb.length) {
|
||||
return 'JSON注册表有更多独有艺术家，建议从JSON迁移到数据库';
|
||||
}
|
||||
|
||||
if (onlyInDb.length > onlyInJson.length) {
|
||||
return '数据库注册表有更多独有艺术家，建议从数据库迁移到JSON';
|
||||
}
|
||||
|
||||
return '两个注册表数据相似，可根据需要选择迁移方向';
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证迁移结果
|
||||
* @param {string} direction 迁移方向
|
||||
* @returns {Object} 验证结果
|
||||
*/
|
||||
async validateMigration(direction) {
|
||||
logger.info(`验证${direction}迁移结果...`);
|
||||
|
||||
try {
|
||||
const comparison = await this.compareRegistries();
|
||||
|
||||
let isValid = false;
|
||||
let message = '';
|
||||
|
||||
if (direction === 'json-to-db') {
|
||||
// 验证数据库是否包含JSON的所有数据
|
||||
isValid = comparison.database.artworks >= comparison.json.artworks &&
|
||||
comparison.differences.artistsOnlyInJson === 0;
|
||||
message = isValid ?
|
||||
'迁移验证成功：数据库包含了JSON的所有数据' :
|
||||
'迁移验证失败：数据库缺少部分JSON数据';
|
||||
} else if (direction === 'db-to-json') {
|
||||
// 验证JSON是否包含数据库的所有数据
|
||||
isValid = comparison.json.artworks >= comparison.database.artworks &&
|
||||
comparison.differences.artistsOnlyInDb === 0;
|
||||
message = isValid ?
|
||||
'迁移验证成功：JSON包含了数据库的所有数据' :
|
||||
'迁移验证失败：JSON缺少部分数据库数据';
|
||||
}
|
||||
|
||||
const result = {
|
||||
valid: isValid,
|
||||
message,
|
||||
comparison
|
||||
};
|
||||
|
||||
logger.info('迁移验证完成', { valid: isValid, message });
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
logger.error('验证迁移结果失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建数据备份
|
||||
* @param {string} type 备份类型 ('json' | 'database')
|
||||
* @returns {string} 备份文件路径
|
||||
*/
|
||||
async createBackup(type) {
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
|
||||
// 检测是否在pkg打包环境中运行
|
||||
const isPkg = process.pkg !== undefined;
|
||||
|
||||
const backupDir = isPkg
|
||||
? path.join(process.cwd(), 'data', 'backups', 'registry') // 打包环境：当前工作目录的data文件夹
|
||||
: path.join(__dirname, '..', '..', 'backups', 'registry'); // 开发环境：项目根目录的backups文件夹
|
||||
|
||||
await fs.ensureDir(backupDir);
|
||||
|
||||
try {
|
||||
if (type === 'json') {
|
||||
// 备份JSON注册表
|
||||
const jsonData = await this.jsonRegistry.exportRegistry();
|
||||
const backupPath = path.join(backupDir, `registry-json-backup-${timestamp}.json`);
|
||||
|
||||
await fs.writeJson(backupPath, jsonData, { spaces: 2 });
|
||||
logger.info(`JSON注册表备份已创建: ${backupPath}`);
|
||||
return backupPath;
|
||||
|
||||
} else if (type === 'database') {
|
||||
// 备份数据库注册表
|
||||
const dbData = await this.databaseRegistry.exportRegistry();
|
||||
const backupPath = path.join(backupDir, `registry-db-backup-${timestamp}.json`);
|
||||
|
||||
await fs.writeJson(backupPath, dbData, { spaces: 2 });
|
||||
logger.info(`数据库注册表备份已创建: ${backupPath}`);
|
||||
return backupPath;
|
||||
}
|
||||
|
||||
throw new Error(`不支持的备份类型: ${type}`);
|
||||
|
||||
} catch (error) {
|
||||
logger.error(`创建${type}备份失败:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行完整的迁移流程（包含备份和验证）
|
||||
* @param {string} direction 迁移方向
|
||||
* @param {boolean} overwrite 是否覆盖
|
||||
* @param {boolean} createBackup 是否创建备份
|
||||
* @returns {Object} 迁移结果
|
||||
*/
|
||||
async performMigration(direction, overwrite = true, createBackup = true) {
|
||||
logger.info(`开始执行完整迁移流程: ${direction}`);
|
||||
|
||||
const result = {
|
||||
success: false,
|
||||
direction,
|
||||
backupPath: null,
|
||||
migrationResult: null,
|
||||
validationResult: null,
|
||||
error: null
|
||||
};
|
||||
|
||||
try {
|
||||
// 创建备份
|
||||
if (createBackup) {
|
||||
const backupType = direction === 'json-to-db' ? 'database' : 'json';
|
||||
result.backupPath = await this.createBackup(backupType);
|
||||
}
|
||||
|
||||
// 执行迁移
|
||||
if (direction === 'json-to-db') {
|
||||
result.migrationResult = await this.migrateJsonToDatabase(overwrite);
|
||||
} else if (direction === 'db-to-json') {
|
||||
result.migrationResult = await this.migrateDatabaseToJson(overwrite);
|
||||
} else {
|
||||
throw new Error(`不支持的迁移方向: ${direction}`);
|
||||
}
|
||||
|
||||
// 验证迁移结果
|
||||
result.validationResult = await this.validateMigration(direction);
|
||||
|
||||
result.success = result.migrationResult.success && result.validationResult.valid;
|
||||
|
||||
logger.info('完整迁移流程完成', {
|
||||
direction,
|
||||
success: result.success,
|
||||
recordsProcessed: result.migrationResult.recordsProcessed
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
result.error = error.message;
|
||||
logger.error('完整迁移流程失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RegistryMigration;
|
||||
@@ -61,6 +61,9 @@ const ModuleColors = {
|
||||
'ArtistService': '\x1b[95m', // 亮紫色
|
||||
'DownloadService': '\x1b[96m', // 亮青色
|
||||
'AbortControllerManager': '\x1b[94m', // 亮蓝色
|
||||
'DatabaseManager': '\x1b[95m', // 亮紫色
|
||||
'RegistrySchema': '\x1b[94m', // 亮蓝色
|
||||
'RegistryDatabase': '\x1b[94m', // 亮蓝色
|
||||
'Default': '\x1b[39m' // 默认颜色
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user