lkddi/certd
1
0
Fork 0
mirror of https://github.com/certd/certd.git synced 2026-04-24 04:17:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf: 优化数据备份效率，流式写入文件

Browse Source
This commit is contained in:
xiaojunnuo
2025-10-27 15:25:41 +08:00
parent 98cec15625
commit c38dbbb1d7
1 changed files with 124 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ui/certd-server/src/plugins/plugin-admin/plugin-db-backup.ts
+124 -106
View File
@@ -6,15 +6,15 @@ import { AbstractPlusTaskPlugin } from "@certd/plugin-plus";
import JSZip from "jszip"; import JSZip from "jszip";
import * as os from "node:os"; import * as os from "node:os";
import { OssClientContext, ossClientFactory, OssClientRemoveByOpts, SshAccess, SshClient } from "@certd/plugin-lib"; import { OssClientContext, ossClientFactory, OssClientRemoveByOpts, SshAccess, SshClient } from "@certd/plugin-lib";
import { pipeline } from "stream/promises";
const defaultBackupDir = 'certd_backup'; const defaultBackupDir = "certd_backup";
const defaultFilePrefix = 'db_backup'; const defaultFilePrefix = "db_backup";
@IsTaskPlugin({ @IsTaskPlugin({
name: 'DBBackupPlugin', name: "DBBackupPlugin",
title: '数据库备份', title: "数据库备份",
icon: 'lucide:database-backup', icon: "lucide:database-backup",
desc: '【仅管理员可用】仅支持备份SQLite数据库', desc: "【仅管理员可用】仅支持备份SQLite数据库",
group: pluginGroups.admin.key, group: pluginGroups.admin.key,
showRunStrategy: true, showRunStrategy: true,
default: { default: {
@@ -22,32 +22,32 @@ const defaultFilePrefix = 'db_backup';
runStrategy: RunStrategy.AlwaysRun, runStrategy: RunStrategy.AlwaysRun,
}, },
}, },
onlyAdmin:true, onlyAdmin: true,
needPlus: true, needPlus: true,
}) })
export class DBBackupPlugin extends AbstractPlusTaskPlugin { export class DBBackupPlugin extends AbstractPlusTaskPlugin {
@TaskInput({ @TaskInput({
title: '备份方式', title: "备份方式",
value: 'local', value: "local",
component: { component: {
name: 'a-select', name: "a-select",
options: [ options: [
{label: '本地复制', value: 'local'}, { label: "本地复制", value: "local" },
{label: 'ssh上传', value: 'ssh'}, { label: "ssh上传", value: "ssh" },
{label: 'oss上传', value: 'oss'}, { label: "oss上传", value: "oss" },
], ],
placeholder: '', placeholder: "",
}, },
helper: '支持本地复制、ssh上传', helper: "支持本地复制、ssh上传",
required: true, required: true,
}) })
backupMode = 'local'; backupMode = "local";
@TaskInput({ @TaskInput({
title: '主机登录授权', title: "主机登录授权",
component: { component: {
name: 'access-selector', name: "access-selector",
type: 'ssh', type: "ssh",
}, },
mergeScript: ` mergeScript: `
return { return {
@@ -60,19 +60,18 @@ export class DBBackupPlugin extends AbstractPlusTaskPlugin {
}) })
sshAccessId!: number; sshAccessId!: number;
@TaskInput({ @TaskInput({
title: 'OSS类型', title: "OSS类型",
component: { component: {
name: 'a-select', name: "a-select",
options: [ options: [
{value: "alioss", label: "阿里云OSS"}, { value: "alioss", label: "阿里云OSS" },
{value: "s3", label: "MinIO/S3"}, { value: "s3", label: "MinIO/S3" },
{value: "qiniuoss", label: "七牛云"}, { value: "qiniuoss", label: "七牛云" },
{value: "tencentcos", label: "腾讯云COS"}, { value: "tencentcos", label: "腾讯云COS" },
{value: "ftp", label: "Ftp"}, { value: "ftp", label: "Ftp" },
{value: "sftp", label: "Sftp"}, { value: "sftp", label: "Sftp" },
] ],
}, },
mergeScript: ` mergeScript: `
return { return {
@@ -86,9 +85,9 @@ export class DBBackupPlugin extends AbstractPlusTaskPlugin {
ossType!: string; ossType!: string;
@TaskInput({ @TaskInput({
title: 'OSS授权', title: "OSS授权",
component: { component: {
name: 'access-selector', name: "access-selector",
}, },
mergeScript: ` mergeScript: `
return { return {
@@ -106,12 +105,11 @@ export class DBBackupPlugin extends AbstractPlusTaskPlugin {
}) })
ossAccessId!: number; ossAccessId!: number;
@TaskInput({ @TaskInput({
title: '备份保存目录', title: "备份保存目录",
component: { component: {
name: 'a-input', name: "a-input",
type: 'value', type: "value",
placeholder: `默认${defaultBackupDir}`, placeholder: `默认${defaultBackupDir}`,
}, },
helper: `ssh方式默认保存在当前用户的${defaultBackupDir}目录下，本地方式默认保存在data/${defaultBackupDir}目录下，也可以填写绝对路径`, helper: `ssh方式默认保存在当前用户的${defaultBackupDir}目录下，本地方式默认保存在data/${defaultBackupDir}目录下，也可以填写绝对路径`,
@@ -120,10 +118,10 @@ export class DBBackupPlugin extends AbstractPlusTaskPlugin {
backupDir: string = defaultBackupDir; backupDir: string = defaultBackupDir;
@TaskInput({ @TaskInput({
title: '备份文件前缀', title: "备份文件前缀",
component: { component: {
name: 'a-input', name: "a-input",
vModel: 'value', vModel: "value",
placeholder: `默认${defaultFilePrefix}`, placeholder: `默认${defaultFilePrefix}`,
}, },
required: false, required: false,
@@ -131,11 +129,11 @@ export class DBBackupPlugin extends AbstractPlusTaskPlugin {
filePrefix: string = defaultFilePrefix; filePrefix: string = defaultFilePrefix;
@TaskInput({ @TaskInput({
title: '附加上传文件', title: "附加上传文件",
value: true, value: true,
component: { component: {
name: 'a-switch', name: "a-switch",
vModel: 'checked', vModel: "checked",
placeholder: `是否备份上传的头像等文件`, placeholder: `是否备份上传的头像等文件`,
}, },
required: false, required: false,
@@ -143,99 +141,119 @@ export class DBBackupPlugin extends AbstractPlusTaskPlugin {
withUpload = true; withUpload = true;
@TaskInput({ @TaskInput({
title: '删除过期备份', title: "删除过期备份",
component: { component: {
name: 'a-input-number', name: "a-input-number",
vModel: 'value', vModel: "value",
placeholder: '20', placeholder: "20",
}, },
helper: '删除多少天前的备份,不填则不删除，windows暂不支持', helper: "删除多少天前的备份,不填则不删除，windows暂不支持",
required: false, required: false,
}) })
retainDays!: number; retainDays!: number;
async onInstance() { async onInstance() {}
}
async execute(): Promise<void> { async execute(): Promise<void> {
if (!this.isAdmin()) { if (!this.isAdmin()) {
throw new Error('只有管理员才能运行此任务'); throw new Error("只有管理员才能运行此任务");
} }
this.logger.info('开始备份数据库'); this.logger.info("开始备份数据库");
let dbPath = process.env.certd_typeorm_dataSource_default_database; let dbPath = process.env.certd_typeorm_dataSource_default_database;
dbPath = dbPath || './data/db.sqlite'; dbPath = dbPath || "./data/db.sqlite";
if (!fs.existsSync(dbPath)) { if (!fs.existsSync(dbPath)) {
this.logger.error('数据库文件不存在：', dbPath); this.logger.error("数据库文件不存在：", dbPath);
return; return;
} }
const dbTmpFilename = `${this.filePrefix}_${dayjs().format('YYYYMMDD_HHmmss')}_sqlite`; const dbTmpFilename = `${this.filePrefix}_${dayjs().format("YYYYMMDD_HHmmss")}_sqlite`;
const dbZipFilename = `${dbTmpFilename}.zip`; const dbZipFilename = `${dbTmpFilename}.zip`;
const tempDir = path.resolve(os.tmpdir(), 'certd_backup'); const tempDir = path.resolve(os.tmpdir(), "certd_backup");
if (!fs.existsSync(tempDir)) { if (!fs.existsSync(tempDir)) {
await fs.promises.mkdir(tempDir, {recursive: true}); await fs.promises.mkdir(tempDir, { recursive: true });
} }
const dbTmpPath = path.resolve(tempDir, dbTmpFilename); const dbTmpPath = path.resolve(tempDir, dbTmpFilename);
const dbZipPath = path.resolve(tempDir, dbZipFilename); const dbZipPath = path.resolve(tempDir, dbZipFilename);
//复制到临时目录 try {
await fs.promises.copyFile(dbPath, dbTmpPath); //复制到临时目录
//本地压缩 await fs.promises.copyFile(dbPath, dbTmpPath);
const zip = new JSZip(); // //本地压缩
const stream = fs.createReadStream(dbTmpPath); // const zip = new JSZip();
// 使用流的方式添加文件内容 // const stream = fs.createReadStream(dbTmpPath);
zip.file(dbTmpFilename, stream, {binary: true, compression: 'DEFLATE'}); // // 使用流的方式添加文件内容
// zip.file(dbTmpFilename, stream, {binary: true, compression: 'DEFLATE'});
const uploadDir = path.resolve('data', 'upload'); // const uploadDir = path.resolve('data', 'upload');
if (this.withUpload && fs.existsSync(uploadDir)) { // if (this.withUpload && fs.existsSync(uploadDir)) {
zip.folder(uploadDir); // zip.folder(uploadDir);
} // }
const content = await zip.generateAsync({type: 'nodebuffer'}); // const content = await zip.generateAsync({type: 'nodebuffer'});
await fs.promises.writeFile(dbZipPath, content); // await fs.promises.writeFile(dbZipPath, content);
this.logger.info(`数据库文件压缩完成:${dbZipPath}`); // 创建可写流
const outputStream = fs.createWriteStream(dbZipPath);
const zip = new JSZip();
this.logger.info('开始备份，当前备份方式：', this.backupMode); // 添加数据库文件
const backupDir = this.backupDir || defaultBackupDir; const dbStream = fs.createReadStream(dbTmpPath);
const backupFilePath = `${backupDir}/${dbZipFilename}`; zip.file(dbTmpFilename, dbStream, { binary: true, compression: "DEFLATE" });
try{ // 处理上传目录
if (this.backupMode === 'local') { const uploadDir = path.resolve("data", "upload");
if (this.withUpload && fs.existsSync(uploadDir)) {
zip.folder("upload"); // 注意：这里应该是相对路径
}
// 使用流式生成
const zipStream = zip.generateNodeStream({
type: "nodebuffer",
streamFiles: true,
compression: "DEFLATE",
});
// 管道传输
await pipeline(zipStream, outputStream);
this.logger.info(`数据库文件压缩完成:${dbZipPath}`);
this.logger.info("开始备份，当前备份方式：", this.backupMode);
const backupDir = this.backupDir || defaultBackupDir;
const backupFilePath = `${backupDir}/${dbZipFilename}`;
if (this.backupMode === "local") {
await this.localBackup(dbZipPath, backupDir, backupFilePath); await this.localBackup(dbZipPath, backupDir, backupFilePath);
} else if (this.backupMode === 'ssh') { } else if (this.backupMode === "ssh") {
await this.sshBackup(dbZipPath, backupDir, backupFilePath); await this.sshBackup(dbZipPath, backupDir, backupFilePath);
} else if (this.backupMode === 'oss') { } else if (this.backupMode === "oss") {
await this.ossBackup(dbZipPath, backupDir, backupFilePath); await this.ossBackup(dbZipPath, backupDir, backupFilePath);
} else { } else {
throw new Error(`不支持的备份方式:${this.backupMode}`); throw new Error(`不支持的备份方式:${this.backupMode}`);
} }
}finally{ } finally {
//删除临时目录 //删除临时目录
await fs.promises.rm(tempDir, {recursive: true, force: true}); await fs.promises.rm(tempDir, { recursive: true, force: true });
} }
this.logger.info("数据库备份完成");
this.logger.info('数据库备份完成');
} }
private async localBackup(dbPath: string, backupDir: string, backupPath: string) { private async localBackup(dbPath: string, backupDir: string, backupPath: string) {
if (!backupPath.startsWith('/')) { if (!backupPath.startsWith("/")) {
backupPath = path.join('./data/', backupPath); backupPath = path.join("./data/", backupPath);
} }
const dir = path.dirname(backupPath); const dir = path.dirname(backupPath);
if (!fs.existsSync(dir)) { if (!fs.existsSync(dir)) {
await fs.promises.mkdir(dir, {recursive: true}); await fs.promises.mkdir(dir, { recursive: true });
} }
backupPath = path.resolve(backupPath); backupPath = path.resolve(backupPath);
await fs.promises.copyFile(dbPath, backupPath); await fs.promises.copyFile(dbPath, backupPath);
this.logger.info('备份文件路径：', backupPath); this.logger.info("备份文件路径：", backupPath);
if (this.retainDays > 0) { if (this.retainDays > 0) {
// 删除过期备份 // 删除过期备份
this.logger.info('开始删除过期备份文件'); this.logger.info("开始删除过期备份文件");
const files = fs.readdirSync(dir); const files = fs.readdirSync(dir);
const now = Date.now(); const now = Date.now();
let count = 0; let count = 0;
@@ -245,76 +263,76 @@ export class DBBackupPlugin extends AbstractPlusTaskPlugin {
if (now - stat.mtimeMs > this.retainDays * 24 * 60 * 60 * 1000) { if (now - stat.mtimeMs > this.retainDays * 24 * 60 * 60 * 1000) {
fs.unlinkSync(filePath as fs.PathLike); fs.unlinkSync(filePath as fs.PathLike);
count++; count++;
this.logger.info('删除过期备份文件：', filePath); this.logger.info("删除过期备份文件：", filePath);
} }
}); });
this.logger.info('删除过期备份文件数:', count); this.logger.info("删除过期备份文件数:", count);
} }
} }
private async sshBackup(dbPath: string, backupDir: string, backupPath: string) { private async sshBackup(dbPath: string, backupDir: string, backupPath: string) {
const access: SshAccess = await this.getAccess(this.sshAccessId); const access: SshAccess = await this.getAccess(this.sshAccessId);
const sshClient = new SshClient(this.logger); const sshClient = new SshClient(this.logger);
this.logger.info('备份目录：', backupPath); this.logger.info("备份目录：", backupPath);
await sshClient.uploadFiles({ await sshClient.uploadFiles({
connectConf: access, connectConf: access,
transports: [{localPath: dbPath, remotePath: backupPath}], transports: [{ localPath: dbPath, remotePath: backupPath }],
mkdirs: true, mkdirs: true,
}); });
this.logger.info('备份文件上传完成'); this.logger.info("备份文件上传完成");
if (this.retainDays > 0) { if (this.retainDays > 0) {
// 删除过期备份 // 删除过期备份
this.logger.info('开始删除过期备份文件'); this.logger.info("开始删除过期备份文件");
const isWin = access.windows; const isWin = access.windows;
let script: string[] = []; let script: string[] = [];
if (isWin) { if (isWin) {
throw new Error('删除过期文件暂不支持windows系统'); throw new Error("删除过期文件暂不支持windows系统");
// script = `forfiles /p ${backupDir} /s /d -${this.retainDays} /c "cmd /c del @path"`; // script = `forfiles /p ${backupDir} /s /d -${this.retainDays} /c "cmd /c del @path"`;
} else { } else {
script = [`cd ${backupDir}`, 'echo 备份目录', 'pwd', `find . -type f -mtime +${this.retainDays} -name '${this.filePrefix}*' -exec rm -f {} \\;`]; script = [`cd ${backupDir}`, "echo 备份目录", "pwd", `find . -type f -mtime +${this.retainDays} -name '${this.filePrefix}*' -exec rm -f {} \\;`];
} }
await sshClient.exec({ await sshClient.exec({
connectConf: access, connectConf: access,
script, script,
}); });
this.logger.info('删除过期备份文件完成'); this.logger.info("删除过期备份文件完成");
} }
} }
private async ossBackup(dbPath: string, backupDir: string, backupPath: string) { private async ossBackup(dbPath: string, backupDir: string, backupPath: string) {
if (!this.ossAccessId) { if (!this.ossAccessId) {
throw new Error('未配置ossAccessId'); throw new Error("未配置ossAccessId");
} }
const access = await this.getAccess(this.ossAccessId); const access = await this.getAccess(this.ossAccessId);
const ossType = this.ossType const ossType = this.ossType;
const ctx: OssClientContext = { const ctx: OssClientContext = {
logger: this.logger, logger: this.logger,
utils: this.ctx.utils, utils: this.ctx.utils,
accessService:this.accessService accessService: this.accessService,
} };
this.logger.info(`开始备份文件到:${ossType}`); this.logger.info(`开始备份文件到:${ossType}`);
const client = await ossClientFactory.createOssClientByType(ossType, { const client = await ossClientFactory.createOssClientByType(ossType, {
access, access,
ctx, ctx,
}) });
await client.upload(backupPath, dbPath); await client.upload(backupPath, dbPath);
if (this.retainDays > 0) { if (this.retainDays > 0) {
// 删除过期备份 // 删除过期备份
this.logger.info('开始删除过期备份文件'); this.logger.info("开始删除过期备份文件");
const removeByOpts: OssClientRemoveByOpts = { const removeByOpts: OssClientRemoveByOpts = {
dir: backupDir, dir: backupDir,
beforeDays: this.retainDays, beforeDays: this.retainDays,
}; };
await client.removeBy(removeByOpts); await client.removeBy(removeByOpts);
this.logger.info('删除过期备份文件完成'); this.logger.info("删除过期备份文件完成");
}else{ } else {
this.logger.info('已禁止删除过期文件'); this.logger.info("已禁止删除过期文件");
} }
} }
} }