From a97cee84f3bfdeeb2083d91f748cac5405fed6ae Mon Sep 17 00:00:00 2001 From: xiaojunnuo Date: Thu, 22 Jan 2026 00:59:28 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E6=94=AF=E6=8C=81=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E5=9F=9F=E5=90=8D=E8=BF=87=E6=9C=9F=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/pipeline/src/context/index.ts | 26 ++ .../src/cert/dns-provider/domain-parser.ts | 14 +- .../src/views/certd/cert/domain/api.ts | 7 + .../src/views/certd/cert/domain/crud.tsx | 24 +- .../src/views/certd/cert/domain/use.tsx | 5 +- .../controller/user/cert/domain-controller.ts | 12 +- .../src/modules/auto/auto-c-register-cron.ts | 29 ++- .../modules/cert/service/domain-service.ts | 233 +++++++++++++----- .../src/modules/cert/service/task-executor.ts | 68 +++-- 9 files changed, 319 insertions(+), 99 deletions(-) diff --git a/packages/core/pipeline/src/context/index.ts b/packages/core/pipeline/src/context/index.ts index 709a1b1af..5fce2dfb6 100644 --- a/packages/core/pipeline/src/context/index.ts +++ b/packages/core/pipeline/src/context/index.ts @@ -34,3 +34,29 @@ export class Pager { this.pageNo = Math.ceil(offset / (this.pageSize ?? 50)) + 1; } } + +export async function doPageTurn(req: { pager: Pager; getPage: (pager: Pager) => Promise>; itemHandle?: (item: T) => Promise; batchHandle?: (pageRes: PageRes) => Promise }) { + let count = 0; + const { pager, getPage, itemHandle, batchHandle } = req; + while (true) { + const pageRes = await getPage(pager); + if (!pageRes || !pageRes.list || pageRes.list.length === 0) { + break; + } + count += pageRes.list.length; + if (batchHandle) { + await batchHandle(pageRes); + } + if (itemHandle) { + for (const item of pageRes.list) { + await itemHandle(item); + } + } + if (pageRes.total && pageRes.total >= 0 && count >= pageRes.total) { + //遍历完成 + break; + } + pager.pageNo++; + } + return count; +} diff --git a/packages/plugins/plugin-lib/src/cert/dns-provider/domain-parser.ts b/packages/plugins/plugin-lib/src/cert/dns-provider/domain-parser.ts index b1c7838da..59d7d4262 100644 --- a/packages/plugins/plugin-lib/src/cert/dns-provider/domain-parser.ts +++ b/packages/plugins/plugin-lib/src/cert/dns-provider/domain-parser.ts @@ -4,6 +4,14 @@ import psl from "psl"; import { ILogger, utils, logger as globalLogger } from "@certd/basic"; import { resolveDomainBySoaRecord } from "@certd/acme-client"; +export function parseDomainByPsl(fullDomain: string) { + const parsed = psl.parse(fullDomain) as psl.ParsedDomain; + if (parsed.error) { + throw new Error(`解析${fullDomain}域名失败:` + JSON.stringify(parsed.error)); + } + return parsed; +} + export class DomainParser implements IDomainParser { subDomainsGetter: ISubDomainsGetter; logger: ILogger; @@ -13,11 +21,7 @@ export class DomainParser implements IDomainParser { } parseDomainByPsl(fullDomain: string) { - const parsed = psl.parse(fullDomain) as psl.ParsedDomain; - if (parsed.error) { - throw new Error(`解析${fullDomain}域名失败:` + JSON.stringify(parsed.error)); - } - return parsed.domain as string; + return parseDomainByPsl(fullDomain).domain as string; } async parse(fullDomain: string) { diff --git a/packages/ui/certd-client/src/views/certd/cert/domain/api.ts b/packages/ui/certd-client/src/views/certd/cert/domain/api.ts index 75d76496d..778281ef6 100644 --- a/packages/ui/certd-client/src/views/certd/cert/domain/api.ts +++ b/packages/ui/certd-client/src/views/certd/cert/domain/api.ts @@ -65,3 +65,10 @@ export async function SyncSubmit(body: any) { data: body, }); } + +export async function SyncDomainsExpiration() { + return await request({ + url: apiPrefix + "/sync/expiration", + method: "post", + }); +} diff --git a/packages/ui/certd-client/src/views/certd/cert/domain/crud.tsx b/packages/ui/certd-client/src/views/certd/cert/domain/crud.tsx index 7178ab718..0bfe78d9e 100644 --- a/packages/ui/certd-client/src/views/certd/cert/domain/crud.tsx +++ b/packages/ui/certd-client/src/views/certd/cert/domain/crud.tsx @@ -7,7 +7,7 @@ import { useUserStore } from "/@/store/user"; import { useSettingStore } from "/@/store/settings"; import { Dicts } from "/@/components/plugins/lib/dicts"; import { createAccessApi } from "/@/views/certd/access/api"; -import { Modal } from "ant-design-vue"; +import { Modal, notification } from "ant-design-vue"; import { useDomainImport } from "./use"; export default function ({ crudExpose, context }: CreateCrudOptionsProps): CreateCrudOptionsRet { @@ -98,7 +98,27 @@ export default function ({ crudExpose, context }: CreateCrudOptionsProps): Creat type: "primary", text: "从域名提供商导入", click: () => { - openDomainImportDialog(); + openDomainImportDialog({ + afterSubmit: () => { + setTimeout(() => { + crudExpose.doRefresh(); + }, 2000); + }, + }); + }, + }, + syncExpirationDate: { + title: "同步域名过期时间", + type: "primary", + text: "同步域名过期时间", + click: async () => { + await api.SyncDomainsExpiration(); + notification.success({ + message: "同步任务已提交", + }); + setTimeout(() => { + crudExpose.doRefresh(); + }, 2000); }, }, }, diff --git a/packages/ui/certd-client/src/views/certd/cert/domain/use.tsx b/packages/ui/certd-client/src/views/certd/cert/domain/use.tsx index 59abac0be..4d77e4e3b 100644 --- a/packages/ui/certd-client/src/views/certd/cert/domain/use.tsx +++ b/packages/ui/certd-client/src/views/certd/cert/domain/use.tsx @@ -48,7 +48,7 @@ export function useDomainImport() { }, }; - return function openDomainImportDialog() { + return function openDomainImportDialog(req: { afterSubmit?: () => void }) { openFormDialog({ title: "从域名提供商导入域名", columns: columns, @@ -58,6 +58,9 @@ export function useDomainImport() { dnsProviderAccessId: form.dnsProviderAccessId, }); message.success("导入任务已提交"); + if (req.afterSubmit) { + req.afterSubmit(); + } }, }); }; diff --git a/packages/ui/certd-server/src/controller/user/cert/domain-controller.ts b/packages/ui/certd-server/src/controller/user/cert/domain-controller.ts index 2fa741f20..422fd126b 100644 --- a/packages/ui/certd-server/src/controller/user/cert/domain-controller.ts +++ b/packages/ui/certd-server/src/controller/user/cert/domain-controller.ts @@ -80,12 +80,20 @@ export class DomainController extends CrudController { @Post('/sync/submit', { summary: Constants.per.authOnly }) - async sync(@Body(ALL) body: any) { + async syncSubmit(@Body(ALL) body: any) { const { dnsProviderType, dnsProviderAccessId } = body; const req = { dnsProviderType, dnsProviderAccessId, userId: this.getUserId(), } - await this.service.syncFromProvider(req); + await this.service.doSyncFromProvider(req); + return this.ok(); + } + + @Post('/sync/expiration', { summary: Constants.per.authOnly }) + async syncExpiration(@Body(ALL) body: any) { + await this.service.doSyncDomainsExpirationDate({ + userId: this.getUserId(), + }) return this.ok(); } diff --git a/packages/ui/certd-server/src/modules/auto/auto-c-register-cron.ts b/packages/ui/certd-server/src/modules/auto/auto-c-register-cron.ts index 762411971..eada89e23 100644 --- a/packages/ui/certd-server/src/modules/auto/auto-c-register-cron.ts +++ b/packages/ui/certd-server/src/modules/auto/auto-c-register-cron.ts @@ -6,11 +6,12 @@ import {SiteInfoService} from '../monitor/index.js'; import {Cron} from '../cron/cron.js'; import {UserSettingsService} from "../mine/service/user-settings-service.js"; import {UserSiteMonitorSetting} from "../mine/service/models.js"; -import {getPlusInfo} from "@certd/plus-core"; +import {getPlusInfo, isPlus} from "@certd/plus-core"; import dayjs from "dayjs"; import {NotificationService} from "../pipeline/service/notification-service.js"; import {UserService} from "../sys/authority/service/user-service.js"; import {Between} from "typeorm"; +import { DomainService } from '../cert/service/domain-service.js'; @Autoload() @Scope(ScopeEnum.Request, { allowDowngrade: true }) @@ -44,6 +45,9 @@ export class AutoCRegisterCron { @Inject() userService: UserService; + @Inject() + domainService: DomainService; + @Init() async init() { @@ -60,7 +64,9 @@ export class AutoCRegisterCron { await this.registerPlusExpireCheckCron(); - await this.registerUserExpireCheckCron() + await this.registerUserExpireCheckCron(); + + await this.registerDomainExpireCheckCron(); } async registerSiteMonitorCron() { @@ -199,4 +205,23 @@ export class AutoCRegisterCron { } }) } + + + registerDomainExpireCheckCron(){ + if (!isPlus()){ + return + } + // 添加域名即将到期检查任务 + const randomWeek = Math.floor(Math.random() * 7) + 1 + const randomHour = Math.floor(Math.random() * 24) + const randomMinute = Math.floor(Math.random() * 60) + logger.info(`注册域名注册过期时间检查任务，每周${randomWeek} ${randomHour}:${randomMinute}检查一次`) + this.cron.register({ + name: 'domain-expire-check', + cron: `0 ${randomMinute} ${randomHour} ? * ${randomWeek}`, // 每周随机一天检查一次 + job: async () => { + await this.domainService.doSyncDomainsExpirationDate({}) + } + }) + } } diff --git a/packages/ui/certd-server/src/modules/cert/service/domain-service.ts b/packages/ui/certd-server/src/modules/cert/service/domain-service.ts index 02a372893..b0163e69c 100644 --- a/packages/ui/certd-server/src/modules/cert/service/domain-service.ts +++ b/packages/ui/certd-server/src/modules/cert/service/domain-service.ts @@ -1,29 +1,33 @@ -import {Inject, Provide, Scope, ScopeEnum} from '@midwayjs/core'; -import {InjectEntityModel} from '@midwayjs/typeorm'; -import {In, Not, Repository} from 'typeorm'; -import {AccessService, BaseService} from '@certd/lib-server'; -import {DomainEntity} from '../entity/domain.js'; -import {SubDomainService} from "../../pipeline/service/sub-domain-service.js"; -import {createDnsProvider, DomainParser} from "@certd/plugin-lib"; -import {DomainVerifiers} from "@certd/plugin-cert"; -import { SubDomainsGetter } from '../../pipeline/service/getter/sub-domain-getter.js'; -import { CnameRecordService } from '../../cname/service/cname-record-service.js'; -import { CnameRecordEntity } from "../../cname/entity/cname-record.js"; import { http, logger, utils } from '@certd/basic'; +import { AccessService, BaseService } from '@certd/lib-server'; +import { doPageTurn, Pager, PageRes } from '@certd/pipeline'; +import { DomainVerifiers } from "@certd/plugin-cert"; +import { createDnsProvider, DomainParser, parseDomainByPsl } from "@certd/plugin-lib"; +import { Inject, Provide, Scope, ScopeEnum } from '@midwayjs/core'; +import { InjectEntityModel } from '@midwayjs/typeorm'; +import dayjs from 'dayjs'; +import { In, Not, Repository } from 'typeorm'; +import { CnameRecordEntity } from "../../cname/entity/cname-record.js"; +import { CnameRecordService } from '../../cname/service/cname-record-service.js'; +import { SubDomainsGetter } from '../../pipeline/service/getter/sub-domain-getter.js'; import { TaskServiceBuilder } from '../../pipeline/service/getter/task-service-getter.js'; -import { Pager } from '@certd/pipeline'; +import { SubDomainService } from "../../pipeline/service/sub-domain-service.js"; +import { DomainEntity } from '../entity/domain.js'; +import { BackTask, taskExecutor } from './task-executor.js'; - export interface SyncFromProviderReq { +export interface SyncFromProviderReq { userId: number; dnsProviderType: string; dnsProviderAccessId: string; } + + /** * */ @Provide() -@Scope(ScopeEnum.Request, {allowDowngrade: true}) +@Scope(ScopeEnum.Request, { allowDowngrade: true }) export class DomainService extends BaseService { @InjectEntityModel(DomainEntity) repository: Repository; @@ -45,7 +49,7 @@ export class DomainService extends BaseService { } async add(param) { - if (param.userId == null ){ + if (param.userId == null) { throw new Error('userId 不能为空'); } if (!param.domain) { @@ -97,9 +101,9 @@ export class DomainService extends BaseService { * @param userId * @param domains //去除* 且去重之后的域名列表 */ - async getDomainVerifiers(userId: number, domains: string[]):Promise { + async getDomainVerifiers(userId: number, domains: string[]): Promise { - const mainDomainMap:Record = {} + const mainDomainMap: Record = {} const subDomainGetter = new SubDomainsGetter(userId, this.subDomainService) const domainParser = new DomainParser(subDomainGetter) @@ -111,7 +115,7 @@ export class DomainService extends BaseService { } //匹配DNS记录 - let allDomains = [...domains,...mainDomains] + let allDomains = [...domains, ...mainDomains] //去重 allDomains = [...new Set(allDomains)] @@ -120,16 +124,16 @@ export class DomainService extends BaseService { where: { domain: In(allDomains), userId, - disabled:false, + disabled: false, } }) - const dnsMap = domainRecords.filter(item=>item.challengeType === 'dns').reduce((pre, item) => { + const dnsMap = domainRecords.filter(item => item.challengeType === 'dns').reduce((pre, item) => { pre[item.domain] = item return pre }, {}) - const httpMap = domainRecords.filter(item=>item.challengeType === 'http').reduce((pre, item) => { + const httpMap = domainRecords.filter(item => item.challengeType === 'http').reduce((pre, item) => { pre[item.domain] = item return pre }, {}) @@ -150,7 +154,7 @@ export class DomainService extends BaseService { }, {}) //构建域名验证计划 - const domainVerifiers:DomainVerifiers = {} + const domainVerifiers: DomainVerifiers = {} for (const domain of domains) { const mainDomain = mainDomainMap[domain] @@ -168,7 +172,7 @@ export class DomainService extends BaseService { } continue } - const cnameRecord:CnameRecordEntity = cnameMap[domain] + const cnameRecord: CnameRecordEntity = cnameMap[domain] if (cnameRecord) { domainVerifiers[domain] = { domain, @@ -194,7 +198,7 @@ export class DomainService extends BaseService { httpUploadRootDir: httpRecord.httpUploadRootDir } } - continue + continue } domainVerifiers[domain] = null } @@ -202,9 +206,18 @@ export class DomainService extends BaseService { return domainVerifiers; } - - async syncFromProvider(req: SyncFromProviderReq) { + async doSyncFromProvider(req: SyncFromProviderReq) { + taskExecutor.start('syncFromProviderTask', new BackTask({ + key: `user_${req.userId}`, + title: `同步用户${req.userId}从域名提供商导入域名`, + run: async (task: BackTask) => { + await this._syncFromProvider(req, task) + }, + })) + } + + private async _syncFromProvider(req: SyncFromProviderReq, task: BackTask) { const { userId, dnsProviderType, dnsProviderAccessId } = req; const subDomainGetter = new SubDomainsGetter(userId, this.subDomainService) const domainParser = new DomainParser(subDomainGetter) @@ -212,20 +225,17 @@ export class DomainService extends BaseService { const access = await this.accessService.getById(dnsProviderAccessId, userId); const context = { access, logger, http, utils, domainParser, serviceGetter }; // 翻页查询dns的记录 - const dnsProvider = await createDnsProvider({dnsProviderType,context}) - + const dnsProvider = await createDnsProvider({ dnsProviderType, context }) + const pager = new Pager({ pageNo: 1, pageSize: 100, }) const challengeType = "dns" - const importDomain = async(domainRecord: any) =>{ + const importDomain = async (domainRecord: any) => { + task.incrementCurrent() const domain = domainRecord.domain - const certProps :any={ - registrationDate: domainRecord.registrationDate, - expirationDate: domainRecord.expirationDate, - } const old = await this.findOne({ where: { @@ -234,15 +244,15 @@ export class DomainService extends BaseService { } }) if (old) { - const updateObj :any={ - id: old.id, - ...certProps + if (old.fromType !== 'auto') { + //如果是手动的，跳过更新校验配置 + return } - if (old.fromType !== 'manual'){ - //如果不是手动的，更新校验配置 - updateObj.dnsProviderType = dnsProviderType - updateObj.dnsProviderAccess = dnsProviderAccessId - updateObj.challengeType = challengeType + const updateObj: any = { + id: old.id, + dnsProviderType, + dnsProviderAccess: dnsProviderAccessId, + challengeType, } //更新 await super.update(updateObj) @@ -256,33 +266,132 @@ export class DomainService extends BaseService { challengeType, disabled: false, fromType: 'auto', - ...certProps }) } } - const start = async ()=>{ - let count = 0 - while(true){ - const pageRes = await dnsProvider.getDomainListPage(pager) - if(!pageRes || !pageRes.list || pageRes.list.length === 0){ - //遍历完成 - break - } - //处理 - for (const domainRecord of pageRes.list) { - await importDomain(domainRecord) - } - - count += pageRes.list.length - if(pageRes.total>0 && count >= pageRes.total){ - //遍历完成 - break - } - pager.pageNo++ - } + const batchHandle = async (pageRes: PageRes) => { + task.setTotal(pageRes.total || 0) + } + const start = async () => { + await doPageTurn({ pager, getPage: dnsProvider.getDomainListPage, itemHandle: importDomain, batchHandle }) } start() } + + async doSyncDomainsExpirationDate(req: { userId?: number }) { + const userId = req.userId + taskExecutor.start('syncDomainsExpirationDateTask', new BackTask({ + key: `user_${userId}`, + title: `同步用户(${userId ?? '全部'})注册域名过期时间`, + run: async (task: BackTask) => { + await this._syncDomainsExpirationDate({ userId, task }) + } + })) + } + + private async _syncDomainsExpirationDate(req: { userId?: number, task: BackTask }) { + //同步所有域名的过期时间 + const pager = new Pager({ + pageNo: 1, + pageSize: 100, + }) + + const dnsJson = await http.request({ + url: "https://data.iana.org/rdap/dns.json", + method: "GET", + }) + const rdapMap: Record = {} + for (const item of dnsJson.services) { + // [["store","work"], ["https://rdap.centralnic.com/store/"]], + const suffixes = item[0] + const urls = item[1] + for (const suffix of suffixes) { + rdapMap[suffix] = urls[0] + } + } + + const getDomainExpirationDate = async (domain: string) => { + const parsed = parseDomainByPsl(domain) + const mainDomain = parsed.domain || '' + if (mainDomain !== domain) { + logger.warn(`${domain}为子域名，跳过同步`) + return + } + const suffix = parsed.tld || '' + const rdapUrl = rdapMap[suffix] + if (!rdapUrl) { + throw new Error(`未找到${suffix}的rdap地址`) + } + // https://rdap.nic.work/domain/handsfree.work + const rdap = await http.request({ + url: `${rdapUrl}domain/${domain}`, + method: "GET", + }) + + let res: any = {} + const events = rdap.events || [] + for (const item of events) { + if (item.eventAction === 'expiration') { + res.expirationDate = dayjs(item.eventDate).valueOf() + } else if (item.eventAction === 'registration') { + res.registrationDate = dayjs(item.eventDate).valueOf() + } + } + return res + } + const query: any = { + challengeType: "dns", + } + if (req.userId!=null) { + query.userId = req.userId + } + const getDomainPage = async (pager: Pager) => { + const pageRes = await this.page({ + query: query, + buildQuery(bq) { + bq.andWhere(" (expiration_date is null or expiration_date < :now) ", { now: dayjs().add(1, 'month').valueOf() }) + }, + page: { + offset: pager.getOffset(), + limit: pager.pageSize, + } + }) + req.task.total = pageRes.total + return { + list: pageRes.records, + total: pageRes.total, + } + } + + const itemHandle = async (item: any) => { + req.task.incrementCurrent() + try { + const res = await getDomainExpirationDate(item.domain) + if (!res) { + return + } + const { expirationDate, registrationDate } = res + if (!expirationDate) { + logger.error(`获取域名${item.domain}过期时间失败`) + return + } + logger.info(`更新域名${item.domain}过期时间:${dayjs(expirationDate).format('YYYY-MM-DD')}`) + const updateObj: any = { + id: item.id, + expirationDate: expirationDate, + registrationDate: registrationDate, + } + //更新 + await super.update(updateObj) + } catch (error) { + logger.error(`更新域名${item.domain}过期时间失败:${error}`) + } finally { + await utils.sleep(1000) + } + } + + await doPageTurn({ pager, getPage: getDomainPage, itemHandle: itemHandle }) + } } diff --git a/packages/ui/certd-server/src/modules/cert/service/task-executor.ts b/packages/ui/certd-server/src/modules/cert/service/task-executor.ts index 366461bc8..1e920ea8a 100644 --- a/packages/ui/certd-server/src/modules/cert/service/task-executor.ts +++ b/packages/ui/certd-server/src/modules/cert/service/task-executor.ts @@ -1,27 +1,35 @@ import { logger } from "@certd/basic" -export class BackTaskExecutor{ - tasks :Record> = {} +export class BackTaskExecutor { + tasks: Record> = {} - add(type:string,task: BackTask){ + start(type: string, task: BackTask) { if (!this.tasks[type]) { this.tasks[type] = {} } + const oldTask = this.tasks[type][task.key] + if (oldTask && oldTask.status === "running") { + throw new Error(`任务 ${task.key} 正在运行中`) + } this.tasks[type][task.key] = task + this.run(type, task); } - get(type: string,key: string){ + get(type: string, key: string) { + if (!this.tasks[type]) { + this.tasks[type] = {} + } return this.tasks[type][key] } - removeIsEnd(type: string,key: string){ + removeIsEnd(type: string, key: string) { const task = this.tasks[type]?.[key] if (task && task.status !== "running") { - this.clear(type,key); + this.clear(type, key); } } - clear(type: string,key: string){ + clear(type: string, key: string) { const task = this.tasks[type]?.[key] if (task) { task.clearTimeout(); @@ -29,33 +37,31 @@ export class BackTaskExecutor{ } } - - - async run(type:string,key: string){ - const task = this.tasks[type]?.[key] - if (!task) { - throw new Error(`任务 ${key} 不存在`) + private async run(type: string, task: any) { + if (task.status === "running") { + throw new Error(`任务 ${task.key} 正在运行中`) } task.startTime = Date.now(); task.clearTimeout(); - try{ + try { task.status = "running"; - return await task.run(); - }catch(e){ + return await task.run(task); + } catch (e) { logger.error(`任务 ${task.title}[${task.key}] 执行失败`, e.message); task.status = "failed"; task.error = e.message; - }finally{ + } finally { task.endTime = Date.now(); task.status = "done"; task.timeoutId = setTimeout(() => { - this.clear(type,task.key); - }, 60*60*1000); + this.clear(type, task.key); + }, 24 * 60 * 60 * 1000); + delete task.run; } } } -export class BackTask{ - key:string; +export class BackTask { + key: string; title: string; total: number = 0; current: number = 0; @@ -66,9 +72,12 @@ export class BackTask{ timeoutId?: NodeJS.Timeout; - run: () => Promise; + run: (task: BackTask) => Promise; - constructor(key:string,title: string,run: () => Promise){ + constructor(opts:{ + key: string, title: string, run: (task: BackTask) => Promise + }) { + const {key, title, run} = opts this.key = key; this.title = title; Object.defineProperty(this, 'run', { @@ -79,10 +88,19 @@ export class BackTask{ }); } - clearTimeout(){ + clearTimeout() { if (this.timeoutId) { clearTimeout(this.timeoutId); this.timeoutId = null; } } -} \ No newline at end of file + + setTotal(total: number) { + this.total = total; + } + incrementCurrent() { + this.current++ + } +} + +export const taskExecutor = new BackTaskExecutor(); \ No newline at end of file