perf: 支持同步域名过期时间

packages/core/pipeline/src/context/index.ts

@@ -34,3 +34,29 @@ export class Pager {
     this.pageNo = Math.ceil(offset / (this.pageSize ?? 50)) + 1;
   }
 }
+
+export async function doPageTurn<T>(req: { pager: Pager; getPage: (pager: Pager) => Promise<PageRes<T>>; itemHandle?: (item: T) => Promise<void>; batchHandle?: (pageRes: PageRes<T>) => Promise<void> }) {
+  let count = 0;
+  const { pager, getPage, itemHandle, batchHandle } = req;
+  while (true) {
+    const pageRes = await getPage(pager);
+    if (!pageRes || !pageRes.list || pageRes.list.length === 0) {
+      break;
+    }
+    count += pageRes.list.length;
+    if (batchHandle) {
+      await batchHandle(pageRes);
+    }
+    if (itemHandle) {
+      for (const item of pageRes.list) {
+        await itemHandle(item);
+      }
+    }
+    if (pageRes.total && pageRes.total >= 0 && count >= pageRes.total) {
+      //遍历完成
+      break;
+    }
+    pager.pageNo++;
+  }
+  return count;
+}

packages/plugins/plugin-lib/src/cert/dns-provider/domain-parser.ts

@@ -4,6 +4,14 @@ import psl from "psl";
 import { ILogger, utils, logger as globalLogger } from "@certd/basic";
 import { resolveDomainBySoaRecord } from "@certd/acme-client";
 
+export function parseDomainByPsl(fullDomain: string) {
+  const parsed = psl.parse(fullDomain) as psl.ParsedDomain;
+  if (parsed.error) {
+    throw new Error(`解析${fullDomain}域名失败:` + JSON.stringify(parsed.error));
+  }
+  return parsed;
+}
+
 export class DomainParser implements IDomainParser {
   subDomainsGetter: ISubDomainsGetter;
   logger: ILogger;
@@ -13,11 +21,7 @@ export class DomainParser implements IDomainParser {
   }
 
   parseDomainByPsl(fullDomain: string) {
-    const parsed = psl.parse(fullDomain) as psl.ParsedDomain;
+    return parseDomainByPsl(fullDomain).domain as string;
-    if (parsed.error) {
-      throw new Error(`解析${fullDomain}域名失败:` + JSON.stringify(parsed.error));
-    }
-    return parsed.domain as string;
   }
 
   async parse(fullDomain: string) {

packages/ui/certd-client/src/views/certd/cert/domain/api.ts

@@ -65,3 +65,10 @@ export async function SyncSubmit(body: any) {
     data: body,
   });
 }
+
+export async function SyncDomainsExpiration() {
+  return await request({
+    url: apiPrefix + "/sync/expiration",
+    method: "post",
+  });
+}

packages/ui/certd-client/src/views/certd/cert/domain/crud.tsx

@@ -7,7 +7,7 @@ import { useUserStore } from "/@/store/user";
 import { useSettingStore } from "/@/store/settings";
 import { Dicts } from "/@/components/plugins/lib/dicts";
 import { createAccessApi } from "/@/views/certd/access/api";
-import { Modal } from "ant-design-vue";
+import { Modal, notification } from "ant-design-vue";
 import { useDomainImport } from "./use";
 
 export default function ({ crudExpose, context }: CreateCrudOptionsProps): CreateCrudOptionsRet {
@@ -98,7 +98,27 @@ export default function ({ crudExpose, context }: CreateCrudOptionsProps): Creat
             type: "primary",
             text: "从域名提供商导入",
             click: () => {
-              openDomainImportDialog();
+              openDomainImportDialog({
+                afterSubmit: () => {
+                  setTimeout(() => {
+                    crudExpose.doRefresh();
+                  }, 2000);
+                },
+              });
+            },
+          },
+          syncExpirationDate: {
+            title: "同步域名过期时间",
+            type: "primary",
+            text: "同步域名过期时间",
+            click: async () => {
+              await api.SyncDomainsExpiration();
+              notification.success({
+                message: "同步任务已提交",
+              });
+              setTimeout(() => {
+                crudExpose.doRefresh();
+              }, 2000);
             },
           },
         },

packages/ui/certd-client/src/views/certd/cert/domain/use.tsx

@@ -48,7 +48,7 @@ export function useDomainImport() {
     },
   };
 
-  return function openDomainImportDialog() {
+  return function openDomainImportDialog(req: { afterSubmit?: () => void }) {
     openFormDialog({
       title: "从域名提供商导入域名",
       columns: columns,
@@ -58,6 +58,9 @@ export function useDomainImport() {
           dnsProviderAccessId: form.dnsProviderAccessId,
         });
         message.success("导入任务已提交");
+        if (req.afterSubmit) {
+          req.afterSubmit();
+        }
       },
     });
   };

packages/ui/certd-server/src/controller/user/cert/domain-controller.ts

@@ -80,12 +80,20 @@ export class DomainController extends CrudController<DomainService> {
 
 
   @Post('/sync/submit', { summary: Constants.per.authOnly })
-  async sync(@Body(ALL) body: any) {
+  async syncSubmit(@Body(ALL) body: any) {
     const { dnsProviderType, dnsProviderAccessId } = body;
     const req = {
       dnsProviderType, dnsProviderAccessId, userId: this.getUserId(),
     }
-    await this.service.syncFromProvider(req);
+    await this.service.doSyncFromProvider(req);
+    return this.ok();
+  }
+
+  @Post('/sync/expiration', { summary: Constants.per.authOnly })
+  async syncExpiration(@Body(ALL) body: any) {
+    await this.service.doSyncDomainsExpirationDate({
+      userId: this.getUserId(),
+    })
     return this.ok();
   }
 

packages/ui/certd-server/src/modules/auto/auto-c-register-cron.ts

@@ -6,11 +6,12 @@ import {SiteInfoService} from '../monitor/index.js';
 import {Cron} from '../cron/cron.js';
 import {UserSettingsService} from "../mine/service/user-settings-service.js";
 import {UserSiteMonitorSetting} from "../mine/service/models.js";
-import {getPlusInfo} from "@certd/plus-core";
+import {getPlusInfo, isPlus} from "@certd/plus-core";
 import dayjs from "dayjs";
 import {NotificationService} from "../pipeline/service/notification-service.js";
 import {UserService} from "../sys/authority/service/user-service.js";
 import {Between} from "typeorm";
+import { DomainService } from '../cert/service/domain-service.js';
 
 @Autoload()
 @Scope(ScopeEnum.Request, { allowDowngrade: true })
@@ -44,6 +45,9 @@ export class AutoCRegisterCron {
   @Inject()
   userService: UserService;
 
+  @Inject()
+  domainService: DomainService;
+
 
   @Init()
   async init() {
@@ -60,7 +64,9 @@ export class AutoCRegisterCron {
 
     await this.registerPlusExpireCheckCron();
 
-    await this.registerUserExpireCheckCron()
+    await this.registerUserExpireCheckCron();
+
+    await this.registerDomainExpireCheckCron();
   }
 
   async registerSiteMonitorCron() {
@@ -199,4 +205,23 @@ export class AutoCRegisterCron {
       }
     })
   }
+
+
+  registerDomainExpireCheckCron(){
+    if (!isPlus()){
+      return
+    }
+    // 添加域名即将到期检查任务
+    const randomWeek = Math.floor(Math.random() * 7) + 1
+    const randomHour = Math.floor(Math.random() * 24)
+    const randomMinute = Math.floor(Math.random() * 60)
+    logger.info(`注册域名注册过期时间检查任务，每周${randomWeek} ${randomHour}:${randomMinute}检查一次`)
+    this.cron.register({
+      name: 'domain-expire-check',
+      cron: `0 ${randomMinute} ${randomHour} ? * ${randomWeek}`, // 每周随机一天检查一次
+      job: async () => {
+        await this.domainService.doSyncDomainsExpirationDate({})
+      }
+    })
+  }
 }

packages/ui/certd-server/src/modules/cert/service/domain-service.ts

@@ -1,17 +1,19 @@
+import { http, logger, utils } from '@certd/basic';
+import { AccessService, BaseService } from '@certd/lib-server';
+import { doPageTurn, Pager, PageRes } from '@certd/pipeline';
+import { DomainVerifiers } from "@certd/plugin-cert";
+import { createDnsProvider, DomainParser, parseDomainByPsl } from "@certd/plugin-lib";
 import { Inject, Provide, Scope, ScopeEnum } from '@midwayjs/core';
 import { InjectEntityModel } from '@midwayjs/typeorm';
+import dayjs from 'dayjs';
 import { In, Not, Repository } from 'typeorm';
-import {AccessService, BaseService} from '@certd/lib-server';
-import {DomainEntity} from '../entity/domain.js';
-import {SubDomainService} from "../../pipeline/service/sub-domain-service.js";
-import {createDnsProvider, DomainParser} from "@certd/plugin-lib";
-import {DomainVerifiers} from "@certd/plugin-cert";
-import { SubDomainsGetter } from '../../pipeline/service/getter/sub-domain-getter.js';
-import { CnameRecordService } from '../../cname/service/cname-record-service.js';
 import { CnameRecordEntity } from "../../cname/entity/cname-record.js";
-import { http, logger, utils } from '@certd/basic';
+import { CnameRecordService } from '../../cname/service/cname-record-service.js';
+import { SubDomainsGetter } from '../../pipeline/service/getter/sub-domain-getter.js';
 import { TaskServiceBuilder } from '../../pipeline/service/getter/task-service-getter.js';
-import { Pager } from '@certd/pipeline';
+import { SubDomainService } from "../../pipeline/service/sub-domain-service.js";
+import { DomainEntity } from '../entity/domain.js';
+import { BackTask, taskExecutor } from './task-executor.js';
 
 export interface SyncFromProviderReq {
   userId: number;
@@ -19,6 +21,8 @@ import { Pager } from '@certd/pipeline';
   dnsProviderAccessId: string;
 }
 
+
+
 /**
  *
  */
@@ -203,8 +207,17 @@ export class DomainService extends BaseService<DomainEntity> {
   }
 
 
+  async doSyncFromProvider(req: SyncFromProviderReq) {
+    taskExecutor.start('syncFromProviderTask', new BackTask({
+      key: `user_${req.userId}`,
+      title: `同步用户${req.userId}从域名提供商导入域名`,
+      run: async (task: BackTask) => {
+        await this._syncFromProvider(req, task)
+      },
+    }))
+  }
 
-  async syncFromProvider(req: SyncFromProviderReq) {
+  private async _syncFromProvider(req: SyncFromProviderReq, task: BackTask) {
     const { userId, dnsProviderType, dnsProviderAccessId } = req;
     const subDomainGetter = new SubDomainsGetter(userId, this.subDomainService)
     const domainParser = new DomainParser(subDomainGetter)
@@ -221,11 +234,8 @@ export class DomainService extends BaseService<DomainEntity> {
     const challengeType = "dns"
 
     const importDomain = async (domainRecord: any) => {
+      task.incrementCurrent()
       const domain = domainRecord.domain
-      const certProps :any={
-        registrationDate: domainRecord.registrationDate,
-        expirationDate: domainRecord.expirationDate,
-      }
 
       const old = await this.findOne({
         where: {
@@ -234,15 +244,15 @@ export class DomainService extends BaseService<DomainEntity> {
         }
       })
       if (old) {
+        if (old.fromType !== 'auto') {
+          //如果是手动的，跳过更新校验配置
+          return
+        }
         const updateObj: any = {
           id: old.id,
-           ...certProps
+          dnsProviderType,
-        }
+          dnsProviderAccess: dnsProviderAccessId,
-        if (old.fromType !== 'manual'){
+          challengeType,
-          //如果不是手动的，更新校验配置
-          updateObj.dnsProviderType = dnsProviderType
-          updateObj.dnsProviderAccess = dnsProviderAccessId
-          updateObj.challengeType = challengeType
         }
         //更新
         await super.update(updateObj)
@@ -256,33 +266,132 @@ export class DomainService extends BaseService<DomainEntity> {
           challengeType,
           disabled: false,
           fromType: 'auto',
-          ...certProps
         })
       }
     }
+    const batchHandle = async (pageRes: PageRes<any>) => {
+        task.setTotal(pageRes.total || 0)
+    }
     const start = async () => {
-      let count = 0
+      await doPageTurn({ pager, getPage: dnsProvider.getDomainListPage, itemHandle: importDomain, batchHandle })
-      while(true){
-        const pageRes = await dnsProvider.getDomainListPage(pager)
-        if(!pageRes || !pageRes.list || pageRes.list.length === 0){
-          //遍历完成
-          break
-        }
-        //处理
-        for (const domainRecord of pageRes.list) {
-          await importDomain(domainRecord)
-        }
-
-        count += pageRes.list.length
-        if(pageRes.total>0 && count >= pageRes.total){
-          //遍历完成
-          break
-        }
-        pager.pageNo++
-      }
     }
 
     start()
 
   }
+
+  async doSyncDomainsExpirationDate(req: { userId?: number }) {
+    const userId = req.userId
+    taskExecutor.start('syncDomainsExpirationDateTask', new BackTask({
+      key: `user_${userId}`,
+      title: `同步用户(${userId ?? '全部'})注册域名过期时间`,
+      run: async (task: BackTask) => {
+        await this._syncDomainsExpirationDate({ userId, task })
+      }
+    }))
+  }
+
+  private async _syncDomainsExpirationDate(req: { userId?: number, task: BackTask }) {
+    //同步所有域名的过期时间
+    const pager = new Pager({
+      pageNo: 1,
+      pageSize: 100,
+    })
+
+    const dnsJson = await http.request({
+      url: "https://data.iana.org/rdap/dns.json",
+      method: "GET",
+    })
+    const rdapMap: Record<string, string> = {}
+    for (const item of dnsJson.services) {
+      //  [["store","work"], ["https://rdap.centralnic.com/store/"]],
+      const suffixes = item[0]
+      const urls = item[1]
+      for (const suffix of suffixes) {
+        rdapMap[suffix] = urls[0]
+      }
+    }
+
+    const getDomainExpirationDate = async (domain: string) => {
+      const parsed = parseDomainByPsl(domain)
+      const mainDomain = parsed.domain || ''
+      if (mainDomain !== domain) {
+        logger.warn(`${domain}为子域名，跳过同步`)
+        return
+      }
+      const suffix = parsed.tld || ''
+      const rdapUrl = rdapMap[suffix]
+      if (!rdapUrl) {
+        throw new Error(`未找到${suffix}的rdap地址`)
+      }
+      // https://rdap.nic.work/domain/handsfree.work
+      const rdap = await http.request({
+        url: `${rdapUrl}domain/${domain}`,
+        method: "GET",
+      })
+
+      let res: any = {}
+      const events = rdap.events || []
+      for (const item of events) {
+        if (item.eventAction === 'expiration') {
+          res.expirationDate = dayjs(item.eventDate).valueOf()
+        } else if (item.eventAction === 'registration') {
+          res.registrationDate = dayjs(item.eventDate).valueOf()
+        }
+      }
+      return res
+    }
+    const query: any = {
+      challengeType: "dns",
+    }
+    if (req.userId!=null) {
+      query.userId = req.userId
+    }
+    const getDomainPage = async (pager: Pager) => {
+      const pageRes = await this.page({
+        query: query,
+        buildQuery(bq) {
+          bq.andWhere(" (expiration_date is null or expiration_date < :now) ", { now: dayjs().add(1, 'month').valueOf() })
+        },
+        page: {
+          offset: pager.getOffset(),
+          limit: pager.pageSize,
+        }
+      })
+      req.task.total = pageRes.total
+      return {
+        list: pageRes.records,
+        total: pageRes.total,
+      }
+    }
+
+    const itemHandle = async (item: any) => {
+      req.task.incrementCurrent()
+      try {
+        const res = await getDomainExpirationDate(item.domain)
+        if (!res) {
+          return
+        }
+        const { expirationDate, registrationDate } = res
+        if (!expirationDate) {
+          logger.error(`获取域名${item.domain}过期时间失败`)
+          return
+        }
+        logger.info(`更新域名${item.domain}过期时间:${dayjs(expirationDate).format('YYYY-MM-DD')}`)
+        const updateObj: any = {
+          id: item.id,
+          expirationDate: expirationDate,
+          registrationDate: registrationDate,
+        }
+        //更新
+        await super.update(updateObj)
+      } catch (error) {
+        logger.error(`更新域名${item.domain}过期时间失败:${error}`)
+      } finally {
+        await utils.sleep(1000)
+      }
+    }
+
+    await doPageTurn({ pager, getPage: getDomainPage, itemHandle: itemHandle })
+  }
 }

packages/ui/certd-server/src/modules/cert/service/task-executor.ts

@@ -3,14 +3,22 @@ import { logger } from "@certd/basic"
 export class BackTaskExecutor {
   tasks: Record<string, Record<string, BackTask>> = {}
 
-  add(type:string,task: BackTask){
+  start(type: string, task: BackTask) {
     if (!this.tasks[type]) {
       this.tasks[type] = {}
     }
+    const oldTask = this.tasks[type][task.key]
+    if (oldTask && oldTask.status === "running") {
+      throw new Error(`任务 ${task.key} 正在运行中`)
+    }
     this.tasks[type][task.key] = task
+    this.run(type, task);
   }
 
   get(type: string, key: string) {
+    if (!this.tasks[type]) {
+      this.tasks[type] = {}
+    }
     return this.tasks[type][key]
   }
 
@@ -29,18 +37,15 @@ export class BackTaskExecutor{
     }
   }
 
-  
+  private async run(type: string, task: any) {
-
+    if (task.status === "running") {
-  async run(type:string,key: string){
+      throw new Error(`任务 ${task.key} 正在运行中`)
-    const task = this.tasks[type]?.[key]
-    if (!task) {
-      throw new Error(`任务 ${key} 不存在`)
     }
     task.startTime = Date.now();
     task.clearTimeout();
     try {
       task.status = "running";
-      return await task.run();
+      return await task.run(task);
     } catch (e) {
       logger.error(`任务 ${task.title}[${task.key}] 执行失败`, e.message);
       task.status = "failed";
@@ -50,7 +55,8 @@ export class BackTaskExecutor{
       task.status = "done";
       task.timeoutId = setTimeout(() => {
         this.clear(type, task.key);
-      }, 60*60*1000);
+      }, 24 * 60 * 60 * 1000);
+      delete task.run;
     }
   }
 }
@@ -66,9 +72,12 @@ export class BackTask{
   timeoutId?: NodeJS.Timeout;
 
 
-  run: () => Promise<void>;
+  run: (task: BackTask) => Promise<void>;
 
-  constructor(key:string,title: string,run: () => Promise<void>){
+  constructor(opts:{
+    key: string, title: string, run: (task: BackTask) => Promise<void>
+  }) {
+    const {key, title, run} = opts
     this.key = key;
     this.title = title;
     Object.defineProperty(this, 'run', {
@@ -85,4 +94,13 @@ export class BackTask{
       this.timeoutId = null;
     }
   }
+
+  setTotal(total: number) {
+    this.total = total;
   }
+  incrementCurrent() {
+    this.current++
+  }
+}
+
+export const taskExecutor = new BackTaskExecutor();