perf(cert-plugin): 证书申请支持失败重试

packages/ui/certd-server/src/plugins/plugin-cert/plugin/cert-plugin/acme.ts

@@ -5,6 +5,7 @@ import { IContext } from "@certd/pipeline";
 import { IDnsProvider, IDomainParser } from "@certd/plugin-lib";
 import punycode from "punycode.js";
 import { IOssClient } from "../../../plugin-lib/index.js";
+import { NonRetryableException } from "@certd/lib-server";
 export type CnameVerifyPlan = {
   type?: string;
   domain: string;
@@ -531,38 +532,47 @@ export class AcmeService {
     };
     /* 自动申请证书 */
     const challengePriority = domainsVerifyPlan && Object.values(domainsVerifyPlan).some((item: any) => item?.type === "dns-persist") ? ["dns-persist-01"] : ["dns-01", "http-01"];
-    const crt = await client.auto({
+    try {
-      csr,
+      const crt = await client.auto({
-      email: email,
+        csr,
-      termsOfServiceAgreed: true,
+        email: email,
-      skipChallengeVerification: this.skipLocalVerify,
+        termsOfServiceAgreed: true,
-      challengePriority,
+        skipChallengeVerification: this.skipLocalVerify,
-      challengeCreateFn: async (
+        challengePriority,
-        authz: acme.Authorization,
+        challengeCreateFn: async (
-        keyAuthorizationGetter: (challenge: Challenge) => Promise<string>
+          authz: acme.Authorization,
-      ): Promise<{ recordReq?: any; recordRes?: any; dnsProvider?: any; challenge: Challenge; keyAuthorization: string }> => {
+          keyAuthorizationGetter: (challenge: Challenge) => Promise<string>
-        return await this.challengeCreateFn(authz, keyAuthorizationGetter, providers);
+        ): Promise<{ recordReq?: any; recordRes?: any; dnsProvider?: any; challenge: Challenge; keyAuthorization: string }> => {
-      },
+          return await this.challengeCreateFn(authz, keyAuthorizationGetter, providers);
-      challengeRemoveFn: async (authz: acme.Authorization, challenge: Challenge, keyAuthorization: string, recordReq: any, recordRes: any, dnsProvider: IDnsProvider, httpUploader: IOssClient): Promise<any> => {
+        },
-        return await this.challengeRemoveFn(authz, challenge, keyAuthorization, recordReq, recordRes, dnsProvider, httpUploader);
+        challengeRemoveFn: async (authz: acme.Authorization, challenge: Challenge, keyAuthorization: string, recordReq: any, recordRes: any, dnsProvider: IDnsProvider, httpUploader: IOssClient): Promise<any> => {
-      },
+          return await this.challengeRemoveFn(authz, challenge, keyAuthorization, recordReq, recordRes, dnsProvider, httpUploader);
-      signal: this.options.signal,
+        },
-      profile,
+        signal: this.options.signal,
-      preferredChain,
+        profile,
-      waitDnsDiffuseTime: this.options.waitDnsDiffuseTime,
+        preferredChain,
-    });
+        waitDnsDiffuseTime: this.options.waitDnsDiffuseTime,
+      });
 
-    const crtString = crt.toString();
+      const crtString = crt.toString();
-    const cert: CertInfo = {
+      const cert: CertInfo = {
-      crt: crtString,
+        crt: crtString,
-      key: key.toString(),
+        key: key.toString(),
-      csr: csr.toString(),
+        csr: csr.toString(),
-    };
+      };
-    /* Done */
+      /* Done */
-    this.logger.debug(`CSR:\n${cert.csr}`);
+      this.logger.debug(`CSR:\n${cert.csr}`);
-    this.logger.debug(`Certificate:\n${cert.crt}`);
+      this.logger.debug(`Certificate:\n${cert.crt}`);
-    this.logger.info("证书申请成功");
+      this.logger.info("证书申请成功");
-    return cert;
+      return cert;
+    } catch (e) {
+      const message = e?.message;
+      const REDUNDANT_WILDCARD_DOMAIN_ERROR = "redundant with a wildcard domain in the same request";
+      if (message != null && message.indexOf(REDUNDANT_WILDCARD_DOMAIN_ERROR) >= 0) {
+        throw new NonRetryableException(`通配符域名已经包含了普通域名，请删除其中一个（${message}）`);
+      }
+      throw e;
+    }
   }
 
   buildCommonNameByDomains(domains: string | string[]): {

packages/ui/certd-server/src/plugins/plugin-cert/plugin/cert-plugin/apply.test.ts

@@ -1,4 +1,6 @@
 import assert from "assert";
+import { utils } from "@certd/basic";
+import { NonRetryableException } from "@certd/lib-server";
 import { CertApplyPlugin } from "./apply.js";
 
 describe("CertApplyPlugin dns-persist verify plan", () => {
@@ -45,3 +47,114 @@ describe("CertApplyPlugin dns-persist verify plan", () => {
     assert.equal(plan["handfree.work"].dnsPersistVerifyPlan?.recordValue, "letsencrypt.org; accounturi=https://acme.example/acct/1; policy=wildcard");
   });
 });
+
+describe("CertApplyPlugin certificate apply retry", () => {
+  it("does not retry by default", async () => {
+    const plugin: any = new CertApplyPlugin();
+    let orderCount = 0;
+    const error = new Error("apply failed");
+    plugin.logger = { warn() {} };
+    plugin.acme = {
+      async order() {
+        orderCount++;
+        throw error;
+      },
+    };
+
+    await assert.rejects(plugin.orderWithRetry({}), error);
+    assert.equal(orderCount, 1);
+    assert.equal(plugin.certApplyRetryCount, 0);
+  });
+
+  it("retries after a 30-second cooldown and succeeds before reaching the limit", async () => {
+    const plugin: any = new CertApplyPlugin();
+    let orderCount = 0;
+    const waitTimes: number[] = [];
+    plugin.certApplyRetryCount = 2;
+    plugin.logger = { warn() {} };
+    plugin.acme = {
+      async order() {
+        orderCount++;
+        if (orderCount < 3) {
+          throw new Error(`apply failed ${orderCount}`);
+        }
+        return { crt: "certificate", key: "private-key" };
+      },
+    };
+    const originalSleep = utils.sleep;
+    utils.sleep = async (waitTime: number) => {
+      waitTimes.push(waitTime);
+    };
+
+    try {
+      const cert = await plugin.orderWithRetry({});
+
+      assert.deepEqual(cert, { crt: "certificate", key: "private-key" });
+      assert.equal(orderCount, 3);
+      assert.deepEqual(waitTimes, [30_000, 30_000]);
+    } finally {
+      utils.sleep = originalSleep;
+    }
+  });
+
+  it("throws the last error after reaching the retry limit", async () => {
+    const plugin: any = new CertApplyPlugin();
+    let orderCount = 0;
+    const error = new Error("apply failed");
+    plugin.certApplyRetryCount = 1;
+    plugin.logger = { warn() {} };
+    plugin.acme = {
+      async order() {
+        orderCount++;
+        throw error;
+      },
+    };
+    const originalSleep = utils.sleep;
+    utils.sleep = async () => {};
+
+    try {
+      await assert.rejects(plugin.orderWithRetry({}), error);
+      assert.equal(orderCount, 2);
+    } finally {
+      utils.sleep = originalSleep;
+    }
+  });
+
+  it("does not retry a cancelled apply", async () => {
+    const plugin: any = new CertApplyPlugin();
+    let orderCount = 0;
+    const error: any = new Error("cancelled");
+    error.name = "CancelError";
+    plugin.certApplyRetryCount = 2;
+    plugin.logger = { warn() {} };
+    plugin.acme = {
+      async order() {
+        orderCount++;
+        throw error;
+      },
+    };
+
+    await assert.rejects(plugin.orderWithRetry({}), error);
+    assert.equal(orderCount, 1);
+  });
+
+  it("throws a non-retryable error for wildcard and normal domain conflicts", async () => {
+    const plugin: any = new CertApplyPlugin();
+    let orderCount = 0;
+    const message = "example.com is redundant with a wildcard domain in the same request";
+    plugin.certApplyRetryCount = 2;
+    plugin.logger = { warn() {} };
+    plugin.acme = {
+      async order() {
+        orderCount++;
+        throw new NonRetryableException(`通配符域名已经包含了普通域名，请删除其中一个（${message}）`);
+      },
+    };
+    await assert.rejects(plugin.orderWithRetry({}), (error: any) => {
+      assert.equal(error instanceof NonRetryableException, true);
+      assert.equal(error.message, `通配符域名已经包含了普通域名，请删除其中一个（${message}）`);
+      return true;
+    });
+    assert.equal(orderCount, 1);
+  });
+});

packages/ui/certd-server/src/plugins/plugin-cert/plugin/cert-plugin/apply.ts

@@ -1,5 +1,6 @@
-import { CancelError, IsTaskPlugin, pluginGroups, RunStrategy, TaskInput } from "@certd/pipeline";
+import { IsTaskPlugin, pluginGroups, RunStrategy, TaskInput } from "@certd/pipeline";
 import { utils } from "@certd/basic";
+import { NonRetryableException } from "@certd/lib-server";
 
 import { AcmeAccountInfo, AcmeService, DomainsVerifyPlan, DomainVerifyPlan, PrivateKeyType, SSLProvider } from "./acme.js";
 import { createDnsProvider, DnsProviderContext, DnsVerifier, DomainVerifiers, HttpVerifier, IDnsProvider, IDomainVerifierGetter, ISubDomainsGetter } from "@certd/plugin-lib";
@@ -61,6 +62,7 @@ const preferredChainConfigs = {
 } as const;
 
 const preferredChainSupportedProviders = Object.keys(preferredChainConfigs);
+const CERT_APPLY_RETRY_DELAY_MS = 30_000;
 
 const preferredChainMergeScript = (() => {
   const configs = JSON.stringify(preferredChainConfigs);
@@ -551,6 +553,20 @@ export class CertApplyPlugin extends CertApplyBasePlugin {
   })
   waitDnsDiffuseTime = 30;
 
+  @TaskInput({
+    title: "证书申请失败重试次数",
+    value: 0,
+    component: {
+      name: "a-input-number",
+      vModel: "value",
+      min: 0,
+      step: 1,
+    },
+    maybeNeed: true,
+    helper: "证书申请失败后，等待30秒再自动重试；0表示不重试",
+  })
+  certApplyRetryCount = 0;
+
   acme!: AcmeService;
 
   eab!: EabAccess;
@@ -651,34 +667,53 @@ export class CertApplyPlugin extends CertApplyBasePlugin {
       dnsProvider = await this.createDnsProvider(dnsProviderType, access);
     }
 
-    try {
+    const cert = await this.orderWithRetry({
-      const cert = await this.acme.order({
+      email,
-        email,
+      domains,
-        domains,
+      dnsProvider,
-        dnsProvider,
+      domainsVerifyPlan,
-        domainsVerifyPlan,
+      csrInfo,
-        csrInfo,
+      privateKeyType: this.privateKeyType,
-        privateKeyType: this.privateKeyType,
+      profile: this.certProfile,
-        profile: this.certProfile,
+      preferredChain: this.preferredChain,
-        preferredChain: this.preferredChain,
+      acmeAccount,
-        acmeAccount,
+    });
-      });
 
-      const certInfo = this.formatCerts(cert);
+    const certInfo = this.formatCerts(cert);
-      return new CertReader(certInfo);
+    return new CertReader(certInfo);
-    } catch (e: any) {
+  }
-      const message: string = e?.message;
+
-      if (message != null && message.indexOf("redundant with a wildcard domain in the same request") >= 0) {
+  private async orderWithRetry(orderOptions: Parameters<AcmeService["order"]>[0]) {
-        this.logger.error(e);
+    const maxRetryCount = this.getCertApplyRetryCount();
-        throw new Error(`通配符域名已经包含了普通域名，请删除其中一个（${message}）`);
+    let retryCount = 0;
+
+    while (true) {
+      try {
+        return await this.acme.order(orderOptions);
+      } catch (e: any) {
+        if (e instanceof NonRetryableException) {
+          throw e;
+        }
+        if (e?.name === "CancelError" || retryCount >= maxRetryCount) {
+          throw e;
+        }
+
+        retryCount++;
+        this.logger.warn(`证书申请失败，等待30秒后重试（${retryCount}/${maxRetryCount}）`, e);
+        await utils.sleep(CERT_APPLY_RETRY_DELAY_MS);
       }
-      if (e.name === "CancelError") {
-        throw new CancelError(e.message);
-      }
-      throw e;
     }
   }
 
+  private getCertApplyRetryCount() {
+    const retryCount = Number(this.certApplyRetryCount);
+    if (!Number.isFinite(retryCount) || retryCount <= 0) {
+      return 0;
+    }
+    return Math.floor(retryCount);
+  }
+
+
   async createDnsProvider(dnsProviderType: string, dnsProviderAccess: any): Promise<IDnsProvider> {
     const domainParser = this.acme.options.domainParser;
     const context: DnsProviderContext = {