From 56b8c689ec2b5cff49010a8c765483dd36803e9d Mon Sep 17 00:00:00 2001 From: Steven Zhu Date: Sun, 7 Jun 2026 22:29:11 -0400 Subject: [PATCH] =?UTF-8?q?perf:=20=E6=B7=BB=E5=8A=A0AWS=20Rate=20Limit?= =?UTF-8?q?=E5=BA=94=E5=AF=B9=E6=8E=AA=E6=96=BD=20(#748)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Parse PEM chain and import certificate chain Split the PEM in certInfo.crt into a leaf certificate and intermediate chain (using a lookbehind regex), trim the blocks, and pass the chain to ImportCertificateCommand only when present. Replace console.log with this.logger.info and log the returned CertificateArn. This ensures the leaf cert is uploaded separately from its chain and avoids sending an empty CertificateChain. * Add AWS retry & CloudFront deployment wait Introduce robust retry and polling helpers to handle AWS throttling and CloudFront propagation. Added AwsClient.withRetry (exponential backoff, handles common throttling errors, default 5 attempts/base 2s) and waitForDistributionDeployed (polls until distribution Status is "Deployed", default 10min timeout/15s interval). Update deploy-to-cloudfront plugin to use withRetry for Get/UpdateDistribution and importCertificate, pass AwsClient into uploadToACM, and wait for each distribution to finish deploying before continuing to avoid PreconditionFailed errors. Improves reliability when facing rate limits and global CloudFront propagation delays; adds informative logging for retry and deployment status. --- .../src/plugins/plugin-aws/libs/aws-client.ts | 47 ++++++++++++++ .../plugins/plugin-deploy-to-cloudfront.ts | 61 +++++++++++-------- 2 files changed, 81 insertions(+), 27 deletions(-) diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts index 0ea42d074..4481b1ae2 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts @@ -188,4 +188,51 @@ export class AwsClient { throw err; } } + + /** + * Retries an AWS SDK call with exponential backoff when throttled. + * Handles: TooManyRequestsException, ThrottlingException, RequestLimitExceeded, Throttling + */ + async withRetry(call: () => Promise, maxAttempts = 5, baseDelayMs = 2000): Promise { + const throttlingCodes = new Set([ + "TooManyRequestsException", + "ThrottlingException", + "RequestLimitExceeded", + "Throttling", + ]); + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await call(); + } catch (err: any) { + const code = err?.name || err?.Code || err?.code || ""; + const isThrottle = throttlingCodes.has(code) || err?.message?.toLowerCase().includes("rate exceeded"); + if (isThrottle && attempt < maxAttempts) { + const delay = baseDelayMs * Math.pow(2, attempt - 1); // 2s, 4s, 8s, 16s … + this.logger.warn(`AWS rate limit hit (${code}), attempt ${attempt}/${maxAttempts}, retrying in ${delay}ms…`); + await utils.sleep(delay); + } else { + throw err; + } + } + } + } + + /** + * Polls a CloudFront distribution until its Status becomes "Deployed". + * CloudFront propagates changes globally and can take several minutes. + */ + async waitForDistributionDeployed(cloudFrontClient: any, distributionId: string, timeoutMs = 600_000, pollIntervalMs = 15_000): Promise { + const { GetDistributionCommand } = await import("@aws-sdk/client-cloudfront"); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const res = await this.withRetry(() => cloudFrontClient.send(new GetDistributionCommand({ Id: distributionId }))); + const status = res?.Distribution?.Status; + this.logger.info(`CloudFront distribution ${distributionId} status: ${status}`); + if (status === "Deployed") { + return; + } + await utils.sleep(pollIntervalMs); + } + throw new Error(`Timed out waiting for CloudFront distribution ${distributionId} to reach Deployed status`); + } } diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts index 8a5b731bf..7fd1afb1b 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts @@ -72,10 +72,16 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { async execute(): Promise { const access = await this.getAccess(this.accessId); + const acmClient = new AwsClient({ + access, + region: this.region, + logger: this.logger, + }); + let certId = this.cert as string; if (typeof this.cert !== "string") { //先上传 - certId = await this.uploadToACM(access, this.cert); + certId = await this.uploadToACM(acmClient, this.cert); } //部署到CloudFront @@ -90,38 +96,39 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { // update-distribution for (const distributionId of this.distributionIds) { - // get-distribution-config - const getDistributionConfigCommand = new GetDistributionConfigCommand({ - Id: distributionId, - }); + // get-distribution-config (with retry for throttling) + const configData = await acmClient.withRetry(() => + cloudFrontClient.send(new GetDistributionConfigCommand({ Id: distributionId })) + ); - const configData = await cloudFrontClient.send(getDistributionConfigCommand); + await acmClient.withRetry(() => + cloudFrontClient.send( + new UpdateDistributionCommand({ + DistributionConfig: { + ...configData.DistributionConfig, + ViewerCertificate: { + ...configData.DistributionConfig.ViewerCertificate, + CloudFrontDefaultCertificate: false, + ACMCertificateArn: certId, + }, + }, + Id: distributionId, + IfMatch: configData.ETag, + }) + ) + ); - const updateDistributionCommand = new UpdateDistributionCommand({ - DistributionConfig: { - ...configData.DistributionConfig, - ViewerCertificate: { - ...configData.DistributionConfig.ViewerCertificate, - CloudFrontDefaultCertificate: false, - ACMCertificateArn: certId, - }, - }, - Id: distributionId, - IfMatch: configData.ETag, - }); - await cloudFrontClient.send(updateDistributionCommand); - this.logger.info(`部署${distributionId}完成:`); + this.logger.info(`证书已提交到 ${distributionId}，等待全局部署完成…`); + // Wait for this distribution to fully propagate before moving to the next one. + // Updating a distribution that is still InProgress results in a PreconditionFailed error. + await acmClient.waitForDistributionDeployed(cloudFrontClient, distributionId); + this.logger.info(`部署 ${distributionId} 完成`); } this.logger.info("部署完成"); } - private async uploadToACM(access: AwsAccess, cert: CertInfo) { - const acmClient = new AwsClient({ - access, - region: this.region, - logger: this.logger, - }); - const awsCertARN = await acmClient.importCertificate(cert); + private async uploadToACM(acmClient: AwsClient, cert: CertInfo) { + const awsCertARN = await acmClient.withRetry(() => acmClient.importCertificate(cert)); this.logger.info("证书上传成功,id=", awsCertARN); return awsCertARN; }