diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts index 0ea42d074..4481b1ae2 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts @@ -188,4 +188,51 @@ export class AwsClient { throw err; } } + + /** + * Retries an AWS SDK call with exponential backoff when throttled. + * Handles: TooManyRequestsException, ThrottlingException, RequestLimitExceeded, Throttling + */ + async withRetry(call: () => Promise, maxAttempts = 5, baseDelayMs = 2000): Promise { + const throttlingCodes = new Set([ + "TooManyRequestsException", + "ThrottlingException", + "RequestLimitExceeded", + "Throttling", + ]); + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await call(); + } catch (err: any) { + const code = err?.name || err?.Code || err?.code || ""; + const isThrottle = throttlingCodes.has(code) || err?.message?.toLowerCase().includes("rate exceeded"); + if (isThrottle && attempt < maxAttempts) { + const delay = baseDelayMs * Math.pow(2, attempt - 1); // 2s, 4s, 8s, 16s … + this.logger.warn(`AWS rate limit hit (${code}), attempt ${attempt}/${maxAttempts}, retrying in ${delay}ms…`); + await utils.sleep(delay); + } else { + throw err; + } + } + } + } + + /** + * Polls a CloudFront distribution until its Status becomes "Deployed". + * CloudFront propagates changes globally and can take several minutes. + */ + async waitForDistributionDeployed(cloudFrontClient: any, distributionId: string, timeoutMs = 600_000, pollIntervalMs = 15_000): Promise { + const { GetDistributionCommand } = await import("@aws-sdk/client-cloudfront"); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const res = await this.withRetry(() => cloudFrontClient.send(new GetDistributionCommand({ Id: distributionId }))); + const status = res?.Distribution?.Status; + this.logger.info(`CloudFront distribution ${distributionId} status: ${status}`); + if (status === "Deployed") { + return; + } + await utils.sleep(pollIntervalMs); + } + throw new Error(`Timed out waiting for CloudFront distribution ${distributionId} to reach Deployed status`); + } } diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts index 8a5b731bf..7fd1afb1b 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts @@ -72,10 +72,16 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { async execute(): Promise { const access = await this.getAccess(this.accessId); + const acmClient = new AwsClient({ + access, + region: this.region, + logger: this.logger, + }); + let certId = this.cert as string; if (typeof this.cert !== "string") { //先上传 - certId = await this.uploadToACM(access, this.cert); + certId = await this.uploadToACM(acmClient, this.cert); } //部署到CloudFront @@ -90,38 +96,39 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { // update-distribution for (const distributionId of this.distributionIds) { - // get-distribution-config - const getDistributionConfigCommand = new GetDistributionConfigCommand({ - Id: distributionId, - }); + // get-distribution-config (with retry for throttling) + const configData = await acmClient.withRetry(() => + cloudFrontClient.send(new GetDistributionConfigCommand({ Id: distributionId })) + ); - const configData = await cloudFrontClient.send(getDistributionConfigCommand); + await acmClient.withRetry(() => + cloudFrontClient.send( + new UpdateDistributionCommand({ + DistributionConfig: { + ...configData.DistributionConfig, + ViewerCertificate: { + ...configData.DistributionConfig.ViewerCertificate, + CloudFrontDefaultCertificate: false, + ACMCertificateArn: certId, + }, + }, + Id: distributionId, + IfMatch: configData.ETag, + }) + ) + ); - const updateDistributionCommand = new UpdateDistributionCommand({ - DistributionConfig: { - ...configData.DistributionConfig, - ViewerCertificate: { - ...configData.DistributionConfig.ViewerCertificate, - CloudFrontDefaultCertificate: false, - ACMCertificateArn: certId, - }, - }, - Id: distributionId, - IfMatch: configData.ETag, - }); - await cloudFrontClient.send(updateDistributionCommand); - this.logger.info(`部署${distributionId}完成:`); + this.logger.info(`证书已提交到 ${distributionId},等待全局部署完成…`); + // Wait for this distribution to fully propagate before moving to the next one. + // Updating a distribution that is still InProgress results in a PreconditionFailed error. + await acmClient.waitForDistributionDeployed(cloudFrontClient, distributionId); + this.logger.info(`部署 ${distributionId} 完成`); } this.logger.info("部署完成"); } - private async uploadToACM(access: AwsAccess, cert: CertInfo) { - const acmClient = new AwsClient({ - access, - region: this.region, - logger: this.logger, - }); - const awsCertARN = await acmClient.importCertificate(cert); + private async uploadToACM(acmClient: AwsClient, cert: CertInfo) { + const awsCertARN = await acmClient.withRetry(() => acmClient.importCertificate(cert)); this.logger.info("证书上传成功,id=", awsCertARN); return awsCertARN; }