mirror of
https://github.com/certd/certd.git
synced 2026-05-13 19:47:55 +08:00
chore: rdap-ss client 重构
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
import { http, logger } from "@certd/basic";
|
||||
import dayjs from "dayjs";
|
||||
import customParseFormat from "dayjs/plugin/customParseFormat.js";
|
||||
|
||||
import type { DomainInfo } from "./tld-client.js";
|
||||
|
||||
dayjs.extend(customParseFormat);
|
||||
|
||||
export class RdapSsClient {
|
||||
private static readonly API = "https://rdap.ss/api/query";
|
||||
private static readonly RATE_LIMITS = [
|
||||
{ windowMs: 60 * 1000, max: 30 },
|
||||
{ windowMs: 60 * 60 * 1000, max: 1200 },
|
||||
{ windowMs: 24 * 60 * 60 * 1000, max: 12000 },
|
||||
];
|
||||
private static readonly MAX_WAIT_MS = 3 * 60 * 1000;
|
||||
private static readonly DATA_RETENTION_MS = 24 * 60 * 60 * 1000;
|
||||
private static requestTimes: number[] = [];
|
||||
|
||||
async getDomainInfo(domain: string): Promise<DomainInfo> {
|
||||
await this.waitRateLimit();
|
||||
|
||||
const result = await http.request({
|
||||
url: `${RdapSsClient.API}?q=${encodeURIComponent(domain)}`,
|
||||
method: "GET",
|
||||
logRes: false,
|
||||
});
|
||||
|
||||
if (!result?.success || !result?.data) {
|
||||
throw new Error(`【${domain}】rdap.ss查询失败`);
|
||||
}
|
||||
|
||||
const data = result.data?.whoisData || result.data?.rawData || {};
|
||||
const res: DomainInfo = {};
|
||||
const expirationDate = this.parseFirstDate(data, ["Expiry Date", "Expiration Date", "Registry Expiry Date", "expires"]);
|
||||
const registrationDate = this.parseFirstDate(data, ["Domain Name Commencement Date", "Created Date", "Creation Date", "Registration Date", "Registered On"]);
|
||||
|
||||
if (expirationDate) {
|
||||
res.expirationDate = expirationDate;
|
||||
}
|
||||
if (registrationDate) {
|
||||
res.registrationDate = registrationDate;
|
||||
}
|
||||
|
||||
if (!res.expirationDate) {
|
||||
throw new Error(`【${domain}】rdap.ss查询未找到过期时间`);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
private async waitRateLimit() {
|
||||
while (true) {
|
||||
const now = Date.now();
|
||||
RdapSsClient.requestTimes = RdapSsClient.requestTimes.filter(time => now - time < RdapSsClient.DATA_RETENTION_MS);
|
||||
|
||||
const waitMs = RdapSsClient.RATE_LIMITS.reduce((maxWaitMs, limit) => {
|
||||
const times = RdapSsClient.requestTimes.filter(time => now - time < limit.windowMs);
|
||||
if (times.length < limit.max) {
|
||||
return maxWaitMs;
|
||||
}
|
||||
const earliestTime = Math.min(...times);
|
||||
return Math.max(maxWaitMs, earliestTime + limit.windowMs - now);
|
||||
}, 0);
|
||||
|
||||
if (waitMs <= 0) {
|
||||
RdapSsClient.requestTimes.push(now);
|
||||
return;
|
||||
}
|
||||
|
||||
if (waitMs > RdapSsClient.MAX_WAIT_MS) {
|
||||
throw new Error(`rdap.ss查询达到速率限制,等待时间超过3分钟`);
|
||||
}
|
||||
|
||||
logger.warn(`rdap.ss查询达到速率限制,将在${waitMs}ms后重试`);
|
||||
await this.sleep(waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
private parseFirstDate(data: Record<string, any>, keys: string[]) {
|
||||
for (const key of keys) {
|
||||
const value = data[key];
|
||||
const values = Array.isArray(value) ? value : [value];
|
||||
for (const item of values) {
|
||||
const timestamp = this.parseDateValue(item);
|
||||
if (timestamp) {
|
||||
return timestamp;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private parseDateValue(value: any) {
|
||||
if (!value || typeof value !== "string") {
|
||||
return;
|
||||
}
|
||||
|
||||
const formats = ["DD-MM-YYYY", "YYYY-MM-DD", "YYYY-MM-DD HH:mm:ss", "YYYY/MM/DD", "YYYY/MM/DD HH:mm:ss", "MMM D YYYY", "MMM D, YYYY"];
|
||||
for (const format of formats) {
|
||||
const parsed = dayjs(value, format, true);
|
||||
if (parsed.isValid()) {
|
||||
return parsed.valueOf();
|
||||
}
|
||||
}
|
||||
|
||||
const parsed = dayjs(value);
|
||||
if (parsed.isValid()) {
|
||||
return parsed.valueOf();
|
||||
}
|
||||
}
|
||||
|
||||
private sleep(ms: number) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import assert from "node:assert/strict";
|
||||
|
||||
import { http } from "@certd/basic";
|
||||
|
||||
import { RdapSsClient } from "./rdap-ss-client.js";
|
||||
import { TldClient } from "./tld-client.js";
|
||||
|
||||
describe("TldClient", () => {
|
||||
@@ -35,10 +36,10 @@ describe("TldClient", () => {
|
||||
it("queries rdap.ss and parses HK whois date fields", async () => {
|
||||
const originalRequest = http.request;
|
||||
let requestedConfig: any;
|
||||
const originalRequestTimes = (TldClient as any).rdapSsRequestTimes;
|
||||
const originalRequestTimes = (RdapSsClient as any).requestTimes;
|
||||
|
||||
try {
|
||||
(TldClient as any).rdapSsRequestTimes = [];
|
||||
(RdapSsClient as any).requestTimes = [];
|
||||
http.request = async (config: any) => {
|
||||
requestedConfig = config;
|
||||
return {
|
||||
@@ -52,7 +53,7 @@ describe("TldClient", () => {
|
||||
};
|
||||
};
|
||||
|
||||
const result = await (new TldClient() as any).getDomainExpirationByRdapSs("google.com.hk");
|
||||
const result = await new RdapSsClient().getDomainInfo("google.com.hk");
|
||||
|
||||
assert.equal(requestedConfig.url, "https://rdap.ss/api/query?q=google.com.hk");
|
||||
assert.equal(requestedConfig.method, "GET");
|
||||
@@ -60,67 +61,67 @@ describe("TldClient", () => {
|
||||
assert.equal(result.expirationDate, 1795104000000);
|
||||
} finally {
|
||||
http.request = originalRequest;
|
||||
(TldClient as any).rdapSsRequestTimes = originalRequestTimes;
|
||||
(RdapSsClient as any).requestTimes = originalRequestTimes;
|
||||
}
|
||||
});
|
||||
|
||||
it("throws when rdap.ss rate-limit wait is over 3 minutes", async () => {
|
||||
const now = Date.now();
|
||||
const originalRequestTimes = (TldClient as any).rdapSsRequestTimes;
|
||||
const originalRequestTimes = (RdapSsClient as any).requestTimes;
|
||||
|
||||
try {
|
||||
(TldClient as any).rdapSsRequestTimes = Array.from({ length: 1200 }, () => now);
|
||||
const client = new TldClient() as any;
|
||||
(RdapSsClient as any).requestTimes = Array.from({ length: 1200 }, () => now);
|
||||
const client = new RdapSsClient() as any;
|
||||
client.sleep = async () => {
|
||||
throw new Error("should not sleep when wait time is over limit");
|
||||
};
|
||||
|
||||
await assert.rejects(() => client.waitRdapSsRateLimit(), /rdap\.ss查询达到速率限制,等待时间超过3分钟/);
|
||||
await assert.rejects(() => client.waitRateLimit(), /rdap\.ss查询达到速率限制,等待时间超过3分钟/);
|
||||
} finally {
|
||||
(TldClient as any).rdapSsRequestTimes = originalRequestTimes;
|
||||
(RdapSsClient as any).requestTimes = originalRequestTimes;
|
||||
}
|
||||
});
|
||||
|
||||
it("clears rdapSsRequestTimes entries older than 24 hours", async () => {
|
||||
it("clears rdap.ss request time entries older than 24 hours", async () => {
|
||||
const now = Date.now();
|
||||
const originalRequestTimes = (TldClient as any).rdapSsRequestTimes;
|
||||
const originalRequestTimes = (RdapSsClient as any).requestTimes;
|
||||
|
||||
try {
|
||||
const recentTime = now - 1000 * 60 * 60;
|
||||
const oldTime = now - 25 * 60 * 60 * 1000;
|
||||
|
||||
(TldClient as any).rdapSsRequestTimes = [oldTime, recentTime];
|
||||
(RdapSsClient as any).requestTimes = [oldTime, recentTime];
|
||||
|
||||
const client = new TldClient() as any;
|
||||
await client.waitRdapSsRateLimit();
|
||||
const client = new RdapSsClient() as any;
|
||||
await client.waitRateLimit();
|
||||
|
||||
const times = (TldClient as any).rdapSsRequestTimes;
|
||||
const times = (RdapSsClient as any).requestTimes;
|
||||
assert.equal(times.length, 2);
|
||||
assert.ok(times.every((t: number) => now - t < 24 * 60 * 60 * 1000));
|
||||
} finally {
|
||||
(TldClient as any).rdapSsRequestTimes = originalRequestTimes;
|
||||
(RdapSsClient as any).requestTimes = originalRequestTimes;
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps rdapSsRequestTimes entries within 24 hours and removes those exactly at boundary", async () => {
|
||||
it("keeps rdap.ss request time entries within 24 hours and removes those exactly at boundary", async () => {
|
||||
const now = Date.now();
|
||||
const originalRequestTimes = (TldClient as any).rdapSsRequestTimes;
|
||||
const originalRequestTimes = (RdapSsClient as any).requestTimes;
|
||||
|
||||
try {
|
||||
const boundaryTime = now - 24 * 60 * 60 * 1000;
|
||||
const withinTime = now - 23 * 60 * 60 * 1000;
|
||||
|
||||
(TldClient as any).rdapSsRequestTimes = [boundaryTime, withinTime];
|
||||
(RdapSsClient as any).requestTimes = [boundaryTime, withinTime];
|
||||
|
||||
const client = new TldClient() as any;
|
||||
await client.waitRdapSsRateLimit();
|
||||
const client = new RdapSsClient() as any;
|
||||
await client.waitRateLimit();
|
||||
|
||||
const times = (TldClient as any).rdapSsRequestTimes as number[];
|
||||
const times = (RdapSsClient as any).requestTimes as number[];
|
||||
assert.equal(times.length, 2);
|
||||
assert.ok(times.every((t: number) => now - t < 24 * 60 * 60 * 1000));
|
||||
assert.ok(times.includes(withinTime));
|
||||
} finally {
|
||||
(TldClient as any).rdapSsRequestTimes = originalRequestTimes;
|
||||
(RdapSsClient as any).requestTimes = originalRequestTimes;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,6 +3,8 @@ import { parseDomainByPsl } from "@certd/plugin-lib";
|
||||
import dayjs from "dayjs";
|
||||
import customParseFormat from "dayjs/plugin/customParseFormat.js";
|
||||
|
||||
import { RdapSsClient } from "./rdap-ss-client.js";
|
||||
|
||||
dayjs.extend(customParseFormat);
|
||||
|
||||
export interface DomainInfo {
|
||||
@@ -11,16 +13,7 @@ export interface DomainInfo {
|
||||
}
|
||||
|
||||
export class TldClient {
|
||||
private static readonly RDAP_SS_API = "https://rdap.ss/api/query";
|
||||
private static readonly RDAP_SS_RATE_LIMITS = [
|
||||
{ windowMs: 60 * 1000, max: 30 },
|
||||
{ windowMs: 60 * 60 * 1000, max: 1200 },
|
||||
{ windowMs: 24 * 60 * 60 * 1000, max: 12000 },
|
||||
];
|
||||
private static readonly RDAP_SS_MAX_WAIT_MS = 3 * 60 * 1000;
|
||||
private static readonly RDAP_SS_DATA_RETENTION_MS = 24 * 60 * 60 * 1000;
|
||||
private static rdapSsRequestTimes: number[] = [];
|
||||
|
||||
private rdapSsClient = new RdapSsClient();
|
||||
private rdapMap: Record<string, string> = {};
|
||||
private isInitialized = false;
|
||||
|
||||
@@ -93,35 +86,7 @@ export class TldClient {
|
||||
}
|
||||
|
||||
private async getDomainExpirationByRdapSs(domain: string): Promise<DomainInfo> {
|
||||
await this.waitRdapSsRateLimit();
|
||||
|
||||
const result = await http.request({
|
||||
url: `${TldClient.RDAP_SS_API}?q=${encodeURIComponent(domain)}`,
|
||||
method: "GET",
|
||||
logRes: false,
|
||||
});
|
||||
|
||||
if (!result?.success || !result?.data) {
|
||||
throw new Error(`【${domain}】rdap.ss查询失败`);
|
||||
}
|
||||
|
||||
const data = result.data?.whoisData || result.data?.rawData || {};
|
||||
const res: DomainInfo = {};
|
||||
const expirationDate = this.parseFirstDate(data, ["Expiry Date", "Expiration Date", "Registry Expiry Date", "expires"]);
|
||||
const registrationDate = this.parseFirstDate(data, ["Domain Name Commencement Date", "Created Date", "Creation Date", "Registration Date", "Registered On"]);
|
||||
|
||||
if (expirationDate) {
|
||||
res.expirationDate = expirationDate;
|
||||
}
|
||||
if (registrationDate) {
|
||||
res.registrationDate = registrationDate;
|
||||
}
|
||||
|
||||
if (!res.expirationDate) {
|
||||
throw new Error(`【${domain}】rdap.ss查询未找到过期时间`);
|
||||
}
|
||||
|
||||
return res;
|
||||
return await this.rdapSsClient.getDomainInfo(domain);
|
||||
}
|
||||
|
||||
private async getDomainExpirationByWhoiser(domain: string): Promise<DomainInfo> {
|
||||
@@ -175,70 +140,6 @@ export class TldClient {
|
||||
return res;
|
||||
}
|
||||
|
||||
private async waitRdapSsRateLimit() {
|
||||
while (true) {
|
||||
const now = Date.now();
|
||||
TldClient.rdapSsRequestTimes = TldClient.rdapSsRequestTimes.filter(time => now - time < TldClient.RDAP_SS_DATA_RETENTION_MS);
|
||||
|
||||
const waitMs = TldClient.RDAP_SS_RATE_LIMITS.reduce((maxWaitMs, limit) => {
|
||||
const times = TldClient.rdapSsRequestTimes.filter(time => now - time < limit.windowMs);
|
||||
if (times.length < limit.max) {
|
||||
return maxWaitMs;
|
||||
}
|
||||
const earliestTime = Math.min(...times);
|
||||
return Math.max(maxWaitMs, earliestTime + limit.windowMs - now);
|
||||
}, 0);
|
||||
|
||||
if (waitMs <= 0) {
|
||||
TldClient.rdapSsRequestTimes.push(now);
|
||||
return;
|
||||
}
|
||||
|
||||
if (waitMs > TldClient.RDAP_SS_MAX_WAIT_MS) {
|
||||
throw new Error(`rdap.ss查询达到速率限制,等待时间超过3分钟`);
|
||||
}
|
||||
|
||||
logger.warn(`rdap.ss查询达到速率限制,将在${waitMs}ms后重试`);
|
||||
await this.sleep(waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
private parseFirstDate(data: Record<string, any>, keys: string[]) {
|
||||
for (const key of keys) {
|
||||
const value = data[key];
|
||||
const values = Array.isArray(value) ? value : [value];
|
||||
for (const item of values) {
|
||||
const timestamp = this.parseDateValue(item);
|
||||
if (timestamp) {
|
||||
return timestamp;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private parseDateValue(value: any) {
|
||||
if (!value || typeof value !== "string") {
|
||||
return;
|
||||
}
|
||||
|
||||
const formats = ["DD-MM-YYYY", "YYYY-MM-DD", "YYYY-MM-DD HH:mm:ss", "YYYY/MM/DD", "YYYY/MM/DD HH:mm:ss", "MMM D YYYY", "MMM D, YYYY"];
|
||||
for (const format of formats) {
|
||||
const parsed = dayjs(value, format, true);
|
||||
if (parsed.isValid()) {
|
||||
return parsed.valueOf();
|
||||
}
|
||||
}
|
||||
|
||||
const parsed = dayjs(value);
|
||||
if (parsed.isValid()) {
|
||||
return parsed.valueOf();
|
||||
}
|
||||
}
|
||||
|
||||
private sleep(ms: number) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
private getErrorMessage(error: unknown) {
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
|
||||
Reference in New Issue
Block a user