certd/packages/core/basic/src/utils/util.request.ts

import axios, { AxiosHeaders, AxiosRequestConfig } from "axios";
import { ILogger, logger } from "./util.log.js";
import { HttpProxyAgent } from "http-proxy-agent";
import { HttpsProxyAgent } from "https-proxy-agent";
import nodeHttp from "http";
import * as https from "node:https";
import { merge } from "lodash-es";
import { safePromise } from "./util.promise.js";
import fs from "fs";
import sleep from "./util.sleep.js";
const errorMap: Record<string, string> = {
  "ssl3_get_record:wrong version number": "http协议错误，服务端要求http协议，请检查是否使用了https请求",
  "getaddrinfo EAI_AGAIN": "无法解析域名，请检查网络连接或dns配置，更换docker-compose.yaml中dns配置",
  "self-signed certificate": "目标站点为自签名证书，请勾选忽略证书校验",
};

export class HttpError extends Error {
  status?: number;
  statusText?: string;
  code?: string;
  request?: { baseURL: string; url: string; method: string; params?: any; data?: any };
  response?: { data: any; headers: AxiosHeaders };
  cause?: any;
  constructor(error: any) {
    if (!error) {
      return;
    }

    let message = error?.message || error?.response?.statusText || error?.code;
    if (message && typeof message === "string" && message.indexOf) {
      for (const key in errorMap) {
        if (message.indexOf(key) > -1) {
          message = `${message}(${errorMap[key]})`;
          break;
        }
      }
    }
    if (!message) {
      message = error.message;
    }
    if (error.errors && error.errors.length > 0) {
      message += " \n" + error.errors.map((item: any) => item.message).join("\n ");
    }
    super(message);

    this.name = error.name;
    this.code = error.code;

    this.status = error.response?.status;
    this.statusText = error.response?.statusText || error.code;

    this.request = {
      baseURL: error.config?.baseURL,
      url: error.config?.url,
      method: error.config?.method,
      params: error.config?.params,
      data: error.config?.data,
    };
    let url = error.config?.url;
    if (error.config?.baseURL) {
      url = (error.config?.baseURL || "") + url;
    }
    if (url) {
      this.message = `${this.message} 【${url}】`;
    }

    this.response = {
      data: error.response?.data,
      headers: error.response?.headers,
    };

    const { stack, cause } = error;
    this.cause = cause;
    this.stack = stack;
    delete error.response;
    delete error.config;
    delete error.request;
    // logger.error(error);
  }
}

export const HttpCommonError = HttpError;

let defaultAgents = createAgent();
const directAgents = createAgent();
let defaultProxyOptions: GlobalProxyOptions = {};
let defaultHeaders: Record<string, string> = {};

export type GlobalProxyOptions = {
  httpProxy?: string;
  httpsProxy?: string;
  noProxy?: string;
};

export function setGlobalProxy(opts: GlobalProxyOptions) {
  logger.info("setGlobalProxy:", opts);
  defaultProxyOptions = { ...opts };
  defaultAgents = createAgent({
    httpProxy: opts.httpProxy,
    httpsProxy: opts.httpsProxy,
  });
  setProxyEnvironment(opts);
}

export function getGlobalAgents() {
  return defaultAgents;
}

export function setGlobalHeaders(headers: Record<string, string> = {}) {
  logger.info("setGlobalHeaders:", Object.keys(headers));
  defaultHeaders = { ...headers };
}

export function getGlobalHeaders() {
  return defaultHeaders;
}

/**
 * @description 创建请求实例
 */
export function createAxiosService({ logger }: { logger: ILogger }) {
  // 创建一个 axios 实例
  const service = axios.create();

  // 请求拦截
  service.interceptors.request.use(
    (config: any) => {
      if (config.logParams == null) {
        config.logParams = false;
      }
      if (config.logRes == null) {
        config.logRes = false;
      }
      if (config.logData == null) {
        config.logData = false;
      }
      if (config.logReq == null) {
        config.logReq = true;
      }

      if (config.logReq !== false) {
        logger.info(`http request:${config.url}，method:${config.method}`);
      }
      if (config.logParams !== false && config.params) {
        logger.info(`params:${JSON.stringify(config.params)}`);
      }
      if (config.logData !== false && config.data) {
        logger.info(`data:${JSON.stringify(config.data)}`);
      }
      if (config.timeout == null) {
        config.timeout = 15000;
      }
      const bypassProxy = shouldBypassProxy(config, defaultProxyOptions.noProxy);
      const useCustomProxy = !!config.httpProxy && !bypassProxy;
      let agents = bypassProxy ? directAgents : defaultAgents;
      if (bypassProxy) {
        logger.info("命中no_proxy配置，跳过代理:", config.url);
      }
      if (config.skipSslVerify || useCustomProxy) {
        const agentOptions: any = {};
        if (config.skipSslVerify) {
          logger.info("忽略接口请求的SSL校验");
          agentOptions.rejectUnauthorized = false;
        }
        if (useCustomProxy) {
          logger.info("使用自定义http代理:", config.httpProxy);
          agentOptions.httpProxy = config.httpProxy;
          agentOptions.httpsProxy = config.httpProxy;
        }

        agents = createAgent(agentOptions);
      }

      delete config.skipSslVerify;
      config.httpsAgent = agents.httpsAgent;
      config.httpAgent = agents.httpAgent;

      if (Object.keys(defaultHeaders).length > 0) {
        const headers = AxiosHeaders.from(defaultHeaders);
        headers.set(config.headers || {});
        config.headers = headers;
      }

      // const agent = new https.Agent({
      //   rejectUnauthorized: false  // 允许自签名证书
      // });
      // config.httpsAgent = agent;
      config.proxy = false; //必须 否则还会走一层代理，

      config.retry = merge(
        {
          status: [421, 524],
          count: 0,
          max: 3,
          delay: 2000,
          includes: ["[524]"],
        },
        config.retry
      );
      return config;
    },
    (error: Error) => {
      // 发送失败
      logger.error("接口请求失败：", error);
      return Promise.reject(error);
    }
  );
  // 响应拦截
  service.interceptors.response.use(
    (response: any) => {
      if (response?.config?.logRes !== false) {
        let resData = response?.data;
        try {
          resData = JSON.stringify(response?.data);
        } catch (e) {}

        logger.info(`http response : status=${response?.status},data=${resData}`);
      } else {
        logger.info("http response status:", response?.status);
      }

      if (response?.config?.returnOriginRes) {
        return response;
      }
      return response.data;
    },
    async (error: any) => {
      const status = error.response?.status;
      let message = "";
      switch (status) {
        case 400:
          message = "请求错误";
          break;
        case 401:
          message = "认证/登录失败";
          break;
        case 403:
          message = "拒绝访问";
          break;
        case 404:
          message = `请求地址出错`;
          break;
        case 408:
          message = "请求超时";
          break;
        case 500:
          message = "服务器内部错误";
          break;
        case 501:
          message = "服务未实现";
          break;
        case 502:
          message = "网关错误";
          break;
        case 503:
          message = "服务不可用";
          break;
        case 504:
          message = "网关超时";
          break;
        case 505:
          message = "HTTP版本不受支持";
          break;
        case 302:
          //重定向
          return Promise.resolve(error.response);
        case 421:
          message = "源站请求超时";
          break;
        default:
          break;
      }
      if (status) {
        message += ` [${status}] `;
      }

      const errorCode = error.code;
      let errorMessage = "";
      if (errorCode === "ECONNABORTED") {
        errorMessage = "请求连接终止";
      } else if (errorCode === "ETIMEDOUT") {
        errorMessage = "请求连接超时";
      } else if (errorCode === "ECONNRESET") {
        errorMessage = "请求连接被重置";
      } else if (errorCode === "ECONNREFUSED") {
        errorMessage = "请求连接被服务端拒绝";
      } else if (errorCode === "ENOTFOUND") {
        errorMessage = "请求地址不存在";
      }
      if (errorCode) {
        errorMessage += ` [${errorCode}] `;
      }
      if (message) {
        errorMessage += `,${message}`;
      }
      if (error.message) {
        errorMessage += `(${error.message})`;
      }
      error.message = errorMessage;
      logger.error(`请求出错：${errorMessage} status:${status},statusText:${error.response?.statusText || error.code},url:${error.config?.url},method:${error.config?.method}。`);
      logger.error("返回数据:", JSON.stringify(error.response?.data));
      if (error.response?.data) {
        const message = error.response?.data?.message || error.response?.data?.msg || error.response?.data?.error;
        if (typeof message === "string") {
          error.message = message;
        }
      }
      if (error instanceof AggregateError) {
        logger.error("AggregateError", error);
      }

      const originalRequest = error.config || {};
      // logger.info(`config`, originalRequest);
      const retry = originalRequest.retry || {};

      const isRetryStatus = retry.status && retry.status.includes(status);
      let isRetryMessage = false;
      if (retry.includes) {
        for (const item of retry.includes) {
          if (error.message?.includes(item)) {
            isRetryMessage = true;
            break;
          }
        }
      }

      if (isRetryStatus || isRetryMessage) {
        if (retry.max > 0 && retry.count < retry.max) {
          // 重试次数增加
          retry.count++;
          const delay = retry.delay * retry.count;
          logger.error(`status=${status}，重试次数${retry.count},将在${delay}ms后重试，请求地址：${originalRequest.url}`);
          await sleep(delay);
          return service.request(originalRequest); // 重试请求
        }
        logger.error(`重试超过最大次数${retry.max}，请求失败:${originalRequest.url}`);
      }

      const err = new HttpError(error);
      if (error.response?.config?.logParams === false) {
        delete err.request?.params;
        delete err.request?.data;
      }
      return Promise.reject(err);
    }
  );
  return service;
}

export const http = createAxiosService({ logger }) as HttpClient;
export type HttpClientResponse<R> = any;
export type HttpRequestConfig<D = any> = {
  skipSslVerify?: boolean;
  skipCheckRes?: boolean;
  logReq?: boolean;
  logParams?: boolean;
  logRes?: boolean;
  logData?: boolean;
  httpProxy?: string;
  returnOriginRes?: boolean;
} & AxiosRequestConfig<D>;
export type HttpClient = {
  request<D = any, R = any>(config: HttpRequestConfig<D>): Promise<HttpClientResponse<R>>;
};

// const http_proxy_backup = process.env.HTTP_PROXY || process.env.http_proxy;
// const https_proxy_backup = process.env.HTTPS_PROXY || process.env.https_proxy;

export type CreateAgentOptions = {
  httpProxy?: string;
  httpsProxy?: string;
} & nodeHttp.AgentOptions;
export function createAgent(opts: CreateAgentOptions = {}) {
  const { httpProxy, httpsProxy, ...agentOptions } = merge(
    {
      autoSelectFamily: true,
      autoSelectFamilyAttemptTimeout: 1000,
      connectTimeout: 5000, // 连接建立超时
    },
    opts
  );

  let httpAgent, httpsAgent;
  if (httpProxy) {
    logger.info("use httpProxy:", httpProxy);
    httpAgent = new HttpProxyAgent(httpProxy, agentOptions as any);
    merge(httpAgent.options, agentOptions);
  } else {
    httpAgent = new nodeHttp.Agent(agentOptions);
  }
  if (httpsProxy) {
    logger.info("use httpsProxy:", httpsProxy);
    httpsAgent = new HttpsProxyAgent(httpsProxy, agentOptions as any);
    merge(httpsAgent.options, agentOptions);
  } else {
    httpsAgent = new https.Agent(agentOptions);
  }
  return {
    httpAgent,
    httpsAgent,
  };
}

function setProxyEnvironment(opts: GlobalProxyOptions = {}) {
  setEnvValue("HTTP_PROXY", opts.httpProxy);
  setEnvValue("http_proxy", opts.httpProxy);
  setEnvValue("HTTPS_PROXY", opts.httpsProxy);
  setEnvValue("https_proxy", opts.httpsProxy);
  const noProxy = normalizeNoProxyText(opts.noProxy);
  setEnvValue("NO_PROXY", noProxy);
  setEnvValue("no_proxy", noProxy);
}

function setEnvValue(key: string, value?: string) {
  process.env[key] = value || "";
}

function shouldBypassProxy(config: AxiosRequestConfig, noProxy?: string) {
  if (!noProxy) {
    return false;
  }
  const target = getRequestTarget(config);
  if (!target) {
    return false;
  }
  return splitNoProxyRules(noProxy).some(item => isNoProxyMatched(item, target));
}

function getRequestTarget(config: AxiosRequestConfig) {
  try {
    const baseURL = config.baseURL || undefined;
    const url = new URL(config.url || "", baseURL);
    return {
      hostname: normalizeHost(url.hostname),
      port: url.port,
    };
  } catch (e) {
    return null;
  }
}

export function isNoProxyMatched(rule: string, target: { hostname: string; port: string }) {
  if (rule === "*") {
    return true;
  }

  const normalizedRule = normalizeNoProxyRule(rule);
  if (!normalizedRule.host) {
    return false;
  }
  if (normalizedRule.port && normalizedRule.port !== target.port) {
    return false;
  }

  const host = normalizeHost(target.hostname);
  if (normalizedRule.host.includes("*")) {
    return wildcardHostMatched(normalizedRule.host, host);
  }
  if (normalizedRule.host.startsWith("*.")) {
    const suffix = normalizedRule.host.substring(1);
    return host.endsWith(suffix);
  }
  if (normalizedRule.host.startsWith(".")) {
    return host === normalizedRule.host.substring(1) || host.endsWith(normalizedRule.host);
  }
  return host === normalizedRule.host || host.endsWith(`.${normalizedRule.host}`);
}

function normalizeNoProxyRule(rule: string) {
  let value = rule.trim().toLowerCase();
  if (value.includes("://")) {
    try {
      const url = new URL(value);
      return {
        host: normalizeHost(url.hostname),
        port: url.port,
      };
    } catch (e) {
      return {
        host: "",
        port: "",
      };
    }
  }

  let port = "";
  if (value.startsWith("[")) {
    const closeIndex = value.indexOf("]");
    const host = value.substring(1, closeIndex);
    const rest = value.substring(closeIndex + 1);
    if (rest.startsWith(":")) {
      port = rest.substring(1);
    }
    return {
      host: normalizeHost(host),
      port,
    };
  }

  const colonCount = (value.match(/:/g) || []).length;
  const portIndex = value.lastIndexOf(":");
  if (colonCount === 1 && portIndex > -1) {
    port = value.substring(portIndex + 1);
    value = value.substring(0, portIndex);
  }
  return {
    host: normalizeHost(value),
    port,
  };
}

function normalizeHost(host: string) {
  let value = host.trim().toLowerCase();
  if (value.startsWith("[") && value.endsWith("]")) {
    value = value.substring(1, value.length - 1);
  }
  return value;
}

function wildcardHostMatched(rule: string, host: string) {
  const pattern = rule.split("*").map(escapeRegExp).join(".*");
  return new RegExp(`^${pattern}$`).test(host);
}

function escapeRegExp(value: string) {
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

function normalizeNoProxyText(noProxy?: string) {
  return splitNoProxyRules(noProxy).join(",");
}

function splitNoProxyRules(noProxy?: string) {
  if (!noProxy) {
    return [];
  }
  return noProxy
    .split(/[,\s]+/)
    .map(item => item.trim())
    .filter(Boolean);
}

export async function download(req: { http: HttpClient; config: HttpRequestConfig; savePath: string; logger: ILogger }) {
  const { http, config, savePath, logger } = req;
  return safePromise((resolve, reject) => {
    http
      .request({
        logRes: false,
        responseType: "stream",
        ...config,
      })
      .then(res => {
        const writer = fs.createWriteStream(savePath);
        res.pipe(writer);
        writer.on("close", () => {
          logger.info("文件下载成功");
          resolve(true);
        });
        //error
        writer.on("error", err => {
          logger.error("下载失败", err);
          reject(err);
        });
        //进度条打印
        const totalLength = res.headers["content-length"];
        let currentLength = 0;
        // 每5%打印一次
        const step = (totalLength / 100) * 5;
        res.on("data", (chunk: any) => {
          currentLength += chunk.length;
          if (currentLength % step < chunk.length) {
            const percent = ((currentLength / totalLength) * 100).toFixed(2);
            logger.info(`下载进度：${percent}%`);
          }
        });
      })
      .catch(err => {
        logger.info("下载失败", err);
        reject(err);
      });
  });
}

export function getCookie(response: any, name: string) {
  const cookies = response.headers["set-cookie"];
  //根据name 返回对应的cookie
  const found = cookies.find((cookie: any) => cookie.includes(name));
  if (!found) {
    return null;
  }
  const cookie = found.split(";")[0];
  return cookie.substring(cookie.indexOf("=") + 1);
}