chore: pipeline utils 转移到basic

packages/core/basic/src/index.ts

@@ -0,0 +1 @@
+export * from './utils/index.js'

packages/core/basic/src/utils/index.ts

@@ -0,0 +1,33 @@
+import sleep from "./util.sleep.js";
+import { http } from "./util.request.js";
+export * from "./util.request.js";
+export * from "./util.log.js";
+export * from "./util.file.js";
+export * from "./util.sp.js";
+export * from "./util.promise.js";
+export * from "./util.hash.js";
+export * from "./util.merge.js";
+export * from "./util.cache.js";
+import { nanoid } from "nanoid";
+import { mergeUtils } from "./util.merge.js";
+import { sp } from "./util.sp.js";
+import { hashUtils } from "./util.hash.js";
+import { promises } from "./util.promise.js";
+import { fileUtils } from "./util.file.js";
+import _ from "lodash-es";
+import { cache } from "./util.cache.js";
+import dayjs from 'dayjs';
+
+export const utils = {
+  sleep,
+  http,
+  sp,
+  hash: hashUtils,
+  promises,
+  file: fileUtils,
+  _,
+  mergeUtils,
+  cache,
+  nanoid,
+  dayjs
+};

packages/core/basic/src/utils/util.cache.ts

@@ -0,0 +1,8 @@
+// LRUCache
+
+import { LRUCache } from "lru-cache";
+
+export const cache = new LRUCache<string, any>({
+  max: 1000,
+  ttl: 1000 * 60 * 10,
+});

packages/core/basic/src/utils/util.file.ts

@@ -0,0 +1,16 @@
+import fs from "fs";
+function getFileRootDir(rootDir?: string) {
+  if (rootDir == null) {
+    const userHome = process.env.HOME || process.env.USERPROFILE;
+    rootDir = userHome + "/.certd/storage/";
+  }
+
+  if (!fs.existsSync(rootDir)) {
+    fs.mkdirSync(rootDir, { recursive: true });
+  }
+  return rootDir;
+}
+
+export const fileUtils = {
+  getFileRootDir,
+};

packages/core/basic/src/utils/util.hash.ts

@@ -0,0 +1,9 @@
+import crypto from "crypto";
+
+function md5(data: string) {
+  return crypto.createHash("md5").update(data).digest("hex");
+}
+
+export const hashUtils = {
+  md5,
+};

packages/core/basic/src/utils/util.log.ts

@@ -0,0 +1,35 @@
+import log4js, { LoggingEvent, Logger } from "log4js";
+
+const OutputAppender = {
+  configure: (config: any, layouts: any, findAppender: any, levels: any) => {
+    let layout = layouts.basicLayout;
+    if (config.layout) {
+      layout = layouts.layout(config.layout.type, config.layout);
+    }
+    function customAppender(layout: any, timezoneOffset: any) {
+      return (loggingEvent: LoggingEvent) => {
+        if (loggingEvent.context.outputHandler?.write) {
+          const text = `${layout(loggingEvent, timezoneOffset)}\n`;
+          loggingEvent.context.outputHandler.write(text);
+        }
+      };
+    }
+    return customAppender(layout, config.timezoneOffset);
+  },
+};
+
+// @ts-ignore
+log4js.configure({
+  appenders: { std: { type: "stdout" }, output: { type: OutputAppender } },
+  categories: { default: { appenders: ["std"], level: "info" }, pipeline: { appenders: ["std", "output"], level: "info" } },
+});
+export const logger = log4js.getLogger("default");
+
+export function buildLogger(write: (text: string) => void) {
+  const logger = log4js.getLogger("pipeline");
+  logger.addContext("outputHandler", {
+    write,
+  });
+  return logger;
+}
+export type ILogger = Logger;

packages/core/basic/src/utils/util.merge.ts

@@ -0,0 +1,64 @@
+import _ from "lodash-es";
+function isUnMergeable(srcValue: any) {
+  return srcValue != null && srcValue instanceof UnMergeable;
+}
+function isUnCloneable(value: any) {
+  return isUnMergeable(value) && !value.cloneable;
+}
+function merge(target: any, ...sources: any) {
+  /**
+   * 如果目标为不可合并对象，比如array、unMergeable、ref,则直接覆盖不合并
+   * @param objValue 被合并对象
+   * @param srcValue 来源对象
+   */
+  function customizer(objValue: any, srcValue: any) {
+    if (srcValue == null) {
+      return;
+    }
+    // 如果被合并对象为数组，则直接被覆盖对象覆盖，只要覆盖对象不为空
+    if (_.isArray(objValue)) {
+      //原对象如果是数组
+      return srcValue; //来源对象
+    }
+
+    if (isUnMergeable(srcValue)) {
+      return srcValue;
+    }
+  }
+
+  let found: any = null;
+  for (const item of sources) {
+    if (isUnMergeable(item)) {
+      found = item;
+    }
+  }
+  if (found) {
+    return found;
+  }
+  return _.mergeWith(target, ...sources, customizer);
+}
+
+function cloneDeep(target: any) {
+  if (isUnCloneable(target)) {
+    return target;
+  }
+  function customizer(value: any) {
+    if (isUnCloneable(value)) {
+      return value;
+    }
+  }
+
+  return _.cloneDeepWith(target, customizer);
+}
+export class UnMergeable {
+  cloneable = false;
+
+  setCloneable(cloneable: any) {
+    this.cloneable = cloneable;
+  }
+}
+
+export const mergeUtils = {
+  merge,
+  cloneDeep,
+};

packages/core/basic/src/utils/util.promise.ts

@@ -0,0 +1,50 @@
+import { logger } from "./util.log.js";
+
+export function TimeoutPromise(callback: () => Promise<void>, ms = 30 * 1000) {
+  let timeout: any;
+  return Promise.race([
+    callback(),
+    new Promise((resolve, reject) => {
+      timeout = setTimeout(() => {
+        reject(new Error(`Task timeout in ${ms} ms`));
+      }, ms);
+    }),
+  ]).finally(() => {
+    clearTimeout(timeout);
+  });
+}
+
+export function safePromise<T>(callback: (resolve: (ret: T) => void, reject: (ret: any) => void) => void): Promise<T> {
+  return new Promise((resolve, reject) => {
+    try {
+      callback(resolve, reject);
+    } catch (e) {
+      logger.error(e);
+      reject(e);
+    }
+  });
+}
+
+export function promisify(func: any) {
+  return function (...args: any) {
+    return new Promise((resolve, reject) => {
+      try {
+        func(...args, (err: any, data: any) => {
+          if (err) {
+            reject(err);
+          } else {
+            resolve(data);
+          }
+        });
+      } catch (e) {
+        reject(e);
+      }
+    });
+  };
+}
+
+export const promises = {
+  TimeoutPromise,
+  safePromise,
+  promisify,
+};

packages/core/basic/src/utils/util.request.ts

@@ -0,0 +1,174 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import { logger } from './util.log.js';
+import { Logger } from 'log4js';
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import nodeHttp from 'http';
+import * as https from 'node:https';
+export class HttpError extends Error {
+  status?: number;
+  statusText?: string;
+  code?: string;
+  request?: { baseURL: string; url: string; method: string; params?: any; data?: any };
+  response?: { data: any };
+  cause?: any;
+  constructor(error: any) {
+    if (!error) {
+      return;
+    }
+    super(error.message);
+
+    if (error?.message?.indexOf('ssl3_get_record:wrong version number') >= 0) {
+      this.message = 'http协议错误，服务端要求http协议，请检查是否使用了https请求';
+    }
+
+    this.name = error.name;
+    this.code = error.code;
+    this.cause = error.cause;
+
+    this.status = error.response?.status;
+    this.statusText = error.response?.statusText;
+    this.request = {
+      baseURL: error.config?.baseURL,
+      url: error.config?.url,
+      method: error.config?.method,
+      params: error.config?.params,
+      data: error.config?.data,
+    };
+    this.response = {
+      data: error.response?.data,
+    };
+
+    delete error.response;
+    delete error.config;
+    delete error.request;
+    // logger.error(error);
+  }
+}
+
+export const HttpCommonError = HttpError;
+/**
+ * @description 创建请求实例
+ */
+export function createAxiosService({ logger }: { logger: Logger }) {
+  // 创建一个 axios 实例
+  const service = axios.create();
+
+  const defaultAgents = createAgent();
+  // 请求拦截
+  service.interceptors.request.use(
+    (config: any) => {
+      logger.info(`http request:${config.url}，method:${config.method}，params:${JSON.stringify(config.params)}`);
+      if (config.timeout == null) {
+        config.timeout = 15000;
+      }
+      let agents = defaultAgents;
+      if (config.skipSslVerify) {
+        logger.info('跳过SSL验证');
+        agents = createAgent({ rejectUnauthorized: false } as any);
+      }
+      delete config.skipSslVerify;
+      config.httpsAgent = agents.httpsAgent;
+      config.httpAgent = agents.httpAgent;
+      config.proxy = false; //必须 否则还会走一层代理，
+      return config;
+    },
+    (error: Error) => {
+      // 发送失败
+      logger.error('接口请求失败：', error);
+      return Promise.reject(error);
+    }
+  );
+  // 响应拦截
+  service.interceptors.response.use(
+    (response: any) => {
+      logger.info('http response:', JSON.stringify(response?.data));
+      return response.data;
+    },
+    (error: any) => {
+      const status = error.response?.status;
+      switch (status) {
+        case 400:
+          error.message = '请求错误';
+          break;
+        case 401:
+          error.message = '未授权，请登录';
+          break;
+        case 403:
+          error.message = '拒绝访问';
+          break;
+        case 404:
+          error.message = `请求地址出错: ${error.response.config.url}`;
+          break;
+        case 408:
+          error.message = '请求超时';
+          break;
+        case 500:
+          error.message = '服务器内部错误';
+          break;
+        case 501:
+          error.message = '服务未实现';
+          break;
+        case 502:
+          error.message = '网关错误';
+          break;
+        case 503:
+          error.message = '服务不可用';
+          break;
+        case 504:
+          error.message = '网关超时';
+          break;
+        case 505:
+          error.message = 'HTTP版本不受支持';
+          break;
+        default:
+          break;
+      }
+      logger.error(
+        `请求出错：status:${error.response?.status},statusText:${error.response?.statusText},url:${error.config?.url},method:${error.config?.method}。`
+      );
+      logger.error('返回数据:', JSON.stringify(error.response?.data));
+      if (error.response?.data) {
+        error.message = error.response.data.message || error.response.data.msg || error.response.data.error || error.response.data;
+      }
+      if (error instanceof AggregateError) {
+        logger.error('AggregateError', error);
+      }
+      const err = new HttpError(error);
+      return Promise.reject(err);
+    }
+  );
+  return service;
+}
+
+export const http = createAxiosService({ logger }) as HttpClient;
+export type HttpClientResponse<R> = any;
+export type HttpRequestConfig<D> = {
+  skipSslVerify?: boolean;
+  skipCheckRes?: boolean;
+} & AxiosRequestConfig<D>;
+export type HttpClient = {
+  request<D = any, R = any>(config: HttpRequestConfig<D>): Promise<HttpClientResponse<R>>;
+};
+
+export function createAgent(opts: nodeHttp.AgentOptions = {}) {
+  let httpAgent, httpsAgent;
+  const httpProxy = process.env.HTTP_PROXY || process.env.http_proxy;
+  if (httpProxy) {
+    logger.info('use httpProxy:', httpProxy);
+    httpAgent = new HttpProxyAgent(httpProxy, opts as any);
+  } else {
+    httpAgent = new nodeHttp.Agent(opts);
+  }
+  const httpsProxy = process.env.HTTPS_PROXY || process.env.https_proxy;
+  if (httpsProxy) {
+    logger.info('use httpsProxy:', httpsProxy);
+    httpsAgent = new HttpsProxyAgent(httpsProxy, opts as any);
+  } else {
+    httpsAgent = new https.Agent(opts);
+  }
+  return {
+    httpAgent,
+    httpsAgent,
+  };
+}

packages/core/basic/src/utils/util.sleep.ts

@@ -0,0 +1,7 @@
+export default function (timeout: number) {
+  return new Promise((resolve) => {
+    setTimeout(() => {
+      resolve({});
+    }, timeout);
+  });
+}

packages/core/basic/src/utils/util.sp.ts

@@ -0,0 +1,112 @@
+//转换为import
+import childProcess from "child_process";
+import { safePromise } from "./util.promise.js";
+import { ILogger, logger } from "./util.log.js";
+
+export type ExecOption = {
+  cmd: string | string[];
+  env: any;
+  logger?: ILogger;
+  options?: any;
+};
+
+async function exec(opts: ExecOption): Promise<string> {
+  let cmd = "";
+  const log = opts.logger || logger;
+  if (opts.cmd instanceof Array) {
+    for (const item of opts.cmd) {
+      if (cmd) {
+        cmd += " && " + item;
+      } else {
+        cmd = item;
+      }
+    }
+  }
+  log.info(`执行命令: ${cmd}`);
+  return safePromise((resolve, reject) => {
+    childProcess.exec(
+      cmd,
+      {
+        env: {
+          ...process.env,
+          ...opts.env,
+        },
+        ...opts.options,
+      },
+      (error, stdout, stderr) => {
+        if (error) {
+          log.error(`exec error: ${error}`);
+          reject(error);
+        } else {
+          const res = stdout.toString("utf-8");
+          log.info(`stdout: ${res}`);
+          resolve(res);
+        }
+      }
+    );
+  });
+}
+
+export type SpawnOption = {
+  cmd: string | string[];
+  onStdout?: (data: string) => void;
+  onStderr?: (data: string) => void;
+  env?: any;
+  logger?: ILogger;
+  options?: any;
+};
+async function spawn(opts: SpawnOption): Promise<string> {
+  let cmd = "";
+  const log = opts.logger || logger;
+  if (opts.cmd instanceof Array) {
+    for (const item of opts.cmd) {
+      if (cmd) {
+        cmd += " && " + item;
+      } else {
+        cmd = item;
+      }
+    }
+  } else {
+    cmd = opts.cmd;
+  }
+  log.info(`执行命令: ${cmd}`);
+  let stdout = "";
+  let stderr = "";
+  return safePromise((resolve, reject) => {
+    const ls = childProcess.spawn(cmd, {
+      shell: true,
+      env: {
+        ...process.env,
+        ...opts.env,
+      },
+      ...opts.options,
+    });
+    ls.stdout.on("data", (data) => {
+      log.info(`stdout: ${data}`);
+      stdout += data;
+    });
+
+    ls.stderr.on("data", (data) => {
+      log.error(`stderr: ${data}`);
+      stderr += data;
+    });
+    ls.on("error", (error) => {
+      log.error(`child process error: ${error}`);
+      reject(error);
+    });
+
+    ls.on("close", (code: number) => {
+      if (code !== 0) {
+        log.error(`child process exited with code ${code}`);
+        reject(new Error(stderr));
+      } else {
+        resolve(stdout);
+      }
+    });
+  });
+}
+
+export const sp = {
+  spawn,
+  exec,
+};