use std::borrow::BorrowMut; use std::io::Write; use std::sync::Arc; use futures::StreamExt; use pnet::packet::ethernet::EthernetPacket; use pnet::packet::ipv4::Ipv4Packet; use tokio::{sync::Mutex, task::JoinSet}; use tokio_util::bytes::{Bytes, BytesMut}; use tonic::transport::Server; use uuid::Uuid; use crate::common::config_fs::ConfigFs; use crate::common::error::Error; use crate::common::global_ctx::{ArcGlobalCtx, GlobalCtx}; use crate::common::netns::NetNS; use crate::connector::direct::DirectConnectorManager; use crate::connector::manual::{ConnectorManagerRpcService, ManualConnectorManager}; use crate::connector::udp_hole_punch::UdpHolePunchConnector; use crate::gateway::icmp_proxy::IcmpProxy; use crate::gateway::tcp_proxy::TcpProxy; use crate::peers::peer_manager::PeerManager; use crate::peers::rpc_service::PeerManagerRpcService; use crate::tunnels::SinkItem; use tokio_stream::wrappers::ReceiverStream; use super::listeners::ListenerManager; use super::virtual_nic; pub struct InstanceConfigWriter { config: ConfigFs, } impl InstanceConfigWriter { pub fn new(inst_name: &str) -> Self { InstanceConfigWriter { config: ConfigFs::new(inst_name), } } pub fn set_ns(self, net_ns: Option) -> Self { let net_ns_in_conf = if let Some(net_ns) = net_ns { net_ns } else { "".to_string() }; self.config .add_file("net_ns") .unwrap() .write_all(net_ns_in_conf.as_bytes()) .unwrap(); self } pub fn set_addr(self, addr: String) -> Self { self.config .add_file("ipv4") .unwrap() .write_all(addr.as_bytes()) .unwrap(); self } } pub struct Instance { inst_name: String, id: uuid::Uuid, virtual_nic: Option>, peer_packet_receiver: Option>, tasks: JoinSet<()>, peer_manager: Arc, listener_manager: Arc>>, conn_manager: Arc, direct_conn_manager: Arc, udp_hole_puncher: Arc>, tcp_proxy: Arc, icmp_proxy: Arc, global_ctx: ArcGlobalCtx, } impl Instance { pub fn new(inst_name: &str) -> Self { let config = ConfigFs::new(inst_name); let net_ns_in_conf = config.get_or_default("net_ns", || "".to_string()).unwrap(); let net_ns = NetNS::new(if net_ns_in_conf.is_empty() { None } else { Some(net_ns_in_conf.clone()) }); let addr = config .get_or_default("ipv4", || "10.144.144.10".to_string()) .unwrap(); log::info!( "[INIT] instance creating. inst_name: {}, addr: {}, netns: {}", inst_name, addr, net_ns_in_conf ); let (peer_packet_sender, peer_packet_receiver) = tokio::sync::mpsc::channel(100); let global_ctx = Arc::new(GlobalCtx::new(inst_name, config, net_ns.clone())); let id = global_ctx.get_id(); let peer_manager = Arc::new(PeerManager::new( global_ctx.clone(), peer_packet_sender.clone(), )); let listener_manager = Arc::new(Mutex::new(ListenerManager::new( id, net_ns.clone(), peer_manager.clone(), ))); let conn_manager = Arc::new(ManualConnectorManager::new( id, global_ctx.clone(), peer_manager.clone(), )); let mut direct_conn_manager = DirectConnectorManager::new(id, global_ctx.clone(), peer_manager.clone()); direct_conn_manager.run(); let udp_hole_puncher = UdpHolePunchConnector::new(global_ctx.clone(), peer_manager.clone()); let arc_tcp_proxy = TcpProxy::new(global_ctx.clone(), peer_manager.clone()); let arc_icmp_proxy = IcmpProxy::new(global_ctx.clone(), peer_manager.clone()).unwrap(); Instance { inst_name: inst_name.to_string(), id, virtual_nic: None, peer_packet_receiver: Some(ReceiverStream::new(peer_packet_receiver)), tasks: JoinSet::new(), peer_manager, listener_manager, conn_manager, direct_conn_manager: Arc::new(direct_conn_manager), udp_hole_puncher: Arc::new(Mutex::new(udp_hole_puncher)), tcp_proxy: arc_tcp_proxy, icmp_proxy: arc_icmp_proxy, global_ctx, } } pub fn get_conn_manager(&self) -> Arc { self.conn_manager.clone() } async fn do_forward_nic_to_peers_ipv4(ret: BytesMut, mgr: &PeerManager) { if let Some(ipv4) = Ipv4Packet::new(&ret) { if ipv4.get_version() != 4 { tracing::info!("[USER_PACKET] not ipv4 packet: {:?}", ipv4); } let dst_ipv4 = ipv4.get_destination(); tracing::trace!( ?ret, "[USER_PACKET] recv new packet from tun device and forward to peers." ); let send_ret = mgr.send_msg_ipv4(ret, dst_ipv4).await; if send_ret.is_err() { tracing::trace!(?send_ret, "[USER_PACKET] send_msg_ipv4 failed") } } else { tracing::warn!(?ret, "[USER_PACKET] not ipv4 packet"); } } async fn do_forward_nic_to_peers_ethernet(mut ret: BytesMut, mgr: &PeerManager) { if let Some(eth) = EthernetPacket::new(&ret) { log::warn!("begin to forward: {:?}, type: {}", eth, eth.get_ethertype()); Self::do_forward_nic_to_peers_ipv4(ret.split_off(14), mgr).await; } else { log::warn!("not ipv4 packet: {:?}", ret); } } fn do_forward_nic_to_peers(&mut self) -> Result<(), Error> { // read from nic and write to corresponding tunnel let nic = self.virtual_nic.as_ref().unwrap(); let nic = nic.clone(); let mgr = self.peer_manager.clone(); self.tasks.spawn(async move { let mut stream = nic.pin_recv_stream(); while let Some(ret) = stream.next().await { if ret.is_err() { log::error!("read from nic failed: {:?}", ret); break; } Self::do_forward_nic_to_peers_ipv4(ret.unwrap(), mgr.as_ref()).await; // Self::do_forward_nic_to_peers_ethernet(ret.into(), mgr.as_ref()).await; } }); Ok(()) } fn do_forward_peers_to_nic( tasks: &mut JoinSet<()>, nic: Arc, channel: Option>, ) { tasks.spawn(async move { let send = nic.pin_send_stream(); let channel = channel.unwrap(); let ret = channel .map(|packet| { log::trace!( "[USER_PACKET] forward packet from peers to nic. packet: {:?}", packet ); Ok(packet) }) .forward(send) .await; if ret.is_err() { panic!("do_forward_tunnel_to_nic"); } }); } pub async fn run(&mut self) -> Result<(), Error> { let ipv4_addr = self.global_ctx.get_ipv4().unwrap(); let mut nic = virtual_nic::VirtualNic::new(self.get_global_ctx()) .create_dev() .await? .link_up() .await? .remove_ip(None) .await? .add_ip(ipv4_addr, 24) .await?; if cfg!(target_os = "macos") { nic = nic.add_route(ipv4_addr, 24).await?; } self.virtual_nic = Some(Arc::new(nic)); self.do_forward_nic_to_peers().unwrap(); Self::do_forward_peers_to_nic( self.tasks.borrow_mut(), self.virtual_nic.as_ref().unwrap().clone(), self.peer_packet_receiver.take(), ); self.listener_manager .lock() .await .prepare_listeners() .await?; self.listener_manager.lock().await.run().await?; self.peer_manager.run().await?; self.run_rpc_server().unwrap(); self.tcp_proxy.start().await.unwrap(); self.icmp_proxy.start().await.unwrap(); self.run_proxy_cidrs_route_updater(); self.udp_hole_puncher.lock().await.run().await?; Ok(()) } pub fn get_peer_manager(&self) -> Arc { self.peer_manager.clone() } pub async fn close_peer_conn(&mut self, peer_id: &Uuid, conn_id: &Uuid) -> Result<(), Error> { self.peer_manager .get_peer_map() .close_peer_conn(peer_id, conn_id) .await?; Ok(()) } pub async fn wait(&mut self) { while let Some(ret) = self.tasks.join_next().await { log::info!("task finished: {:?}", ret); ret.unwrap(); } } pub fn id(&self) -> uuid::Uuid { self.id } fn run_rpc_server(&mut self) -> Result<(), Box> { let addr = "0.0.0.0:15888".parse()?; let peer_mgr = self.peer_manager.clone(); let conn_manager = self.conn_manager.clone(); let net_ns = self.global_ctx.net_ns.clone(); self.tasks.spawn(async move { let _g = net_ns.guard(); log::info!("[INIT RPC] start rpc server. addr: {}", addr); Server::builder() .add_service( crate::rpc::peer_manage_rpc_server::PeerManageRpcServer::new( PeerManagerRpcService::new(peer_mgr), ), ) .add_service( crate::rpc::connector_manage_rpc_server::ConnectorManageRpcServer::new( ConnectorManagerRpcService(conn_manager.clone()), ), ) .serve(addr) .await .unwrap(); }); Ok(()) } fn run_proxy_cidrs_route_updater(&mut self) { let peer_mgr = self.peer_manager.clone(); let net_ns = self.global_ctx.net_ns.clone(); let nic = self.virtual_nic.as_ref().unwrap().clone(); self.tasks.spawn(async move { let mut cur_proxy_cidrs = vec![]; loop { let mut proxy_cidrs = vec![]; let routes = peer_mgr.list_routes().await; for r in routes { for cidr in r.proxy_cidrs { let Ok(cidr) = cidr.parse::() else { continue; }; proxy_cidrs.push(cidr); } } // if route is in cur_proxy_cidrs but not in proxy_cidrs, delete it. for cidr in cur_proxy_cidrs.iter() { if proxy_cidrs.contains(cidr) { continue; } let _g = net_ns.guard(); let ret = nic .get_ifcfg() .remove_ipv4_route( nic.ifname(), cidr.first_address(), cidr.network_length(), ) .await; if ret.is_err() { tracing::trace!( cidr = ?cidr, err = ?ret, "remove route failed.", ); } } for cidr in proxy_cidrs.iter() { if cur_proxy_cidrs.contains(cidr) { continue; } let _g = net_ns.guard(); let ret = nic .get_ifcfg() .add_ipv4_route(nic.ifname(), cidr.first_address(), cidr.network_length()) .await; if ret.is_err() { tracing::trace!( cidr = ?cidr, err = ?ret, "add route failed.", ); } } cur_proxy_cidrs = proxy_cidrs; tokio::time::sleep(std::time::Duration::from_secs(1)).await; } }); } pub fn get_global_ctx(&self) -> ArcGlobalCtx { self.global_ctx.clone() } }