use std::net::SocketAddr; use std::sync::Arc; use arc_swap::ArcSwapOption; use async_trait::async_trait; use log::{debug, trace}; use futures::io::{AsyncReadExt, AsyncWriteExt}; use kuska_handshake::async_std::{handshake_server, BoxStream}; use tokio::net::TcpStream; use tokio::select; use tokio::sync::{mpsc, watch}; use tokio_util::compat::*; #[cfg(feature = "telemetry")] use opentelemetry::{ trace::{FutureExt, Span, SpanKind, TraceContextExt, TraceId, Tracer}, Context, KeyValue, }; #[cfg(feature = "telemetry")] use opentelemetry_contrib::trace::propagator::binary::*; #[cfg(feature = "telemetry")] use rand::{thread_rng, Rng}; use crate::error::*; use crate::message::*; use crate::netapp::*; use crate::recv::*; use crate::send::*; use crate::stream::*; use crate::util::*; // The client and server connection structs (client.rs and server.rs) // build upon the chunking mechanism which is exclusively contained // in proto.rs. // Here, we just care about sending big messages without size limit. // The format of these messages is described below. // Chunking happens independently. // Request message format (client -> server): // - u8 priority // - u8 path length // - [u8; path length] path // - [u8; *] data // Response message format (server -> client): // - u8 response code // - [u8; *] response pub(crate) struct ServerConn { pub(crate) remote_addr: SocketAddr, pub(crate) peer_id: NodeID, netapp: Arc, resp_send: ArcSwapOption>, } impl ServerConn { pub(crate) async fn run( netapp: Arc, socket: TcpStream, must_exit: watch::Receiver, ) -> Result<(), Error> { let remote_addr = socket.peer_addr()?; let mut socket = socket.compat(); // Do handshake to authenticate client let handshake = handshake_server( &mut socket, netapp.netid.clone(), netapp.id, netapp.privkey.clone(), ) .await?; let peer_id = handshake.peer_pk; debug!( "Handshake complete (server) with {}@{}", hex::encode(&peer_id), remote_addr ); // Create BoxStream layer that encodes content let (read, write) = socket.split(); let (read, mut write) = BoxStream::from_handshake(read, write, handshake, 0x8000).split_read_write(); // Before doing anything, send version tag, so that client // can check and disconnect if version is wrong write.write_all(&netapp.version_tag[..]).await?; write.flush().await?; // Build and launch stuff that handles requests server-side let (resp_send, resp_recv) = mpsc::unbounded_channel(); let conn = Arc::new(ServerConn { netapp: netapp.clone(), remote_addr, peer_id, resp_send: ArcSwapOption::new(Some(Arc::new(resp_send))), }); netapp.connected_as_server(peer_id, conn.clone()); let conn2 = conn.clone(); let recv_future = tokio::spawn(async move { select! { r = conn2.recv_loop(read) => r, _ = await_exit(must_exit) => Ok(()) } }); let send_future = tokio::spawn(conn.clone().send_loop(resp_recv, write)); recv_future.await.log_err("ServerConn recv_loop"); conn.resp_send.store(None); send_future.await.log_err("ServerConn send_loop"); netapp.disconnected_as_server(&peer_id, conn); Ok(()) } async fn recv_handler_aux(self: &Arc, req_enc: ReqEnc) -> Result { let path = String::from_utf8(req_enc.path.to_vec())?; let handler_opt = { let endpoints = self.netapp.endpoints.read().unwrap(); endpoints.get(&path[..]).map(|e| e.clone_endpoint()) }; if let Some(handler) = handler_opt { cfg_if::cfg_if! { if #[cfg(feature = "telemetry")] { let tracer = opentelemetry::global::tracer("netapp"); let mut span = if !req_enc.telemetry_id.is_empty() { let propagator = BinaryPropagator::new(); let context = propagator.from_bytes(req_enc.telemetry_id.to_vec()); let context = Context::new().with_remote_span_context(context); tracer.span_builder(format!(">> RPC {}", path)) .with_kind(SpanKind::Server) .start_with_context(&tracer, &context) } else { let mut rng = thread_rng(); let trace_id = TraceId::from_bytes(rng.gen()); tracer .span_builder(format!(">> RPC {}", path)) .with_kind(SpanKind::Server) .with_trace_id(trace_id) .start(&tracer) }; span.set_attribute(KeyValue::new("path", path.to_string())); span.set_attribute(KeyValue::new("len_query_msg", req_enc.msg.len() as i64)); handler.handle(req_enc, self.peer_id) .with_context(Context::current_with_span(span)) .await } else { handler.handle(req_enc, self.peer_id).await } } } else { Err(Error::NoHandler) } } } impl SendLoop for ServerConn {} #[async_trait] impl RecvLoop for ServerConn { fn recv_handler(self: &Arc, id: RequestID, stream: ByteStream) { let resp_send = self.resp_send.load_full().unwrap(); let self2 = self.clone(); tokio::spawn(async move { trace!("ServerConn recv_handler {}", id); let (prio, resp_enc) = match ReqEnc::decode(stream).await { Ok(req_enc) => { let prio = req_enc.prio; let resp = self2.recv_handler_aux(req_enc).await; ( prio, match resp { Ok(resp_enc) => resp_enc, Err(e) => RespEnc::from_err(e), }, ) } Err(e) => (PRIO_NORMAL, RespEnc::from_err(e)), }; trace!("ServerConn sending response to {}: ", id); resp_send .send((id, prio, resp_enc.encode())) .log_err("ServerConn recv_handler send resp bytes"); Ok::<_, Error>(()) }); } }