From bb04d94fa92740eebeee513030faf136bfd26da7 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 18 Feb 2022 20:39:55 +0100 Subject: Update to Netapp 0.4 which supports distributed tracing --- Cargo.lock | 37 ++++++++++++++++++++++++++++++------- Cargo.nix | 45 ++++++++++++++++++++++++++++++++++++--------- src/api/api_server.rs | 10 +++++++--- src/garage/Cargo.toml | 5 +++-- src/garage/main.rs | 4 ++-- src/model/Cargo.toml | 3 ++- src/model/block.rs | 24 ++++++++++++++++++++++-- src/rpc/Cargo.toml | 4 +++- src/rpc/rpc_helper.rs | 11 ++--------- src/rpc/system.rs | 16 ++++++++-------- src/util/Cargo.toml | 3 ++- 11 files changed, 117 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b7b3174..b5c896c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -821,7 +821,7 @@ dependencies = [ "http", "hyper", "kuska-sodiumoxide", - "netapp", + "netapp 0.4.0", "pretty_env_logger", "rand", "rmp-serde 0.15.5", @@ -909,7 +909,7 @@ dependencies = [ "garage_util 0.5.1", "hex", "log", - "netapp", + "netapp 0.3.1", "rand", "rmp-serde 0.15.5", "serde", @@ -933,7 +933,7 @@ dependencies = [ "garage_table 0.6.0", "garage_util 0.6.0", "hex", - "netapp", + "netapp 0.4.0", "opentelemetry", "rand", "rmp-serde 0.15.5", @@ -962,7 +962,7 @@ dependencies = [ "hyper", "kuska-sodiumoxide", "log", - "netapp", + "netapp 0.3.1", "rand", "rmp-serde 0.15.5", "serde", @@ -988,7 +988,7 @@ dependencies = [ "k8s-openapi", "kube", "kuska-sodiumoxide", - "netapp", + "netapp 0.4.0", "openssl", "opentelemetry", "pnet", @@ -1060,7 +1060,7 @@ dependencies = [ "http", "hyper", "log", - "netapp", + "netapp 0.3.1", "rand", "rmp-serde 0.15.5", "serde", @@ -1085,7 +1085,7 @@ dependencies = [ "hyper", "k8s-openapi", "kube", - "netapp", + "netapp 0.4.0", "rand", "rmp-serde 0.15.5", "serde", @@ -1794,6 +1794,29 @@ dependencies = [ "tokio-util 0.6.9", ] +[[package]] +name = "netapp" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22c545a13b0c47b47e8052b35c4884dbe33c9ea62607371b0f4f1b0490cafd38" +dependencies = [ + "arc-swap", + "async-trait", + "bytes 0.6.0", + "cfg-if", + "err-derive 0.2.4", + "futures", + "hex", + "kuska-handshake", + "kuska-sodiumoxide", + "log", + "rmp-serde 0.14.4", + "serde", + "tokio", + "tokio-stream", + "tokio-util 0.6.9", +] + [[package]] name = "nom" version = "7.1.0" diff --git a/Cargo.nix b/Cargo.nix index e417c86b..c4d3d36f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1162,7 +1162,7 @@ in git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; - netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; pretty_env_logger = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; @@ -1289,7 +1289,7 @@ in garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; - netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; @@ -1348,7 +1348,7 @@ in k8s_openapi = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; }; kube = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; - netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; openssl = rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl."0.10.38" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; pnet = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet."0.28.0" { inherit profileName; }; @@ -1453,7 +1453,7 @@ in hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; k8s_openapi = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; }; kube = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; }; - netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; @@ -2396,6 +2396,33 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" = overridableMkRustCrate (profileName: rec { + name = "netapp"; + version = "0.4.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "22c545a13b0c47b47e8052b35c4884dbe33c9ea62607371b0f4f1b0490cafd38"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + arc_swap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }; + async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }; + bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."0.6.0" { inherit profileName; }; + cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; + err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.2.4" { profileName = "__noProfile"; }; + futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; + hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; + kuska_handshake = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" { inherit profileName; }; + sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.14.4" { inherit profileName; }; + serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; + tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }; + tokio_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.9" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".nom."7.1.0" = overridableMkRustCrate (profileName: rec { name = "nom"; version = "7.1.0"; @@ -3734,7 +3761,7 @@ in ]; dependencies = { bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" { inherit profileName; }; static_init_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; }; @@ -4729,11 +4756,11 @@ in [ "default" ] ]; dependencies = { - ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; }; }); diff --git a/src/api/api_server.rs b/src/api/api_server.rs index 15b00dde..00d582d1 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -9,7 +9,7 @@ use hyper::{Body, Method, Request, Response, Server}; use opentelemetry::{ trace::{FutureExt, TraceContextExt, Tracer}, - Context, + Context, KeyValue, }; use garage_util::data::*; @@ -50,15 +50,19 @@ pub async fn run_api_server( let garage = garage.clone(); let tracer = opentelemetry::global::tracer("garage"); - let uuid = gen_uuid(); + let trace_id = gen_uuid(); let span = tracer .span_builder("S3 API call (unknown)") .with_trace_id( opentelemetry::trace::TraceId::from_hex(&hex::encode( - &uuid.as_slice()[..16], + &trace_id.as_slice()[..16], )) .unwrap(), ) + .with_attributes(vec![ + KeyValue::new("method", format!("{}", req.method())), + KeyValue::new("uri", req.uri().path().to_string()), + ]) .start(&tracer); handler(garage, req, client_addr).with_context(Context::current_with_span(span)) diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 351fa62f..ff0666a6 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -50,8 +50,9 @@ futures = "0.3" futures-util = "0.3" tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } -netapp = "0.3.0" +#netapp = { version = "0.3.2", git = "https://git.deuxfleurs.fr/lx/netapp" } +#netapp = { version = "0.4", path = "../../../netapp" } +netapp = "0.4" [dev-dependencies] aws-sdk-s3 = "0.6" diff --git a/src/garage/main.rs b/src/garage/main.rs index 7de7740f..08ec912b 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -55,7 +55,7 @@ struct Opt { #[tokio::main] async fn main() { if std::env::var("RUST_LOG").is_err() { - std::env::set_var("RUST_LOG", "garage=info") + std::env::set_var("RUST_LOG", "netapp=info,garage=info") } pretty_env_logger::init(); sodiumoxide::init().expect("Unable to init sodiumoxide"); @@ -106,7 +106,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { // Generate a temporary keypair for our RPC client let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair(); - let netapp = NetApp::new(network_key, sk); + let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk); // Find and parse the address of the target host let (id, addr) = if let Some(h) = opt.rpc_host { diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 4465af1c..8083445e 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -39,4 +39,5 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi opentelemetry = "0.17" #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } -netapp = "0.3.0" +#netapp = { version = "0.4", path = "../../../netapp" } +netapp = "0.4" diff --git a/src/model/block.rs b/src/model/block.rs index ddda5e57..3799c6aa 100644 --- a/src/model/block.rs +++ b/src/model/block.rs @@ -14,7 +14,10 @@ use tokio::fs; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::{watch, Mutex, Notify}; -use opentelemetry::KeyValue; +use opentelemetry::{ + trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer}, + Context, KeyValue, +}; use garage_util::data::*; use garage_util::error::*; @@ -554,7 +557,24 @@ impl BlockManager { let start_time = SystemTime::now(); let hash = Hash::try_from(&hash_bytes[..]).unwrap(); - let res = self.resync_block(&hash).await; + + let tracer = opentelemetry::global::tracer("garage"); + let trace_id = gen_uuid(); + let span = tracer + .span_builder("Resync block") + .with_trace_id( + opentelemetry::trace::TraceId::from_hex(&hex::encode( + &trace_id.as_slice()[..16], + )) + .unwrap(), + ) + .with_attributes(vec![KeyValue::new("block", format!("{:?}", hash))]) + .start(&tracer); + + let res = self + .resync_block(&hash) + .with_context(Context::current_with_span(span)) + .await; self.metrics.resync_counter.add(1); self.metrics diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 00f609c9..3ab9e7da 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -46,6 +46,8 @@ tokio-stream = { version = "0.1", features = ["net"] } opentelemetry = "0.17" #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } -netapp = "0.3.1" +#netapp = { version = "0.4", path = "../../../netapp", features = ["telemetry"] } +netapp = "0.4" + hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] } diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index 97716b18..4b4235f1 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -152,14 +152,8 @@ impl RpcHelper { self.0.metrics.rpc_counter.add(1, &metric_tags); let rpc_start_time = SystemTime::now(); - let tracer = opentelemetry::global::tracer("garage"); - let mut span = tracer.start(format!("RPC {}", endpoint.path())); - span.set_attribute(KeyValue::new("to", format!("{:?}", to))); - let node_id = to.into(); - let rpc_call = endpoint - .call(&node_id, &msg, strat.rs_priority) - .with_context(Context::current_with_span(span)); + let rpc_call = endpoint.call(&node_id, msg, strat.rs_priority); select! { res = rpc_call => { @@ -246,7 +240,7 @@ impl RpcHelper { let quorum = strategy.rs_quorum.unwrap_or(to.len()); let tracer = opentelemetry::global::tracer("garage"); - let mut span = tracer.start(format!("RPC {} to {:?}", endpoint.path(), to)); + let mut span = tracer.start(format!("RPC {} to {}", endpoint.path(), to.len())); span.set_attribute(KeyValue::new("to", format!("{:?}", to))); span.set_attribute(KeyValue::new("quorum", quorum as i64)); @@ -329,7 +323,6 @@ impl RpcHelper { // reach quorum, start some new requests. while successes.len() + resp_stream.len() < quorum { if let Some((_, _, _, req_to, fut)) = requests.next() { - let tracer = opentelemetry::global::tracer("garage"); let span = tracer.start(format!("RPC to {:?}", req_to)); resp_stream.push(tokio::spawn( fut.with_context(Context::current_with_span(span)), diff --git a/src/rpc/system.rs b/src/rpc/system.rs index c8fc0ad5..2123a37f 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -38,6 +38,9 @@ const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60); const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10); const PING_TIMEOUT: Duration = Duration::from_secs(2); +/// Version tag used for version check upon Netapp connection +pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650006; // garage 0x0006 + /// RPC endpoint used for calls related to membership pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc"; @@ -194,7 +197,10 @@ impl System { ) -> Arc { let node_key = gen_node_key(&config.metadata_dir).expect("Unable to read or generate node ID"); - info!("Node public key: {}", hex::encode(&node_key.public_key())); + info!( + "Node ID of this node: {}", + hex::encode(&node_key.public_key()[..8]) + ); let persist_cluster_layout = Persister::new(&config.metadata_dir, "cluster_layout"); let persist_peer_list = Persister::new(&config.metadata_dir, "peer_list"); @@ -222,13 +228,7 @@ impl System { let ring = Ring::new(cluster_layout, replication_factor); let (update_ring, ring) = watch::channel(Arc::new(ring)); - if let Some(addr) = config.rpc_public_addr { - println!("{}@{}", hex::encode(&node_key.public_key()), addr); - } else { - println!("{}", hex::encode(&node_key.public_key())); - } - - let netapp = NetApp::new(network_key, node_key); + let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, node_key); let fullmesh = FullMeshPeeringStrategy::new( netapp.clone(), config.bootstrap_peers.clone(), diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 3cc2031d..76b73f4b 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -34,7 +34,8 @@ futures = "0.3" tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } -netapp = "0.3.0" +#netapp = { version = "0.4", path = "../../../netapp" } +netapp = "0.4" http = "0.2" hyper = "0.14" -- cgit v1.2.3