diff options
| author | Alex Auvolat <alex@adnab.me> | 2022-02-17 23:28:23 +0100 |
|---|---|---|
| committer | Alex Auvolat <alex@adnab.me> | 2022-03-14 10:52:13 +0100 |
| commit | 8c2fb0c066af7f68fdcfcdec96fa030af059bf63 (patch) | |
| tree | 58a416058e31eda2cdb3a15c07e565a9ad674857 /src/admin | |
| parent | b6561f6e1bcb6a8de13a186405a480e356df89d8 (diff) | |
| download | garage-8c2fb0c066af7f68fdcfcdec96fa030af059bf63.tar.gz garage-8c2fb0c066af7f68fdcfcdec96fa030af059bf63.zip | |
Add tracing integration with opentelemetry
Diffstat (limited to 'src/admin')
| -rw-r--r-- | src/admin/Cargo.toml | 7 | ||||
| -rw-r--r-- | src/admin/lib.rs | 3 | ||||
| -rw-r--r-- | src/admin/metrics.rs | 40 | ||||
| -rw-r--r-- | src/admin/tracing_setup.rs | 37 |
4 files changed, 77 insertions, 10 deletions
diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 6f646869..b6bf2b3b 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -16,12 +16,15 @@ path = "lib.rs" garage_model = { version = "0.6.0", path = "../model" } garage_util = { version = "0.6.0", path = "../util" } +hex = "0.4" + futures = "0.3" futures-util = "0.3" http = "0.2" hyper = "0.14" -log = "0.4" +tracing = "0.1.30" -opentelemetry = "0.17" +opentelemetry = { version = "0.17", features = [ "rt-tokio" ] } opentelemetry-prometheus = "0.10" +opentelemetry-otlp = "0.10" prometheus = "0.13" diff --git a/src/admin/lib.rs b/src/admin/lib.rs index f1e8ddd7..b5b0775b 100644 --- a/src/admin/lib.rs +++ b/src/admin/lib.rs @@ -1,5 +1,6 @@ //! Crate for handling the admin and metric HTTP APIs #[macro_use] -extern crate log; +extern crate tracing; pub mod metrics; +pub mod tracing_setup; diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs index 44fd4cb2..e12373ab 100644 --- a/src/admin/metrics.rs +++ b/src/admin/metrics.rs @@ -1,20 +1,26 @@ +use std::convert::Infallible; +use std::sync::Arc; +use std::time::SystemTime; + +use futures::future::*; use hyper::{ header::CONTENT_TYPE, service::{make_service_fn, service_fn}, Body, Method, Request, Response, Server, }; + use opentelemetry::{ global, metrics::{BoundCounter, BoundValueRecorder}, + trace::{FutureExt, TraceContextExt, Tracer}, + Context, }; use opentelemetry_prometheus::PrometheusExporter; + use prometheus::{Encoder, TextEncoder}; -use std::convert::Infallible; -use std::sync::Arc; -use std::time::SystemTime; -use futures::future::*; use garage_model::garage::Garage; +use garage_util::data::*; use garage_util::error::Error as GarageError; // serve_req on metric endpoint @@ -22,7 +28,7 @@ async fn serve_req( req: Request<Body>, admin_server: Arc<AdminServer>, ) -> Result<Response<Body>, hyper::Error> { - println!("Receiving request at path {}", req.uri()); + info!("Receiving request at path {}", req.uri()); let request_start = SystemTime::now(); admin_server.metrics.http_counter.add(1); @@ -31,7 +37,12 @@ async fn serve_req( (&Method::GET, "/metrics") => { let mut buffer = vec![]; let encoder = TextEncoder::new(); - let metric_families = admin_server.exporter.registry().gather(); + + let tracer = opentelemetry::global::tracer("garage"); + let metric_families = tracer.in_span("admin/gather_metrics", |_| { + admin_server.exporter.registry().gather() + }); + encoder.encode(&metric_families, &mut buffer).unwrap(); admin_server .metrics @@ -112,7 +123,22 @@ impl AdminServer { // `service_fn` is a helper to convert a function that // returns a Response into a `Service`. async move { - Ok::<_, Infallible>(service_fn(move |req| serve_req(req, admin_server.clone()))) + Ok::<_, Infallible>(service_fn(move |req| { + let tracer = opentelemetry::global::tracer("garage"); + let uuid = gen_uuid(); + let span = tracer + .span_builder("admin/request") + .with_trace_id( + opentelemetry::trace::TraceId::from_hex(&hex::encode( + &uuid.as_slice()[..16], + )) + .unwrap(), + ) + .start(&tracer); + + serve_req(req, admin_server.clone()) + .with_context(Context::current_with_span(span)) + })) } }); diff --git a/src/admin/tracing_setup.rs b/src/admin/tracing_setup.rs new file mode 100644 index 00000000..83fa5622 --- /dev/null +++ b/src/admin/tracing_setup.rs @@ -0,0 +1,37 @@ +use std::time::Duration; + +use opentelemetry::sdk::{ + trace::{self, IdGenerator, Sampler}, + Resource, +}; +use opentelemetry::KeyValue; +use opentelemetry_otlp::WithExportConfig; + +use garage_util::data::*; +use garage_util::error::*; + +pub fn init_tracing(export_to: &str, node_id: Uuid) -> Result<(), Error> { + let node_id = hex::encode(&node_id.as_slice()[..8]); + + opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint(export_to) + .with_timeout(Duration::from_secs(3)), + ) + .with_trace_config( + trace::config() + .with_id_generator(IdGenerator::default()) + .with_sampler(Sampler::TraceIdRatioBased(0.01f64)) + .with_resource(Resource::new(vec![ + KeyValue::new("service.name", "garage"), + KeyValue::new("service.instance.id", node_id), + ])), + ) + .install_batch(opentelemetry::runtime::Tokio) + .ok_or_message("Unable to initialize tracing")?; + + Ok(()) +} |