From e349af13a7268d567c1bacc819af5b89c2d4231f Mon Sep 17 00:00:00 2001 From: mricher Date: Tue, 28 Sep 2021 08:57:20 +0200 Subject: Update dependencies and add admin module with metrics - Global dependencies updated in Cargo.lock - New module created in src/admin to host: - the (future) admin REST API - the metric collection - add configuration block No metrics implemented yet --- src/admin/Cargo.toml | 28 ++++++++++ src/admin/lib.rs | 6 +++ src/admin/metrics.rs | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 src/admin/Cargo.toml create mode 100644 src/admin/lib.rs create mode 100644 src/admin/metrics.rs (limited to 'src/admin') diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml new file mode 100644 index 00000000..9775b667 --- /dev/null +++ b/src/admin/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "garage_admin" +version = "0.6.0" +authors = ["Maximilien Richer "] +edition = "2018" +license = "AGPL-3.0" +description = "Administration and metrics REST HTTP server for Garage" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model = { version = "0.6.0", path = "../model" } +garage_util = { version = "0.6.0", path = "../util" } + +futures = "0.3" +futures-util = "0.3" +http = "0.2" +hyper = "0.14" +log = "0.4" + +opentelemetry = "0.17" +opentelemetry-prometheus = "0.10" +prometheus = "0.13" +lazy_static = "1.4" diff --git a/src/admin/lib.rs b/src/admin/lib.rs new file mode 100644 index 00000000..443361be --- /dev/null +++ b/src/admin/lib.rs @@ -0,0 +1,6 @@ +//! Crate for handling the admin and metric HTTP APIs +#[macro_use] +extern crate log; +extern crate lazy_static; + +pub mod metrics; diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs new file mode 100644 index 00000000..547ee4c8 --- /dev/null +++ b/src/admin/metrics.rs @@ -0,0 +1,141 @@ +use hyper::{ + header::CONTENT_TYPE, + service::{make_service_fn, service_fn}, + Body, Method, Request, Response, Server, +}; +use lazy_static::lazy_static; +use opentelemetry::{ + global, + metrics::{BoundCounter, BoundValueRecorder}, + KeyValue, +}; +use opentelemetry_prometheus::PrometheusExporter; +use prometheus::{Encoder, TextEncoder}; +use std::convert::Infallible; +use std::sync::Arc; +use std::time::SystemTime; + +use futures::future::*; +use garage_model::garage::Garage; +use garage_util::error::Error as GarageError; + +lazy_static! { + // This defines the differennt tags that will be referenced by the object + static ref HANDLER_ALL: [KeyValue; 1] = [KeyValue::new("handler", "all")]; +} + +// serve_req on metric endpoint +async fn serve_req( + req: Request, + admin_server: Arc, +) -> Result, hyper::Error> { + println!("Receiving request at path {}", req.uri()); + let request_start = SystemTime::now(); + + admin_server.metrics.http_counter.add(1); + + let response = match (req.method(), req.uri().path()) { + (&Method::GET, "/metrics") => { + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + let metric_families = admin_server.exporter.registry().gather(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + admin_server + .metrics + .http_body_gauge + .record(buffer.len() as u64); + + Response::builder() + .status(200) + .header(CONTENT_TYPE, encoder.format_type()) + .body(Body::from(buffer)) + .unwrap() + } + _ => Response::builder() + .status(404) + .body(Body::from("Not implemented")) + .unwrap(), + }; + + admin_server + .metrics + .http_req_histogram + .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); + Ok(response) +} + +// AdminServer hold the admin server internal admin_server and the metric exporter +pub struct AdminServer { + exporter: PrometheusExporter, + metrics: AdminServerMetrics, +} + +// GarageMetricadmin_server holds the metrics counter definition for Garage +// FIXME: we would rather have that split up among the different libraries? +struct AdminServerMetrics { + http_counter: BoundCounter, + http_body_gauge: BoundValueRecorder, + http_req_histogram: BoundValueRecorder, + bucket_v2_merkle_updater_todo_queue_length: BoundValueRecorder, +} + +impl AdminServer { + /// init initilialize the AdminServer and background metric server + pub fn init() -> AdminServer { + let exporter = opentelemetry_prometheus::exporter().init(); + let meter = global::meter("garage/admin_server"); + AdminServer { + exporter, + metrics: AdminServerMetrics { + http_counter: meter + .u64_counter("router.http_requests_total") + .with_description("Total number of HTTP requests made.") + .init() + .bind(HANDLER_ALL.as_ref()), + http_body_gauge: meter + .u64_value_recorder("example.http_response_size_bytes") + .with_description("The metrics HTTP response sizes in bytes.") + .init() + .bind(HANDLER_ALL.as_ref()), + http_req_histogram: meter + .f64_value_recorder("example.http_request_duration_seconds") + .with_description("The HTTP request latencies in seconds.") + .init() + .bind(HANDLER_ALL.as_ref()), + bucket_v2_merkle_updater_todo_queue_length: meter + .f64_value_recorder("bucket_v2.merkle_updater.todo_queue_length") + .with_description("Bucket merkle updater TODO queue length.") + .init() + .bind(HANDLER_ALL.as_ref()), + }, + } + } + /// run execute the admin server on the designated HTTP port and listen for requests + pub async fn run( + self, + garage: Arc, + shutdown_signal: impl Future, + ) -> Result<(), GarageError> { + let admin_server = Arc::new(self); + // For every connection, we must make a `Service` to handle all + // incoming HTTP requests on said connection. + let make_svc = make_service_fn(move |_conn| { + let admin_server = admin_server.clone(); + // This is the `Service` that will handle the connection. + // `service_fn` is a helper to convert a function that + // returns a Response into a `Service`. + async move { + Ok::<_, Infallible>(service_fn(move |req| serve_req(req, admin_server.clone()))) + } + }); + + let addr = &garage.config.admin_api.bind_addr; + + let server = Server::bind(&addr).serve(make_svc); + let graceful = server.with_graceful_shutdown(shutdown_signal); + info!("Admin server listening on http://{}", addr); + + graceful.await?; + Ok(()) + } +} -- cgit v1.2.3