Implement /health admin API endpoint to check node health

author: Alex Auvolat <alex@adnab.me> 2022-12-05 14:59:15 +0100
committer: Alex Auvolat <alex@adnab.me> 2022-12-05 14:59:15 +0100
commit: 2065f011ca3f7c736feecffd108c89d3f8019e85 (patch)
tree: 86d761c1d5bea219ea2013a7e72762b42097291a /src
parent: 243b7c9a1cbf0e1dc4e9cf3b935bd833e29dcf9d (diff)
download: garage-2065f011ca3f7c736feecffd108c89d3f8019e85.tar.gz
garage-2065f011ca3f7c736feecffd108c89d3f8019e85.zip
4 files changed, 103 insertions, 0 deletions
diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs
index 2896d058..9beeda1f 100644
--- a/src/api/admin/api_server.rs
+++ b/src/api/admin/api_server.rs
@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+use std::fmt::Write;
 use std::net::SocketAddr;
 use std::sync::Arc;
 
@@ -15,6 +17,8 @@ use opentelemetry_prometheus::PrometheusExporter;
 use prometheus::{Encoder, TextEncoder};
 
 use garage_model::garage::Garage;
+use garage_rpc::layout::NodeRoleV;
+use garage_util::data::Uuid;
 use garage_util::error::Error as GarageError;
 
 use crate::generic_server::*;
@@ -76,6 +80,94 @@ impl AdminApiServer {
 			.body(Body::empty())?)
 	}
 
+	fn handle_health(&self) -> Result<Response<Body>, Error> {
+		let ring: Arc<_> = self.garage.system.ring.borrow().clone();
+		let quorum = self.garage.replication_mode.write_quorum();
+		let replication_factor = self.garage.replication_mode.replication_factor();
+
+		let nodes = self
+			.garage
+			.system
+			.get_known_nodes()
+			.into_iter()
+			.map(|n| (n.id, n))
+			.collect::<HashMap<Uuid, _>>();
+		let n_nodes_connected = nodes.iter().filter(|(_, n)| n.is_up).count();
+
+		let storage_nodes = ring
+			.layout
+			.roles
+			.items()
+			.iter()
+			.filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_some()))
+			.collect::<Vec<_>>();
+		let n_storage_nodes_ok = storage_nodes
+			.iter()
+			.filter(|(x, _, _)| nodes.get(x).map(|n| n.is_up).unwrap_or(false))
+			.count();
+
+		let partitions = ring.partitions();
+		let partitions_n_up = partitions
+			.iter()
+			.map(|(_, h)| {
+				let pn = ring.get_nodes(h, ring.replication_factor);
+				pn.iter()
+					.filter(|x| nodes.get(x).map(|n| n.is_up).unwrap_or(false))
+					.count()
+			})
+			.collect::<Vec<usize>>();
+		let n_partitions_full_ok = partitions_n_up
+			.iter()
+			.filter(|c| **c == replication_factor)
+			.count();
+		let n_partitions_quorum = partitions_n_up.iter().filter(|c| **c >= quorum).count();
+
+		let (status, status_str) = if n_partitions_quorum == partitions.len()
+			&& n_storage_nodes_ok == storage_nodes.len()
+		{
+			(StatusCode::OK, "Garage is fully operational")
+		} else if n_partitions_quorum == partitions.len() {
+			(
+				StatusCode::OK,
+				"Garage is operational but some storage nodes are unavailable",
+			)
+		} else {
+			(
+				StatusCode::SERVICE_UNAVAILABLE,
+				"Quorum is not available for some/all partitions, reads and writes will fail",
+			)
+		};
+
+		let mut buf = status_str.to_string();
+		writeln!(
+			&mut buf,
+			"\nAll nodes: {} connected, {} known",
+			n_nodes_connected,
+			nodes.len()
+		)
+		.unwrap();
+		writeln!(
+			&mut buf,
+			"Storage nodes: {} connected, {} in layout",
+			n_storage_nodes_ok,
+			storage_nodes.len()
+		)
+		.unwrap();
+		writeln!(&mut buf, "Number of partitions: {}", partitions.len()).unwrap();
+		writeln!(&mut buf, "Partitions with quorum: {}", n_partitions_quorum).unwrap();
+		writeln!(
+			&mut buf,
+			"Partitions with all nodes available: {}",
+			n_partitions_full_ok
+		)
+		.unwrap();
+
+		Ok(Response::builder()
+			.status(status)
+			.header(http::header::CONTENT_TYPE, "text/plain")
+			.body(Body::from(buf))?)
+	}
+
 	fn handle_metrics(&self) -> Result<Response<Body>, Error> {
 		#[cfg(feature = "metrics")]
 		{
@@ -124,6 +216,7 @@ impl ApiHandler for AdminApiServer {
 	) -> Result<Response<Body>, Error> {
 		let expected_auth_header =
 			match endpoint.authorization_type() {
+				Authorization::None => None,
 				Authorization::MetricsToken => self.metrics_token.as_ref(),
 				Authorization::AdminToken => match &self.admin_token {
 					None => return Err(Error::forbidden(
@@ -147,6 +240,7 @@ impl ApiHandler for AdminApiServer {
 
 		match endpoint {
 			Endpoint::Options => self.handle_options(&req),
+			Endpoint::Health => self.handle_health(),
 			Endpoint::Metrics => self.handle_metrics(),
 			Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await,
 			Endpoint::ConnectClusterNodes => handle_connect_cluster_nodes(&self.garage, req).await,
diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs
index 3eee8b67..14411f75 100644
--- a/src/api/admin/router.rs
+++ b/src/api/admin/router.rs
@@ -6,6 +6,7 @@ use crate::admin::error::*;
 use crate::router_macros::*;
 
 pub enum Authorization {
+	None,
 	MetricsToken,
 	AdminToken,
 }
@@ -16,6 +17,7 @@ router_match! {@func
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Endpoint {
 	Options,
+	Health,
 	Metrics,
 	GetClusterStatus,
 	ConnectClusterNodes,
@@ -88,6 +90,7 @@ impl Endpoint {
 
 		let res = router_match!(@gen_path_parser (req.method(), path, query) [
 			OPTIONS _ => Options,
+			GET "/health" => Health,
 			GET "/metrics" => Metrics,
 			GET "/v0/status" => GetClusterStatus,
 			POST "/v0/connect" => ConnectClusterNodes,
@@ -130,6 +133,7 @@ impl Endpoint {
 	/// Get the kind of authorization which is required to perform the operation.
 	pub fn authorization_type(&self) -> Authorization {
 		match self {
+			Self::Health => Authorization::None,
 			Self::Metrics => Authorization::MetricsToken,
 			_ => Authorization::AdminToken,
 		}
diff --git a/src/model/garage.rs b/src/model/garage.rs
index 75012952..c2aabea1 100644
--- a/src/model/garage.rs
+++ b/src/model/garage.rs
@@ -34,6 +34,9 @@ pub struct Garage {
 	/// The parsed configuration Garage is running
 	pub config: Config,
 
+	/// The replication mode of this cluster
+	pub replication_mode: ReplicationMode,
+
 	/// The local database
 	pub db: db::Db,
 	/// A background job runner
@@ -258,6 +261,7 @@ impl Garage {
 		// -- done --
 		Ok(Arc::new(Self {
 			config,
+			replication_mode,
 			db,
 			background,
 			system,
diff --git a/src/table/replication/mode.rs b/src/table/replication/mode.rs
index c6f84c45..e244e063 100644
--- a/src/table/replication/mode.rs
+++ b/src/table/replication/mode.rs
@@ -1,3 +1,4 @@
+#[derive(Clone, Copy)]
 pub enum ReplicationMode {
 	None,
 	TwoWay,
author	Alex Auvolat <alex@adnab.me>	2022-12-05 14:59:15 +0100
committer	Alex Auvolat <alex@adnab.me>	2022-12-05 14:59:15 +0100
commit	2065f011ca3f7c736feecffd108c89d3f8019e85 (patch)
tree	86d761c1d5bea219ea2013a7e72762b42097291a /src
parent	243b7c9a1cbf0e1dc4e9cf3b935bd833e29dcf9d (diff)
download	garage-2065f011ca3f7c736feecffd108c89d3f8019e85.tar.gz garage-2065f011ca3f7c736feecffd108c89d3f8019e85.zip