diff options
Diffstat (limited to 'src/garage')
-rw-r--r-- | src/garage/Cargo.toml | 45 | ||||
-rw-r--r-- | src/garage/admin.rs | 7 | ||||
-rw-r--r-- | src/garage/cli/structs.rs | 112 | ||||
-rw-r--r-- | src/garage/main.rs | 48 | ||||
-rw-r--r-- | src/garage/server.rs | 125 | ||||
-rw-r--r-- | src/garage/tests/lib.rs | 1 |
6 files changed, 219 insertions, 119 deletions
diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 8948e750..b08e9439 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.7.0" +version = "0.8.0" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" @@ -22,13 +22,13 @@ path = "tests/lib.rs" [dependencies] garage_db = { version = "0.8.0", path = "../db" } -garage_api = { version = "0.7.0", path = "../api" } -garage_block = { version = "0.7.0", path = "../block" } -garage_model = { version = "0.7.0", path = "../model" } -garage_rpc = { version = "0.7.0", path = "../rpc" } -garage_table = { version = "0.7.0", path = "../table" } -garage_util = { version = "0.7.0", path = "../util" } -garage_web = { version = "0.7.0", path = "../web" } +garage_api = { version = "0.8.0", path = "../api" } +garage_block = { version = "0.8.0", path = "../block" } +garage_model = { version = "0.8.0", path = "../model" } +garage_rpc = { version = "0.8.0", path = "../rpc" } +garage_table = { version = "0.8.0", path = "../table" } +garage_util = { version = "0.8.0", path = "../util" } +garage_web = { version = "0.8.0", path = "../web" } bytes = "1.0" bytesize = "1.1" @@ -55,9 +55,9 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi netapp = "0.4" opentelemetry = { version = "0.17", features = [ "rt-tokio" ] } -opentelemetry-prometheus = "0.10" -opentelemetry-otlp = "0.10" -prometheus = "0.13" +opentelemetry-prometheus = { version = "0.10", optional = true } +opentelemetry-otlp = { version = "0.10", optional = true } +prometheus = { version = "0.13", optional = true } [dev-dependencies] aws-sdk-s3 = "0.8" @@ -74,5 +74,26 @@ base64 = "0.13" [features] -kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ] +default = [ "bundled-libs", "metrics", "sled" ] + k2v = [ "garage_util/k2v", "garage_api/k2v" ] + +# Database engines, Sled is still our default even though we don't like it +sled = [ "garage_model/sled" ] +lmdb = [ "garage_model/lmdb" ] +sqlite = [ "garage_model/sqlite" ] + +# Automatic registration and discovery via Kubernetes API +kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ] +# Prometheus exporter (/metrics endpoint). +metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ] +# Exporter for the OpenTelemetry Collector. +telemetry-otlp = [ "opentelemetry-otlp" ] + +# NOTE: bundled-libs and system-libs should be treat as mutually exclusive; +# exactly one of them should be enabled. + +# Use bundled libsqlite instead of linking against system-provided. +bundled-libs = [ "garage_db/bundled-libs" ] +# Link against system-provided libsodium and libzstd. +system-libs = [ "garage_block/system-libs", "garage_rpc/system-libs", "sodiumoxide/use-pkg-config" ] diff --git a/src/garage/admin.rs b/src/garage/admin.rs index 76261050..b4d2d1a1 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -741,8 +741,11 @@ impl AdminRpcHandler { let mut ret = String::new(); writeln!( &mut ret, - "\nGarage version: {}", - self.garage.system.garage_version(), + "\nGarage version: {} [features: {}]", + garage_util::version::garage_version(), + garage_util::version::garage_features() + .map(|list| list.join(", ")) + .unwrap_or_else(|| "(unknown)".into()), ) .unwrap(); writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap(); diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index 0388cef5..06548e89 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -1,65 +1,65 @@ use serde::{Deserialize, Serialize}; - -use garage_util::version; use structopt::StructOpt; +use garage_util::version::garage_version; + #[derive(StructOpt, Debug)] pub enum Command { /// Run Garage server - #[structopt(name = "server", version = version::garage())] + #[structopt(name = "server", version = garage_version())] Server, /// Get network status - #[structopt(name = "status", version = version::garage())] + #[structopt(name = "status", version = garage_version())] Status, /// Operations on individual Garage nodes - #[structopt(name = "node", version = version::garage())] + #[structopt(name = "node", version = garage_version())] Node(NodeOperation), /// Operations on the assignation of node roles in the cluster layout - #[structopt(name = "layout", version = version::garage())] + #[structopt(name = "layout", version = garage_version())] Layout(LayoutOperation), /// Operations on buckets - #[structopt(name = "bucket", version = version::garage())] + #[structopt(name = "bucket", version = garage_version())] Bucket(BucketOperation), /// Operations on S3 access keys - #[structopt(name = "key", version = version::garage())] + #[structopt(name = "key", version = garage_version())] Key(KeyOperation), /// Run migrations from previous Garage version /// (DO NOT USE WITHOUT READING FULL DOCUMENTATION) - #[structopt(name = "migrate", version = version::garage())] + #[structopt(name = "migrate", version = garage_version())] Migrate(MigrateOpt), /// Start repair of node data on remote node - #[structopt(name = "repair", version = version::garage())] + #[structopt(name = "repair", version = garage_version())] Repair(RepairOpt), /// Offline reparation of node data (these repairs must be run offline /// directly on the server node) - #[structopt(name = "offline-repair", version = version::garage())] + #[structopt(name = "offline-repair", version = garage_version())] OfflineRepair(OfflineRepairOpt), /// Gather node statistics - #[structopt(name = "stats", version = version::garage())] + #[structopt(name = "stats", version = garage_version())] Stats(StatsOpt), /// Manage background workers - #[structopt(name = "worker", version = version::garage())] + #[structopt(name = "worker", version = garage_version())] Worker(WorkerOpt), } #[derive(StructOpt, Debug)] pub enum NodeOperation { /// Print identifier (public key) of this Garage node - #[structopt(name = "id", version = version::garage())] + #[structopt(name = "id", version = garage_version())] NodeId(NodeIdOpt), /// Connect to Garage node that is currently isolated from the system - #[structopt(name = "connect", version = version::garage())] + #[structopt(name = "connect", version = garage_version())] Connect(ConnectNodeOpt), } @@ -80,23 +80,23 @@ pub struct ConnectNodeOpt { #[derive(StructOpt, Debug)] pub enum LayoutOperation { /// Assign role to Garage node - #[structopt(name = "assign", version = version::garage())] + #[structopt(name = "assign", version = garage_version())] Assign(AssignRoleOpt), /// Remove role from Garage cluster node - #[structopt(name = "remove", version = version::garage())] + #[structopt(name = "remove", version = garage_version())] Remove(RemoveRoleOpt), /// Show roles currently assigned to nodes and changes staged for commit - #[structopt(name = "show", version = version::garage())] + #[structopt(name = "show", version = garage_version())] Show, /// Apply staged changes to cluster layout - #[structopt(name = "apply", version = version::garage())] + #[structopt(name = "apply", version = garage_version())] Apply(ApplyLayoutOpt), /// Revert staged changes to cluster layout - #[structopt(name = "revert", version = version::garage())] + #[structopt(name = "revert", version = garage_version())] Revert(RevertLayoutOpt), } @@ -151,43 +151,43 @@ pub struct RevertLayoutOpt { #[derive(Serialize, Deserialize, StructOpt, Debug)] pub enum BucketOperation { /// List buckets - #[structopt(name = "list", version = version::garage())] + #[structopt(name = "list", version = garage_version())] List, /// Get bucket info - #[structopt(name = "info", version = version::garage())] + #[structopt(name = "info", version = garage_version())] Info(BucketOpt), /// Create bucket - #[structopt(name = "create", version = version::garage())] + #[structopt(name = "create", version = garage_version())] Create(BucketOpt), /// Delete bucket - #[structopt(name = "delete", version = version::garage())] + #[structopt(name = "delete", version = garage_version())] Delete(DeleteBucketOpt), /// Alias bucket under new name - #[structopt(name = "alias", version = version::garage())] + #[structopt(name = "alias", version = garage_version())] Alias(AliasBucketOpt), /// Remove bucket alias - #[structopt(name = "unalias", version = version::garage())] + #[structopt(name = "unalias", version = garage_version())] Unalias(UnaliasBucketOpt), /// Allow key to read or write to bucket - #[structopt(name = "allow", version = version::garage())] + #[structopt(name = "allow", version = garage_version())] Allow(PermBucketOpt), /// Deny key from reading or writing to bucket - #[structopt(name = "deny", version = version::garage())] + #[structopt(name = "deny", version = garage_version())] Deny(PermBucketOpt), /// Expose as website or not - #[structopt(name = "website", version = version::garage())] + #[structopt(name = "website", version = garage_version())] Website(WebsiteOpt), /// Set the quotas for this bucket - #[structopt(name = "set-quotas", version = version::garage())] + #[structopt(name = "set-quotas", version = garage_version())] SetQuotas(SetQuotasOpt), } @@ -293,35 +293,35 @@ pub struct SetQuotasOpt { #[derive(Serialize, Deserialize, StructOpt, Debug)] pub enum KeyOperation { /// List keys - #[structopt(name = "list", version = version::garage())] + #[structopt(name = "list", version = garage_version())] List, /// Get key info - #[structopt(name = "info", version = version::garage())] + #[structopt(name = "info", version = garage_version())] Info(KeyOpt), /// Create new key - #[structopt(name = "new", version = version::garage())] + #[structopt(name = "new", version = garage_version())] New(KeyNewOpt), /// Rename key - #[structopt(name = "rename", version = version::garage())] + #[structopt(name = "rename", version = garage_version())] Rename(KeyRenameOpt), /// Delete key - #[structopt(name = "delete", version = version::garage())] + #[structopt(name = "delete", version = garage_version())] Delete(KeyDeleteOpt), /// Set permission flags for key - #[structopt(name = "allow", version = version::garage())] + #[structopt(name = "allow", version = garage_version())] Allow(KeyPermOpt), /// Unset permission flags for key - #[structopt(name = "deny", version = version::garage())] + #[structopt(name = "deny", version = garage_version())] Deny(KeyPermOpt), /// Import key - #[structopt(name = "import", version = version::garage())] + #[structopt(name = "import", version = garage_version())] Import(KeyImportOpt), } @@ -393,7 +393,7 @@ pub struct MigrateOpt { #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] pub enum MigrateWhat { /// Migrate buckets and permissions from v0.5.0 - #[structopt(name = "buckets050", version = version::garage())] + #[structopt(name = "buckets050", version = garage_version())] Buckets050, } @@ -414,19 +414,19 @@ pub struct RepairOpt { #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] pub enum RepairWhat { /// Only do a full sync of metadata tables - #[structopt(name = "tables", version = version::garage())] + #[structopt(name = "tables", version = garage_version())] Tables, /// Only repair (resync/rebalance) the set of stored blocks - #[structopt(name = "blocks", version = version::garage())] + #[structopt(name = "blocks", version = garage_version())] Blocks, /// Only redo the propagation of object deletions to the version table (slow) - #[structopt(name = "versions", version = version::garage())] + #[structopt(name = "versions", version = garage_version())] Versions, /// Only redo the propagation of version deletions to the block ref table (extremely slow) - #[structopt(name = "block_refs", version = version::garage())] + #[structopt(name = "block_refs", version = garage_version())] BlockRefs, /// Verify integrity of all blocks on disc (extremely slow, i/o intensive) - #[structopt(name = "scrub", version = version::garage())] + #[structopt(name = "scrub", version = garage_version())] Scrub { #[structopt(subcommand)] cmd: ScrubCmd, @@ -436,19 +436,19 @@ pub enum RepairWhat { #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] pub enum ScrubCmd { /// Start scrub - #[structopt(name = "start", version = version::garage())] + #[structopt(name = "start", version = garage_version())] Start, /// Pause scrub (it will resume automatically after 24 hours) - #[structopt(name = "pause", version = version::garage())] + #[structopt(name = "pause", version = garage_version())] Pause, /// Resume paused scrub - #[structopt(name = "resume", version = version::garage())] + #[structopt(name = "resume", version = garage_version())] Resume, /// Cancel scrub in progress - #[structopt(name = "cancel", version = version::garage())] + #[structopt(name = "cancel", version = garage_version())] Cancel, /// Set tranquility level for in-progress and future scrubs - #[structopt(name = "set-tranquility", version = version::garage())] + #[structopt(name = "set-tranquility", version = garage_version())] SetTranquility { #[structopt()] tranquility: u32, @@ -469,10 +469,10 @@ pub struct OfflineRepairOpt { pub enum OfflineRepairWhat { /// Repair K2V item counters #[cfg(feature = "k2v")] - #[structopt(name = "k2v_item_counters", version = version::garage())] + #[structopt(name = "k2v_item_counters", version = garage_version())] K2VItemCounters, /// Repair object counters - #[structopt(name = "object_counters", version = version::garage())] + #[structopt(name = "object_counters", version = garage_version())] ObjectCounters, } @@ -496,13 +496,13 @@ pub struct WorkerOpt { #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] pub enum WorkerCmd { /// List all workers on Garage node - #[structopt(name = "list", version = version::garage())] + #[structopt(name = "list", version = garage_version())] List { #[structopt(flatten)] opt: WorkerListOpt, }, /// Set worker parameter - #[structopt(name = "set", version = version::garage())] + #[structopt(name = "set", version = garage_version())] Set { #[structopt(subcommand)] opt: WorkerSetCmd, @@ -522,12 +522,12 @@ pub struct WorkerListOpt { #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] pub enum WorkerSetCmd { /// Set tranquility of scrub operations - #[structopt(name = "scrub-tranquility", version = version::garage())] + #[structopt(name = "scrub-tranquility", version = garage_version())] ScrubTranquility { tranquility: u32 }, /// Set number of concurrent block resync workers - #[structopt(name = "resync-n-workers", version = version::garage())] + #[structopt(name = "resync-n-workers", version = garage_version())] ResyncNWorkers { n_workers: usize }, /// Set tranquility of block resync operations - #[structopt(name = "resync-tranquility", version = version::garage())] + #[structopt(name = "resync-tranquility", version = garage_version())] ResyncTranquility { tranquility: u32 }, } diff --git a/src/garage/main.rs b/src/garage/main.rs index 89888884..77d5db24 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -8,8 +8,15 @@ mod admin; mod cli; mod repair; mod server; +#[cfg(feature = "telemetry-otlp")] mod tracing_setup; +#[cfg(not(any(feature = "bundled-libs", feature = "system-libs")))] +compile_error!("Either bundled-libs or system-libs Cargo feature must be enabled"); + +#[cfg(all(feature = "bundled-libs", feature = "system-libs"))] +compile_error!("Only one of bundled-libs and system-libs Cargo features must be enabled"); + use std::net::SocketAddr; use std::path::PathBuf; @@ -22,7 +29,6 @@ use garage_util::error::*; use garage_rpc::system::*; use garage_rpc::*; -use garage_util::version; use garage_model::helper::error::Error as HelperError; @@ -30,7 +36,10 @@ use admin::*; use cli::*; #[derive(StructOpt, Debug)] -#[structopt(name = "garage", version = version::garage(), about = "S3-compatible object store for self-hosted geo-distributed deployments")] +#[structopt( + name = "garage", + about = "S3-compatible object store for self-hosted geo-distributed deployments" +)] struct Opt { /// Host to connect to for admin operations, in the format: /// <public-key>@<ip>:<port> @@ -68,7 +77,40 @@ async fn main() { std::process::abort(); })); - let opt = Opt::from_args(); + // Initialize version and features info + let features = &[ + #[cfg(feature = "k2v")] + "k2v", + #[cfg(feature = "sled")] + "sled", + #[cfg(feature = "lmdb")] + "lmdb", + #[cfg(feature = "sqlite")] + "sqlite", + #[cfg(feature = "kubernetes-discovery")] + "kubernetes-discovery", + #[cfg(feature = "metrics")] + "metrics", + #[cfg(feature = "telemetry-otlp")] + "telemetry-otlp", + #[cfg(feature = "bundled-libs")] + "bundled-libs", + #[cfg(feature = "system-libs")] + "system-libs", + ][..]; + if let Some(git_version) = option_env!("GIT_VERSION") { + garage_util::version::init_version(git_version); + } + garage_util::version::init_features(features); + + // Parse arguments + let version = format!( + "{} [features: {}]", + garage_util::version::garage_version(), + features.join(", ") + ); + let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches()); + let res = match opt.cmd { Command::Server => server::run_server(opt.config_file).await, Command::OfflineRepair(repair_opt) => { diff --git a/src/garage/server.rs b/src/garage/server.rs index 6321357a..aeef02a2 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -9,12 +9,13 @@ use garage_util::error::Error; use garage_api::admin::api_server::AdminApiServer; use garage_api::s3::api_server::S3ApiServer; use garage_model::garage::Garage; -use garage_web::run_web_server; +use garage_web::WebServer; #[cfg(feature = "k2v")] use garage_api::k2v::api_server::K2VApiServer; use crate::admin::*; +#[cfg(feature = "telemetry-otlp")] use crate::tracing_setup::*; async fn wait_from(mut chan: watch::Receiver<bool>) { @@ -29,6 +30,8 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Loading configuration..."); let config = read_config(config_file)?; + // ---- Initialize Garage internals ---- + info!("Initializing background runner..."); let watch_cancel = netapp::util::watch_ctrl_c(); let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone()); @@ -36,9 +39,14 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Initializing Garage main data store..."); let garage = Garage::new(config.clone(), background)?; - info!("Initialize tracing..."); - if let Some(export_to) = config.admin.trace_sink { - init_tracing(&export_to, garage.system.id)?; + if config.admin.trace_sink.is_some() { + info!("Initialize tracing..."); + + #[cfg(feature = "telemetry-otlp")] + init_tracing(config.admin.trace_sink.as_ref().unwrap(), garage.system.id)?; + + #[cfg(not(feature = "telemetry-otlp"))] + error!("Garage was built without OTLP exporter, admin.trace_sink is ignored."); } info!("Initialize Admin API server and metrics collector..."); @@ -50,53 +58,78 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Create admin RPC handler..."); AdminRpcHandler::new(garage.clone()); - info!("Initializing S3 API server..."); - let s3_api_server = tokio::spawn(S3ApiServer::run( - garage.clone(), - wait_from(watch_cancel.clone()), - )); - - #[cfg(feature = "k2v")] - let k2v_api_server = { - info!("Initializing K2V API server..."); - tokio::spawn(K2VApiServer::run( - garage.clone(), - wait_from(watch_cancel.clone()), - )) - }; - - info!("Initializing web server..."); - let web_server = tokio::spawn(run_web_server( - garage.clone(), - wait_from(watch_cancel.clone()), - )); - - info!("Launching Admin API server..."); - let admin_server = tokio::spawn(admin_server.run(wait_from(watch_cancel.clone()))); + // ---- Launch public-facing API servers ---- + + let mut servers = vec![]; + + if let Some(s3_bind_addr) = &config.s3_api.api_bind_addr { + info!("Initializing S3 API server..."); + servers.push(( + "S3 API", + tokio::spawn(S3ApiServer::run( + garage.clone(), + *s3_bind_addr, + config.s3_api.s3_region.clone(), + wait_from(watch_cancel.clone()), + )), + )); + } - // Stuff runs + if config.k2v_api.is_some() { + #[cfg(feature = "k2v")] + { + info!("Initializing K2V API server..."); + servers.push(( + "K2V API", + tokio::spawn(K2VApiServer::run( + garage.clone(), + config.k2v_api.as_ref().unwrap().api_bind_addr, + config.s3_api.s3_region.clone(), + wait_from(watch_cancel.clone()), + )), + )); + } + #[cfg(not(feature = "k2v"))] + error!("K2V is not enabled in this build, cannot start K2V API server"); + } - // When a cancel signal is sent, stuff stops - if let Err(e) = s3_api_server.await? { - warn!("S3 API server exited with error: {}", e); - } else { - info!("S3 API server exited without error."); + if let Some(web_config) = &config.s3_web { + info!("Initializing web server..."); + servers.push(( + "Web", + tokio::spawn(WebServer::run( + garage.clone(), + web_config.bind_addr, + web_config.root_domain.clone(), + wait_from(watch_cancel.clone()), + )), + )); } - #[cfg(feature = "k2v")] - if let Err(e) = k2v_api_server.await? { - warn!("K2V API server exited with error: {}", e); - } else { - info!("K2V API server exited without error."); + + if let Some(admin_bind_addr) = &config.admin.api_bind_addr { + info!("Launching Admin API server..."); + servers.push(( + "Admin", + tokio::spawn(admin_server.run(*admin_bind_addr, wait_from(watch_cancel.clone()))), + )); } - if let Err(e) = web_server.await? { - warn!("Web server exited with error: {}", e); - } else { - info!("Web server exited without error."); + + #[cfg(not(feature = "metrics"))] + if config.admin.metrics_token.is_some() { + warn!("This Garage version is built without the metrics feature"); } - if let Err(e) = admin_server.await? { - warn!("Admin web server exited with error: {}", e); - } else { - info!("Admin API server exited without error."); + + // Stuff runs + + // When a cancel signal is sent, stuff stops + + // Collect stuff + for (desc, join_handle) in servers { + if let Err(e) = join_handle.await? { + error!("{} server exited with error: {}", desc, e); + } else { + info!("{} server exited without error.", desc); + } } // Remove RPC handlers for system to break reference cycles diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs index 0106ad10..d15639b9 100644 --- a/src/garage/tests/lib.rs +++ b/src/garage/tests/lib.rs @@ -3,5 +3,6 @@ mod common; mod admin; mod bucket; +#[cfg(feature = "k2v")] mod k2v; mod s3; |