From f0717dd169d49cef2fb9caa091117f9d7eb152aa Mon Sep 17 00:00:00 2001 From: teutat3s <10206665+teutat3s@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:06:43 +0100 Subject: Bump pnet_datalink 0.28 -> 0.33 Motivation: building garage on illumos is only possible since pnet_datalink version 0.30 Changelog: https://github.com/libpnet/libpnet/compare/v0.28.0...v0.33.0 --- Cargo.lock | 25 +++++++++++++++++-------- src/rpc/Cargo.toml | 3 +-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4fddd215..4d0e1534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1737,9 +1737,9 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "ipnetwork" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4088d739b183546b239688ddbc79891831df421773df95e236daf7867866d355" +checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" dependencies = [ "serde", ] @@ -2213,6 +2213,12 @@ dependencies = [ "tokio-util 0.7.4", ] +[[package]] +name = "no-std-net" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65" + [[package]] name = "nom" version = "7.1.3" @@ -2597,15 +2603,18 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "pnet_base" -version = "0.28.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25488cd551a753dcaaa6fffc9f69a7610a412dd8954425bf7ffad5f7d1156fb8" +checksum = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e" +dependencies = [ + "no-std-net", +] [[package]] name = "pnet_datalink" -version = "0.28.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d1f8ab1ef6c914cf51dc5dfe0be64088ea5f3b08bbf5a31abc70356d271198" +checksum = "c302da22118d2793c312a35fb3da6846cb0fab6c3ad53fd67e37809b06cdafce" dependencies = [ "ipnetwork", "libc", @@ -2616,9 +2625,9 @@ dependencies = [ [[package]] name = "pnet_sys" -version = "0.28.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a3f32b0df45515befd19eed04616f6b56a488da92afc61164ef455e955f07f" +checksum = "faf7a58b2803d818a374be9278a1fe8f88fce14b936afbe225000cfcd9c73f16" dependencies = [ "libc", "winapi", diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 87ae15ac..2c1260a9 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -37,8 +37,7 @@ k8s-openapi = { version = "0.16", features = ["v1_22"], optional = true } schemars = { version = "0.8", optional = true } reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls-manual-roots", "json"] } -# newer version requires rust edition 2021 -pnet_datalink = "0.28" +pnet_datalink = "0.33" futures = "0.3" futures-util = "0.3" -- cgit v1.2.3 From b70cc0a94087bfd70931ff6741299823b48ad291 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 9 Mar 2023 15:34:14 +0000 Subject: block/repair.rs: Added migration for ScrubWorkerPersisted's time_next_run_scrub. Fixes: #520. --- src/block/repair.rs | 54 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/src/block/repair.rs b/src/block/repair.rs index 10d46291..006cc866 100644 --- a/src/block/repair.rs +++ b/src/block/repair.rs @@ -5,7 +5,6 @@ use std::time::Duration; use async_trait::async_trait; use rand::Rng; -use serde::{Deserialize, Serialize}; use tokio::fs; use tokio::select; use tokio::sync::mpsc; @@ -162,6 +161,50 @@ impl Worker for RepairWorker { // and whose parameter (esp. speed) can be controlled at runtime. // ---- ---- ---- +mod v081 { + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize)] + pub struct ScrubWorkerPersisted { + pub tranquility: u32, + pub(crate) time_last_complete_scrub: u64, + pub(crate) corruptions_detected: u64, + } + + impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {} +} + +mod v082 { + use serde::{Deserialize, Serialize}; + + use super::v081; + + #[derive(Serialize, Deserialize)] + pub struct ScrubWorkerPersisted { + pub tranquility: u32, + pub(crate) time_last_complete_scrub: u64, + pub(crate) time_next_run_scrub: u64, + pub(crate) corruptions_detected: u64, + } + + impl garage_util::migrate::Migrate for ScrubWorkerPersisted { + type Previous = v081::ScrubWorkerPersisted; + + fn migrate(old: v081::ScrubWorkerPersisted) -> ScrubWorkerPersisted { + use crate::repair::randomize_next_scrub_run_time; + + ScrubWorkerPersisted { + tranquility: old.tranquility, + time_last_complete_scrub: old.time_last_complete_scrub, + time_next_run_scrub: randomize_next_scrub_run_time(), + corruptions_detected: old.corruptions_detected, + } + } + } +} + +pub use v082::*; + pub struct ScrubWorker { manager: Arc, rx_cmd: mpsc::Receiver, @@ -172,14 +215,6 @@ pub struct ScrubWorker { persister: PersisterShared, } -#[derive(Serialize, Deserialize)] -pub struct ScrubWorkerPersisted { - pub tranquility: u32, - pub(crate) time_last_complete_scrub: u64, - pub(crate) time_next_run_scrub: u64, - pub(crate) corruptions_detected: u64, -} - fn randomize_next_scrub_run_time() -> u64 { // Take SCRUB_INTERVAL and mix in a random interval of 10 days to attempt to // balance scrub load across different cluster nodes. @@ -194,7 +229,6 @@ fn randomize_next_scrub_run_time() -> u64 { next_run_timestamp } -impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {} impl Default for ScrubWorkerPersisted { fn default() -> Self { ScrubWorkerPersisted { -- cgit v1.2.3 From 7b65dd24e2bacc20afa747dc2b1f3fb81249f688 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 9 Mar 2023 16:32:22 +0000 Subject: block/repair.rs: Added a timestamp argument to randomize_next_scrub_run_time(). --- src/block/repair.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/block/repair.rs b/src/block/repair.rs index 006cc866..5476bf8a 100644 --- a/src/block/repair.rs +++ b/src/block/repair.rs @@ -196,7 +196,7 @@ mod v082 { ScrubWorkerPersisted { tranquility: old.tranquility, time_last_complete_scrub: old.time_last_complete_scrub, - time_next_run_scrub: randomize_next_scrub_run_time(), + time_next_run_scrub: randomize_next_scrub_run_time(old.time_last_complete_scrub), corruptions_detected: old.corruptions_detected, } } @@ -215,11 +215,11 @@ pub struct ScrubWorker { persister: PersisterShared, } -fn randomize_next_scrub_run_time() -> u64 { +fn randomize_next_scrub_run_time(timestamp: u64) -> u64 { // Take SCRUB_INTERVAL and mix in a random interval of 10 days to attempt to // balance scrub load across different cluster nodes. - let next_run_timestamp = now_msec() + let next_run_timestamp = timestamp + SCRUB_INTERVAL .saturating_add(Duration::from_secs( rand::thread_rng().gen_range(0..3600 * 24 * 10), @@ -233,7 +233,7 @@ impl Default for ScrubWorkerPersisted { fn default() -> Self { ScrubWorkerPersisted { time_last_complete_scrub: 0, - time_next_run_scrub: randomize_next_scrub_run_time(), + time_next_run_scrub: randomize_next_scrub_run_time(now_msec()), tranquility: INITIAL_SCRUB_TRANQUILITY, corruptions_detected: 0, } @@ -395,7 +395,7 @@ impl Worker for ScrubWorker { } else { self.persister.set_with(|p| { p.time_last_complete_scrub = now_msec(); - p.time_next_run_scrub = randomize_next_scrub_run_time(); + p.time_next_run_scrub = randomize_next_scrub_run_time(now_msec()); })?; self.work = ScrubWorkerState::Finished; self.tranquilizer.clear(); -- cgit v1.2.3 From d218f475cbe6450f0402424e01b99c4630cb8a2f Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 9 Mar 2023 17:08:47 +0000 Subject: block/manager.rs: Set defaults for scrub_persister. --- src/block/manager.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/block/manager.rs b/src/block/manager.rs index bfa66069..26278974 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -152,6 +152,7 @@ impl BlockManager { tx_scrub_command: ArcSwapOption::new(None), }); block_manager.endpoint.set_handler(block_manager.clone()); + block_manager.scrub_persister.set_with(|_| ()).unwrap(); block_manager } -- cgit v1.2.3 From 3b498c7c4725ab83edd5fdeba65b32ae6bdc3e66 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 2 Mar 2023 16:20:45 +0100 Subject: update cargo.nix --- Cargo.nix | 54 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 642735cf..69274c07 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -32,7 +32,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "9941a539c374cfd67691e633af6c6a5da9e8badbb7b14159b3ff24a7cf5ddf8a"; + nixifiedLockHash = "12ff12e9210c87ac93e2f2bdb7007b6232e7efcfa302c272bf0caee6e78abd10"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -1783,7 +1783,7 @@ in sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; - pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; }).out; + pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.33.0" { inherit profileName; }).out; rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.14" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.11" { inherit profileName; }).out; @@ -2420,11 +2420,11 @@ in ]; }); - "registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.18.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.20.0" = overridableMkRustCrate (profileName: rec { name = "ipnetwork"; - version = "0.18.0"; + version = "0.20.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "4088d739b183546b239688ddbc79891831df421773df95e236daf7867866d355"; }; + src = fetchCratesIo { inherit name version; sha256 = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e"; }; features = builtins.concatLists [ [ "default" ] [ "serde" ] @@ -3102,6 +3102,16 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".no-std-net."0.6.0" = overridableMkRustCrate (profileName: rec { + name = "no-std-net"; + version = "0.6.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".nom."7.1.3" = overridableMkRustCrate (profileName: rec { name = "nom"; version = "7.1.3"; @@ -3636,32 +3646,42 @@ in src = fetchCratesIo { inherit name version; sha256 = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"; }; }); - "registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.28.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.33.0" = overridableMkRustCrate (profileName: rec { name = "pnet_base"; - version = "0.28.0"; + version = "0.33.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "25488cd551a753dcaaa6fffc9f69a7610a412dd8954425bf7ffad5f7d1156fb8"; }; + src = fetchCratesIo { inherit name version; sha256 = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + dependencies = { + no_std_net = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".no-std-net."0.6.0" { inherit profileName; }).out; + }; }); - "registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.33.0" = overridableMkRustCrate (profileName: rec { name = "pnet_datalink"; - version = "0.28.0"; + version = "0.33.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d4d1f8ab1ef6c914cf51dc5dfe0be64088ea5f3b08bbf5a31abc70356d271198"; }; + src = fetchCratesIo { inherit name version; sha256 = "c302da22118d2793c312a35fb3da6846cb0fab6c3ad53fd67e37809b06cdafce"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; dependencies = { - ipnetwork = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.18.0" { inherit profileName; }).out; + ipnetwork = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.20.0" { inherit profileName; }).out; libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; - pnet_base = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.28.0" { inherit profileName; }).out; - pnet_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.28.0" { inherit profileName; }).out; + pnet_base = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.33.0" { inherit profileName; }).out; + pnet_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.33.0" { inherit profileName; }).out; ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; }; }); - "registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.28.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.33.0" = overridableMkRustCrate (profileName: rec { name = "pnet_sys"; - version = "0.28.0"; + version = "0.33.0"; registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "d9a3f32b0df45515befd19eed04616f6b56a488da92afc61164ef455e955f07f"; }; + src = fetchCratesIo { inherit name version; sha256 = "faf7a58b2803d818a374be9278a1fe8f88fce14b936afbe225000cfcd9c73f16"; }; dependencies = { libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; -- cgit v1.2.3 From dc6be3983300ee3ee10f601cfc04a06563d86f48 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Sun, 12 Mar 2023 15:17:11 +0000 Subject: doc: cli.md: Added s5cmd example. --- doc/book/connect/cli.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/book/connect/cli.md b/doc/book/connect/cli.md index 74e2d7ed..9b74bb5d 100644 --- a/doc/book/connect/cli.md +++ b/doc/book/connect/cli.md @@ -12,6 +12,7 @@ These tools are particularly suitable for debug, backups, website deployments or | [AWS CLI](#aws-cli) | ✅ | Recommended | | [rclone](#rclone) | ✅ | | | [s3cmd](#s3cmd) | ✅ | | +| [s5cmd](#s5cmd) | ✅ | | | [(Cyber)duck](#cyberduck) | ✅ | | | [WinSCP (libs3)](#winscp) | ✅ | CLI instructions only | | [sftpgo](#sftpgo) | ✅ | | @@ -178,6 +179,26 @@ s3cmd put /tmp/hello.txt s3://my-bucket/ s3cmd get s3://my-bucket/hello.txt hello.txt ``` +## `s5cmd` + +Configure a credentials file as follows: + +```bash +export AWS_ACCESS_KEY_ID=GK... +export AWS_SECRET_ACCESS_KEY= +export AWS_DEFAULT_REGION='garage' +export AWS_ENDPOINT='http://localhost:3900' +``` + +After adding these environment variables in your shell, `s5cmd` can be used +with: + +```bash +s5cmd --endpoint-url=$AWS_ENDPOINT ls +``` + +See its usage output for other commands available. + ## Cyberduck & duck {#cyberduck} Both Cyberduck (the GUI) and duck (the CLI) have a concept of "Connection Profiles" that contain some presets for a specific provider. -- cgit v1.2.3