aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xscript/dev-bucket.sh6
-rwxr-xr-xscript/dev-cluster.sh27
-rwxr-xr-xscript/dev-configure.sh6
-rwxr-xr-xscript/test-smoke.sh6
-rw-r--r--src/garage/cli.rs35
-rw-r--r--src/garage/main.rs117
-rw-r--r--src/rpc/rpc_helper.rs2
-rw-r--r--src/rpc/system.rs59
-rw-r--r--src/table/replication/fullcopy.rs6
-rw-r--r--src/util/config.rs7
10 files changed, 192 insertions, 79 deletions
diff --git a/script/dev-bucket.sh b/script/dev-bucket.sh
index 05d8c105..438d934d 100755
--- a/script/dev-bucket.sh
+++ b/script/dev-bucket.sh
@@ -9,11 +9,11 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
-garage bucket create eprouvette
-KEY_INFO=`garage key new --name opérateur`
+garage -c /tmp/config.1.toml bucket create eprouvette
+KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur)
ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
-garage bucket allow eprouvette --read --write --key $ACCESS_KEY
+garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --key $ACCESS_KEY
echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."
diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh
index 0afdd97c..d56fa6e3 100755
--- a/script/dev-cluster.sh
+++ b/script/dev-cluster.sh
@@ -17,6 +17,10 @@ MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
WHICH_GARAGE=$(which garage || exit 1)
echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n"
+NETWORK_SECRET="$(openssl rand -hex 32)"
+
+
+# <<<<<<<<< BEGIN FOR LOOP ON NODES
for count in $(seq 1 3); do
CONF_PATH="/tmp/config.$count.toml"
LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
@@ -26,13 +30,11 @@ block_size = 1048576 # objects are split in blocks of maximum this number of b
metadata_dir = "/tmp/garage-meta-$count"
data_dir = "/tmp/garage-data-$count"
rpc_bind_addr = "0.0.0.0:$((3900+$count))" # the port other Garage nodes will use to talk to this node
-bootstrap_peers = [
- "127.0.0.1:3901",
- "127.0.0.1:3902",
- "127.0.0.1:3903"
-]
+rpc_public_addr = "127.0.0.1:$((3900+$count))"
+bootstrap_peers = []
max_concurrent_rpc_requests = 12
replication_mode = "3"
+rpc_secret = "$NETWORK_SECRET"
[s3_api]
api_bind_addr = "0.0.0.0:$((3910+$count))" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
@@ -61,11 +63,21 @@ if [ -z "$SKIP_HTTPS" ]; then
socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
fi
-(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
+(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
+done
+# >>>>>>>>>>>>>>>> END FOR LOOP ON NODES
+
+sleep 5
+# Establish connections between nodes
+for count in $(seq 1 3); do
+ NODE=$(garage -c /tmp/config.$count.toml node-id -q)
+ for count2 in $(seq 1 3); do
+ garage -c /tmp/config.$count2.toml node connect $NODE
+ done
done
RETRY=120
-until garage status 2>&1|grep -q Healthy ; do
+until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
(( RETRY-- ))
if (( RETRY <= 0 )); then
echo -en "${MAIN_LABEL} Garage did not start"
@@ -74,6 +86,7 @@ until garage status 2>&1|grep -q Healthy ; do
echo -en "${MAIN_LABEL} cluster starting...\n"
sleep 1
done
+
echo -en "${MAIN_LABEL} cluster started\n"
wait
diff --git a/script/dev-configure.sh b/script/dev-configure.sh
index fdae959b..72fb67ea 100755
--- a/script/dev-configure.sh
+++ b/script/dev-configure.sh
@@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
sleep 5
RETRY=120
-until garage status 2>&1|grep -q Healthy ; do
+until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
(( RETRY-- ))
if (( RETRY <= 0 )); then
echo "garage did not start in time, failing."
@@ -21,10 +21,10 @@ until garage status 2>&1|grep -q Healthy ; do
sleep 1
done
-garage status \
+garage -c /tmp/config.1.toml status \
| grep UNCONFIGURED \
| grep -Po '^[0-9a-f]+' \
| while read id; do
- garage node configure -z dc1 -c 1 $id
+ garage -c /tmp/config.1.toml node configure -z dc1 -c 1 $id
done
diff --git a/script/test-smoke.sh b/script/test-smoke.sh
index 335d55e9..ce9c032a 100755
--- a/script/test-smoke.sh
+++ b/script/test-smoke.sh
@@ -21,9 +21,9 @@ ${SCRIPT_FOLDER}/dev-configure.sh
${SCRIPT_FOLDER}/dev-bucket.sh
which garage
-garage status
-garage key list
-garage bucket list
+garage -c /tmp/config.1.toml status
+garage -c /tmp/config.1.toml key list
+garage -c /tmp/config.1.toml bucket list
dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline storage (< INLINE_THRESHOLD = 3072 bytes)
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
diff --git a/src/garage/cli.rs b/src/garage/cli.rs
index 940a5a85..67606b97 100644
--- a/src/garage/cli.rs
+++ b/src/garage/cli.rs
@@ -1,5 +1,4 @@
use std::collections::HashSet;
-use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use structopt::StructOpt;
@@ -21,7 +20,12 @@ use crate::admin_rpc::*;
pub enum Command {
/// Run Garage server
#[structopt(name = "server")]
- Server(ServerOpt),
+ Server,
+
+ /// Print identifier (public key) of this garage node.
+ /// Generates a new keypair if necessary.
+ #[structopt(name = "node-id")]
+ NodeId(NodeIdOpt),
/// Get network status
#[structopt(name = "status")]
@@ -49,13 +53,6 @@ pub enum Command {
}
#[derive(StructOpt, Debug)]
-pub struct ServerOpt {
- /// Configuration file
- #[structopt(short = "c", long = "config", default_value = "./config.toml")]
- pub config_file: PathBuf,
-}
-
-#[derive(StructOpt, Debug)]
pub enum NodeOperation {
/// Connect to Garage node that is currently isolated from the system
#[structopt(name = "connect")]
@@ -71,6 +68,13 @@ pub enum NodeOperation {
}
#[derive(StructOpt, Debug)]
+pub struct NodeIdOpt {
+ /// Do not print usage instructions to stderr
+ #[structopt(short = "q", long = "quiet")]
+ pub(crate) quiet: bool,
+}
+
+#[derive(StructOpt, Debug)]
pub struct ConnectNodeOpt {
/// Node public key and address, in the format:
/// `<public key hexadecimal>@<ip or hostname>:<port>`
@@ -384,7 +388,8 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
.any(|(id, _)| !status_keys.contains(id));
if failure_case_1 || failure_case_2 {
println!("\nFailed nodes:");
- let mut failed_nodes = vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()];
+ let mut failed_nodes =
+ vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()];
for adv in status.iter().filter(|adv| !adv.is_up) {
if let Some(cfg) = config.members.get(&adv.id) {
failed_nodes.push(format!(
@@ -421,14 +426,15 @@ pub async fn cmd_connect(
rpc_host: NodeID,
args: ConnectNodeOpt,
) -> Result<(), Error> {
- match rpc_cli.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL).await?? {
+ match rpc_cli
+ .call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL)
+ .await??
+ {
SystemRpc::Ok => {
println!("Success.");
Ok(())
}
- r => {
- Err(Error::BadRpc(format!("Unexpected response: {:?}", r)))
- }
+ r => Err(Error::BadRpc(format!("Unexpected response: {:?}", r))),
}
}
@@ -654,4 +660,3 @@ pub fn find_matching_node(
Ok(candidates[0])
}
}
-
diff --git a/src/garage/main.rs b/src/garage/main.rs
index 543860ca..f7ef19cd 100644
--- a/src/garage/main.rs
+++ b/src/garage/main.rs
@@ -9,9 +9,11 @@ mod cli;
mod repair;
mod server;
+use std::path::PathBuf;
+
use structopt::StructOpt;
-use netapp::util::parse_peer_addr;
+use netapp::util::parse_and_resolve_peer_addr;
use netapp::NetworkKey;
use garage_util::error::Error;
@@ -34,6 +36,10 @@ struct Opt {
#[structopt(short = "s", long = "rpc-secret")]
pub rpc_secret: Option<String>,
+ /// Configuration file (garage.toml)
+ #[structopt(short = "c", long = "config", default_value = "/etc/garage.toml")]
+ pub config_file: PathBuf,
+
#[structopt(subcommand)]
cmd: Command,
}
@@ -45,16 +51,18 @@ async fn main() {
let opt = Opt::from_args();
- let res = if let Command::Server(server_opt) = opt.cmd {
- // Abort on panic (same behavior as in Go)
- std::panic::set_hook(Box::new(|panic_info| {
- error!("{}", panic_info.to_string());
- std::process::abort();
- }));
-
- server::run_server(server_opt.config_file).await
- } else {
- cli_command(opt).await
+ let res = match opt.cmd {
+ Command::Server => {
+ // Abort on panic (same behavior as in Go)
+ std::panic::set_hook(Box::new(|panic_info| {
+ error!("{}", panic_info.to_string());
+ std::process::abort();
+ }));
+
+ server::run_server(opt.config_file).await
+ }
+ Command::NodeId(node_id_opt) => node_id_command(opt.config_file, node_id_opt.quiet),
+ _ => cli_command(opt).await,
};
if let Err(e) = res {
@@ -63,16 +71,42 @@ async fn main() {
}
async fn cli_command(opt: Opt) -> Result<(), Error> {
- let net_key_hex_str = &opt.rpc_secret.expect("No RPC secret provided");
+ let config = if opt.rpc_secret.is_none() || opt.rpc_host.is_none() {
+ Some(garage_util::config::read_config(opt.config_file.clone())
+ .map_err(|e| Error::Message(format!("Unable to read configuration file {}: {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy(), e)))?)
+ } else {
+ None
+ };
+
+ // Find and parse network RPC secret
+ let net_key_hex_str = opt
+ .rpc_secret
+ .as_ref()
+ .or_else(|| config.as_ref().map(|c| &c.rpc_secret))
+ .expect("No RPC secret provided");
let network_key = NetworkKey::from_slice(
&hex::decode(net_key_hex_str).expect("Invalid RPC secret key (bad hex)")[..],
)
.expect("Invalid RPC secret provided (wrong length)");
+
+ // Generate a temporary keypair for our RPC client
let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair();
let netapp = NetApp::new(network_key, sk);
- let (id, addr) =
- parse_peer_addr(&opt.rpc_host.expect("No RPC host provided")).expect("Invalid RPC host");
+
+ // Find and parse the address of the target host
+ let (id, addr) = if let Some(h) = opt.rpc_host {
+ let (id, addrs) = parse_and_resolve_peer_addr(&h).expect("Invalid RPC host");
+ (id, addrs[0])
+ } else if let Some(a) = config.as_ref().map(|c| c.rpc_public_addr).flatten() {
+ let node_key = garage_rpc::system::gen_node_key(&config.unwrap().metadata_dir)
+ .map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
+ (node_key.public_key(), a)
+ } else {
+ return Err(Error::Message("No RPC host provided".into()));
+ };
+
+ // Connect to target host
netapp.clone().try_connect(addr, id).await?;
let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
@@ -80,3 +114,58 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
cli_cmd(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await
}
+
+fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
+ let config = garage_util::config::read_config(config_file.clone()).map_err(|e| {
+ Error::Message(format!(
+ "Unable to read configuration file {}: {}",
+ config_file.to_string_lossy(),
+ e
+ ))
+ })?;
+
+ let node_key = garage_rpc::system::gen_node_key(&config.metadata_dir)
+ .map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
+
+ let idstr = if let Some(addr) = config.rpc_public_addr {
+ let idstr = format!("{}@{}", hex::encode(&node_key.public_key()), addr);
+ println!("{}", idstr);
+ idstr
+ } else {
+ let idstr = hex::encode(&node_key.public_key());
+ println!("{}", idstr);
+
+ if !quiet {
+ eprintln!("WARNING: I don't know the public address to reach this node.");
+ eprintln!("In all of the instructions below, replace 127.0.0.1:3901 by the appropriate address and port.");
+ }
+
+ format!("{}@127.0.0.1:3901", idstr)
+ };
+
+ if !quiet {
+ eprintln!("");
+ eprintln!(
+ "To instruct a node to connect to this node, run the following command on that node:"
+ );
+ eprintln!(" garage [-c <config file path>] node connect {}", idstr);
+ eprintln!("");
+ eprintln!("Or instruct them to connect from here by running:");
+ eprintln!(
+ " garage -c {} -h <remote node> node connect {}",
+ config_file.to_string_lossy(),
+ idstr
+ );
+ eprintln!(
+ "where <remote_node> is their own node identifier in the format: <pubkey>@<ip>:<port>"
+ );
+ eprintln!("");
+ eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:");
+ eprintln!(" bootstrap_peers = [");
+ eprintln!(" \"{}\",", idstr);
+ eprintln!(" ...");
+ eprintln!(" ]");
+ }
+
+ Ok(())
+}
diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs
index 9f735ab4..90b3cf15 100644
--- a/src/rpc/rpc_helper.rs
+++ b/src/rpc/rpc_helper.rs
@@ -14,8 +14,8 @@ pub use netapp::proto::*;
pub use netapp::{NetApp, NodeID};
use garage_util::background::BackgroundRunner;
-use garage_util::error::Error;
use garage_util::data::Uuid;
+use garage_util::error::Error;
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/src/rpc/system.rs b/src/rpc/system.rs
index 886811b1..51abede3 100644
--- a/src/rpc/system.rs
+++ b/src/rpc/system.rs
@@ -18,8 +18,8 @@ use tokio::sync::Mutex;
use netapp::endpoint::{Endpoint, EndpointHandler};
use netapp::peering::fullmesh::FullMeshPeeringStrategy;
use netapp::proto::*;
-use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
use netapp::util::parse_and_resolve_peer_addr;
+use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
use garage_util::background::BackgroundRunner;
use garage_util::data::Uuid;
@@ -110,7 +110,7 @@ pub struct KnownNodeInfo {
pub status: NodeStatus,
}
-fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
+pub fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
let mut key_file = metadata_dir.to_path_buf();
key_file.push("node_key");
if key_file.as_path().exists() {
@@ -246,8 +246,12 @@ impl System {
}
async fn handle_connect(&self, node: &str) -> Result<SystemRpc, Error> {
- let (pubkey, addrs) = parse_and_resolve_peer_addr(node)
- .ok_or_else(|| Error::Message(format!("Unable to parse or resolve node specification: {}", node)))?;
+ let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| {
+ Error::Message(format!(
+ "Unable to parse or resolve node specification: {}",
+ node
+ ))
+ })?;
let mut errors = vec![];
for ip in addrs.iter() {
match self.netapp.clone().try_connect(*ip, pubkey).await {
@@ -257,7 +261,10 @@ impl System {
}
}
}
- return Err(Error::Message(format!("Could not connect to specified peers. Errors: {:?}", errors)));
+ return Err(Error::Message(format!(
+ "Could not connect to specified peers. Errors: {:?}",
+ errors
+ )));
}
fn handle_pull_config(&self) -> SystemRpc {
@@ -267,23 +274,25 @@ impl System {
fn handle_get_known_nodes(&self) -> SystemRpc {
let node_status = self.node_status.read().unwrap();
- let known_nodes =
- self.fullmesh
- .get_peer_list()
- .iter()
- .map(|n| KnownNodeInfo {
- id: n.id.into(),
- addr: n.addr,
- is_up: n.is_up(),
- status: node_status.get(&n.id.into()).cloned().map(|(_, st)| st).unwrap_or(
- NodeStatus {
- hostname: "?".to_string(),
- replication_factor: 0,
- config_version: 0,
- },
- ),
- })
- .collect::<Vec<_>>();
+ let known_nodes = self
+ .fullmesh
+ .get_peer_list()
+ .iter()
+ .map(|n| KnownNodeInfo {
+ id: n.id.into(),
+ addr: n.addr,
+ is_up: n.is_up(),
+ status: node_status
+ .get(&n.id.into())
+ .cloned()
+ .map(|(_, st)| st)
+ .unwrap_or(NodeStatus {
+ hostname: "?".to_string(),
+ replication_factor: 0,
+ config_version: 0,
+ }),
+ })
+ .collect::<Vec<_>>();
SystemRpc::ReturnKnownNodes(known_nodes)
}
@@ -330,14 +339,14 @@ impl System {
drop(update_ring);
let self2 = self.clone();
- let adv2 = adv.clone();
+ let adv = adv.clone();
self.background.spawn_cancellable(async move {
self2
.rpc
.broadcast(
&self2.system_endpoint,
- SystemRpc::AdvertiseConfig(adv2),
- RequestStrategy::with_priority(PRIO_NORMAL),
+ SystemRpc::AdvertiseConfig(adv),
+ RequestStrategy::with_priority(PRIO_HIGH),
)
.await;
Ok(())
diff --git a/src/table/replication/fullcopy.rs b/src/table/replication/fullcopy.rs
index ae6851fb..8f01fbdd 100644
--- a/src/table/replication/fullcopy.rs
+++ b/src/table/replication/fullcopy.rs
@@ -28,11 +28,7 @@ impl TableReplication for TableFullReplication {
fn write_nodes(&self, _hash: &Hash) -> Vec<Uuid> {
let ring = self.system.ring.borrow();
- ring.config
- .members
- .keys()
- .cloned()
- .collect::<Vec<_>>()
+ ring.config.members.keys().cloned().collect::<Vec<_>>()
}
fn write_quorum(&self) -> usize {
let nmembers = self.system.ring.borrow().config.members.len();
diff --git a/src/util/config.rs b/src/util/config.rs
index fe0a6fa8..95c5cfc0 100644
--- a/src/util/config.rs
+++ b/src/util/config.rs
@@ -6,8 +6,8 @@ use std::path::PathBuf;
use serde::de::Error as SerdeError;
use serde::{de, Deserialize};
-use netapp::NodeID;
use netapp::util::parse_and_resolve_peer_addr;
+use netapp::NodeID;
use crate::error::Error;
@@ -117,8 +117,9 @@ where
let mut ret = vec![];
for peer in <Vec<&str>>::deserialize(deserializer)? {
- let (pubkey, addrs) = parse_and_resolve_peer_addr(peer)
- .ok_or_else(|| D::Error::custom(format!("Unable to parse or resolve peer: {}", peer)))?;
+ let (pubkey, addrs) = parse_and_resolve_peer_addr(peer).ok_or_else(|| {
+ D::Error::custom(format!("Unable to parse or resolve peer: {}", peer))
+ })?;
for ip in addrs {
ret.push((pubkey.clone(), ip));
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-10 17:40:14 +0000