Merge pull request 'Garage v0.9' (#473) from next into main

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/473

4 files changed, 99 insertions, 43 deletions

diff --git a/src/garage/admin/block.rs b/src/garage/admin/block.rs
index e9e3ff96..c4a45738 100644
--- a/src/garage/admin/block.rs
+++ b/src/garage/admin/block.rs
@@ -34,6 +34,7 @@ impl AdminRpcHandler {
 			.get_range(&hash, None, None, 10000, Default::default())
 			.await?;
 		let mut versions = vec![];
+		let mut uploads = vec![];
 		for br in block_refs {
 			if let Some(v) = self
 				.garage
@@ -41,6 +42,11 @@ impl AdminRpcHandler {
 				.get(&br.version, &EmptyKey)
 				.await?
 			{
+				if let VersionBacklink::MultipartUpload { upload_id } = &v.backlink {
+					if let Some(u) = self.garage.mpu_table.get(upload_id, &EmptyKey).await? {
+						uploads.push(u);
+					}
+				}
 				versions.push(Ok(v));
 			} else {
 				versions.push(Err(br.version));
@@ -50,6 +56,7 @@ impl AdminRpcHandler {
 			hash,
 			refcount,
 			versions,
+			uploads,
 		})
 	}
 
@@ -93,6 +100,7 @@ impl AdminRpcHandler {
 		}
 
 		let mut obj_dels = 0;
+		let mut mpu_dels = 0;
 		let mut ver_dels = 0;
 
 		for hash in blocks {
@@ -105,56 +113,80 @@ impl AdminRpcHandler {
 				.await?;
 
 			for br in block_refs {
-				let version = match self
+				if let Some(version) = self
 					.garage
 					.version_table
 					.get(&br.version, &EmptyKey)
 					.await?
 				{
-					Some(v) => v,
-					None => continue,
-				};
+					self.handle_block_purge_version_backlink(
+						&version,
+						&mut obj_dels,
+						&mut mpu_dels,
+					)
+					.await?;
 
-				if let Some(object) = self
-					.garage
-					.object_table
-					.get(&version.bucket_id, &version.key)
-					.await?
-				{
-					let ov = object.versions().iter().rev().find(|v| v.is_complete());
-					if let Some(ov) = ov {
-						if ov.uuid == br.version {
-							let del_uuid = gen_uuid();
-							let deleted_object = Object::new(
-								version.bucket_id,
-								version.key.clone(),
-								vec![ObjectVersion {
-									uuid: del_uuid,
-									timestamp: ov.timestamp + 1,
-									state: ObjectVersionState::Complete(
-										ObjectVersionData::DeleteMarker,
-									),
-								}],
-							);
-							self.garage.object_table.insert(&deleted_object).await?;
-							obj_dels += 1;
-						}
+					if !version.deleted.get() {
+						let deleted_version = Version::new(version.uuid, version.backlink, true);
+						self.garage.version_table.insert(&deleted_version).await?;
+						ver_dels += 1;
 					}
 				}
-
-				if !version.deleted.get() {
-					let deleted_version =
-						Version::new(version.uuid, version.bucket_id, version.key.clone(), true);
-					self.garage.version_table.insert(&deleted_version).await?;
-					ver_dels += 1;
-				}
 			}
 		}
+
 		Ok(AdminRpc::Ok(format!(
-			"{} blocks were purged: {} object deletion markers added, {} versions marked deleted",
+			"Purged {} blocks, {} versions, {} objects, {} multipart uploads",
 			blocks.len(),
+			ver_dels,
 			obj_dels,
-			ver_dels
+			mpu_dels,
 		)))
 	}
+
+	async fn handle_block_purge_version_backlink(
+		&self,
+		version: &Version,
+		obj_dels: &mut usize,
+		mpu_dels: &mut usize,
+	) -> Result<(), Error> {
+		let (bucket_id, key, ov_id) = match &version.backlink {
+			VersionBacklink::Object { bucket_id, key } => (*bucket_id, key.clone(), version.uuid),
+			VersionBacklink::MultipartUpload { upload_id } => {
+				if let Some(mut mpu) = self.garage.mpu_table.get(upload_id, &EmptyKey).await? {
+					if !mpu.deleted.get() {
+						mpu.parts.clear();
+						mpu.deleted.set();
+						self.garage.mpu_table.insert(&mpu).await?;
+						*mpu_dels += 1;
+					}
+					(mpu.bucket_id, mpu.key.clone(), *upload_id)
+				} else {
+					return Ok(());
+				}
+			}
+		};
+
+		if let Some(object) = self.garage.object_table.get(&bucket_id, &key).await? {
+			let ov = object.versions().iter().rev().find(|v| v.is_complete());
+			if let Some(ov) = ov {
+				if ov.uuid == ov_id {
+					let del_uuid = gen_uuid();
+					let deleted_object = Object::new(
+						bucket_id,
+						key,
+						vec![ObjectVersion {
+							uuid: del_uuid,
+							timestamp: ov.timestamp + 1,
+							state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker),
+						}],
+					);
+					self.garage.object_table.insert(&deleted_object).await?;
+					*obj_dels += 1;
+				}
+			}
+		}
+
+		Ok(())
+	}
 }
diff --git a/src/garage/admin/bucket.rs b/src/garage/admin/bucket.rs
index 11bb8730..0781cb8b 100644
--- a/src/garage/admin/bucket.rs
+++ b/src/garage/admin/bucket.rs
@@ -73,6 +73,15 @@ impl AdminRpcHandler {
 			.map(|x| x.filtered_values(&self.garage.system.ring.borrow()))
 			.unwrap_or_default();
 
+		let mpu_counters = self
+			.garage
+			.mpu_counter_table
+			.table
+			.get(&bucket_id, &EmptyKey)
+			.await?
+			.map(|x| x.filtered_values(&self.garage.system.ring.borrow()))
+			.unwrap_or_default();
+
 		let mut relevant_keys = HashMap::new();
 		for (k, _) in bucket
 			.state
@@ -112,6 +121,7 @@ impl AdminRpcHandler {
 			bucket,
 			relevant_keys,
 			counters,
+			mpu_counters,
 		})
 	}
 
diff --git a/src/garage/admin/key.rs b/src/garage/admin/key.rs
index cab13bcf..1c92670c 100644
--- a/src/garage/admin/key.rs
+++ b/src/garage/admin/key.rs
@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use garage_table::*;
 
-use garage_model::helper::error::Error;
+use garage_model::helper::error::*;
 use garage_model::key_table::*;
 
 use crate::cli::*;
@@ -14,7 +14,7 @@ impl AdminRpcHandler {
 		match cmd {
 			KeyOperation::List => self.handle_list_keys().await,
 			KeyOperation::Info(query) => self.handle_key_info(query).await,
-			KeyOperation::New(query) => self.handle_create_key(query).await,
+			KeyOperation::Create(query) => self.handle_create_key(query).await,
 			KeyOperation::Rename(query) => self.handle_rename_key(query).await,
 			KeyOperation::Delete(query) => self.handle_delete_key(query).await,
 			KeyOperation::Allow(query) => self.handle_allow_key(query).await,
@@ -41,12 +41,17 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::KeyList(key_ids))
 	}
 
-	async fn handle_key_info(&self, query: &KeyOpt) -> Result<AdminRpc, Error> {
-		let key = self
+	async fn handle_key_info(&self, query: &KeyInfoOpt) -> Result<AdminRpc, Error> {
+		let mut key = self
 			.garage
 			.key_helper()
 			.get_existing_matching_key(&query.key_pattern)
 			.await?;
+
+		if !query.show_secret {
+			key.state.as_option_mut().unwrap().secret_key = "(redacted)".into();
+		}
+
 		self.key_info_result(key).await
 	}
 
@@ -118,11 +123,17 @@ impl AdminRpcHandler {
 	}
 
 	async fn handle_import_key(&self, query: &KeyImportOpt) -> Result<AdminRpc, Error> {
+		if !query.yes {
+			return Err(Error::BadRequest("This command is intended to re-import keys that were previously generated by Garage. If you want to create a new key, use `garage key new` instead. Add the --yes flag if you really want to re-import a key.".to_string()));
+		}
+
 		let prev_key = self.garage.key_table.get(&EmptyKey, &query.key_id).await?;
 		if prev_key.is_some() {
 			return Err(Error::BadRequest(format!("Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.", query.key_id)));
 		}
-		let imported_key = Key::import(&query.key_id, &query.secret_key, &query.name);
+
+		let imported_key = Key::import(&query.key_id, &query.secret_key, &query.name)
+			.ok_or_bad_request("Invalid key format")?;
 		self.garage.key_table.insert(&imported_key).await?;
 
 		self.key_info_result(imported_key).await
diff --git a/src/garage/admin/mod.rs b/src/garage/admin/mod.rs
index 2709f08a..b6f9c426 100644
--- a/src/garage/admin/mod.rs
+++ b/src/garage/admin/mod.rs
@@ -28,6 +28,7 @@ use garage_model::garage::Garage;
 use garage_model::helper::error::{Error, OkOrBadRequest};
 use garage_model::key_table::*;
 use garage_model::migrate::Migrate;
+use garage_model::s3::mpu_table::MultipartUpload;
 use garage_model::s3::version_table::Version;
 
 use crate::cli::*;
@@ -53,6 +54,7 @@ pub enum AdminRpc {
 		bucket: Bucket,
 		relevant_keys: HashMap<String, Key>,
 		counters: HashMap<String, i64>,
+		mpu_counters: HashMap<String, i64>,
 	},
 	KeyList(Vec<(String, String)>),
 	KeyInfo(Key, HashMap<Uuid, Bucket>),
@@ -67,6 +69,7 @@ pub enum AdminRpc {
 		hash: Hash,
 		refcount: u64,
 		versions: Vec<Result<Version, Uuid>>,
+		uploads: Vec<MultipartUpload>,
 	},
 }
 
@@ -274,7 +277,7 @@ impl AdminRpcHandler {
 		// Gather storage node and free space statistics
 		let layout = &self.garage.system.ring.borrow().layout;
 		let mut node_partition_count = HashMap::<Uuid, u64>::new();
-		for short_id in layout.ring_assignation_data.iter() {
+		for short_id in layout.ring_assignment_data.iter() {
 			let id = layout.node_id_vec[*short_id as usize];
 			*node_partition_count.entry(id).or_default() += 1;
 		}