Merge pull request 'Fix #204 (full Multipart Uploads semantics)' (#553) from nlnet-task1 into next

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/553

1 files changed, 141 insertions, 85 deletions

diff --git a/src/garage/repair/online.rs b/src/garage/repair/online.rs
index 0e14ed51..abfaf9f9 100644
--- a/src/garage/repair/online.rs
+++ b/src/garage/repair/online.rs
@@ -5,11 +5,16 @@ use async_trait::async_trait;
 use tokio::sync::watch;
 
 use garage_block::repair::ScrubWorkerCommand;
+
 use garage_model::garage::Garage;
 use garage_model::s3::block_ref_table::*;
+use garage_model::s3::mpu_table::*;
 use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
+
+use garage_table::replication::*;
 use garage_table::*;
+
 use garage_util::background::*;
 use garage_util::error::Error;
 use garage_util::migrate::Migrate;
@@ -32,11 +37,15 @@ pub async fn launch_online_repair(
 		}
 		RepairWhat::Versions => {
 			info!("Repairing the versions table");
-			bg.spawn_worker(RepairVersionsWorker::new(garage.clone()));
+			bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairVersions));
+		}
+		RepairWhat::MultipartUploads => {
+			info!("Repairing the multipart uploads table");
+			bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairMpu));
 		}
 		RepairWhat::BlockRefs => {
 			info!("Repairing the block refs table");
-			bg.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
+			bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs));
 		}
 		RepairWhat::Blocks => {
 			info!("Repairing the stored blocks");
@@ -67,70 +76,70 @@ pub async fn launch_online_repair(
 
 // ----
 
-struct RepairVersionsWorker {
+#[async_trait]
+trait TableRepair: Send + Sync + 'static {
+	type T: TableSchema;
+
+	fn table(garage: &Garage) -> &Table<Self::T, TableShardedReplication>;
+
+	async fn process(
+		&mut self,
+		garage: &Garage,
+		entry: <<Self as TableRepair>::T as TableSchema>::E,
+	) -> Result<bool, Error>;
+}
+
+struct TableRepairWorker<T: TableRepair> {
 	garage: Arc<Garage>,
 	pos: Vec<u8>,
 	counter: usize,
+	repairs: usize,
+	inner: T,
 }
 
-impl RepairVersionsWorker {
-	fn new(garage: Arc<Garage>) -> Self {
+impl<R: TableRepair> TableRepairWorker<R> {
+	fn new(garage: Arc<Garage>, inner: R) -> Self {
 		Self {
 			garage,
+			inner,
 			pos: vec![],
 			counter: 0,
+			repairs: 0,
 		}
 	}
 }
 
 #[async_trait]
-impl Worker for RepairVersionsWorker {
+impl<R: TableRepair> Worker for TableRepairWorker<R> {
 	fn name(&self) -> String {
-		"Version repair worker".into()
+		format!("{} repair worker", R::T::TABLE_NAME)
 	}
 
 	fn status(&self) -> WorkerStatus {
 		WorkerStatus {
-			progress: Some(self.counter.to_string()),
+			progress: Some(format!("{} ({})", self.counter, self.repairs)),
 			..Default::default()
 		}
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let (item_bytes, next_pos) = match self.garage.version_table.data.store.get_gt(&self.pos)? {
+		let (item_bytes, next_pos) = match R::table(&self.garage).data.store.get_gt(&self.pos)? {
 			Some((k, v)) => (v, k),
 			None => {
-				info!("repair_versions: finished, done {}", self.counter);
+				info!(
+					"{}: finished, done {}, fixed {}",
+					self.name(),
+					self.counter,
+					self.repairs
+				);
 				return Ok(WorkerState::Done);
 			}
 		};
 
-		let version = Version::decode(&item_bytes).ok_or_message("Cannot decode Version")?;
-		if !version.deleted.get() {
-			let object = self
-				.garage
-				.object_table
-				.get(&version.bucket_id, &version.key)
-				.await?;
-			let version_exists = match object {
-				Some(o) => o
-					.versions()
-					.iter()
-					.any(|x| x.uuid == version.uuid && x.state != ObjectVersionState::Aborted),
-				None => false,
-			};
-			if !version_exists {
-				info!("Repair versions: marking version as deleted: {:?}", version);
-				self.garage
-					.version_table
-					.insert(&Version::new(
-						version.uuid,
-						version.bucket_id,
-						version.key,
-						true,
-					))
-					.await?;
-			}
+		let entry = <R::T as TableSchema>::E::decode(&item_bytes)
+			.ok_or_message("Cannot decode table entry")?;
+		if self.inner.process(&self.garage, entry).await? {
+			self.repairs += 1;
 		}
 
 		self.counter += 1;
@@ -146,77 +155,124 @@ impl Worker for RepairVersionsWorker {
 
 // ----
 
-struct RepairBlockrefsWorker {
-	garage: Arc<Garage>,
-	pos: Vec<u8>,
-	counter: usize,
-}
+struct RepairVersions;
 
-impl RepairBlockrefsWorker {
-	fn new(garage: Arc<Garage>) -> Self {
-		Self {
-			garage,
-			pos: vec![],
-			counter: 0,
+#[async_trait]
+impl TableRepair for RepairVersions {
+	type T = VersionTable;
+
+	fn table(garage: &Garage) -> &Table<Self::T, TableShardedReplication> {
+		&garage.version_table
+	}
+
+	async fn process(&mut self, garage: &Garage, version: Version) -> Result<bool, Error> {
+		if !version.deleted.get() {
+			let ref_exists = match &version.backlink {
+				VersionBacklink::Object { bucket_id, key } => garage
+					.object_table
+					.get(bucket_id, key)
+					.await?
+					.map(|o| {
+						o.versions().iter().any(|x| {
+							x.uuid == version.uuid && x.state != ObjectVersionState::Aborted
+						})
+					})
+					.unwrap_or(false),
+				VersionBacklink::MultipartUpload { upload_id } => garage
+					.mpu_table
+					.get(upload_id, &EmptyKey)
+					.await?
+					.map(|u| !u.deleted.get())
+					.unwrap_or(false),
+			};
+
+			if !ref_exists {
+				info!("Repair versions: marking version as deleted: {:?}", version);
+				garage
+					.version_table
+					.insert(&Version::new(version.uuid, version.backlink, true))
+					.await?;
+				return Ok(true);
+			}
 		}
+
+		Ok(false)
 	}
 }
 
+// ----
+
+struct RepairBlockRefs;
+
 #[async_trait]
-impl Worker for RepairBlockrefsWorker {
-	fn name(&self) -> String {
-		"Block refs repair worker".into()
-	}
+impl TableRepair for RepairBlockRefs {
+	type T = BlockRefTable;
 
-	fn status(&self) -> WorkerStatus {
-		WorkerStatus {
-			progress: Some(self.counter.to_string()),
-			..Default::default()
-		}
+	fn table(garage: &Garage) -> &Table<Self::T, TableShardedReplication> {
+		&garage.block_ref_table
 	}
 
-	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let (item_bytes, next_pos) =
-			match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
-				Some((k, v)) => (v, k),
-				None => {
-					info!("repair_block_ref: finished, done {}", self.counter);
-					return Ok(WorkerState::Done);
-				}
-			};
-
-		let block_ref = BlockRef::decode(&item_bytes).ok_or_message("Cannot decode BlockRef")?;
+	async fn process(&mut self, garage: &Garage, mut block_ref: BlockRef) -> Result<bool, Error> {
 		if !block_ref.deleted.get() {
-			let version = self
-				.garage
+			let ref_exists = garage
 				.version_table
 				.get(&block_ref.version, &EmptyKey)
-				.await?;
-			// The version might not exist if it has been GC'ed
-			let ref_exists = version.map(|v| !v.deleted.get()).unwrap_or(false);
+				.await?
+				.map(|v| !v.deleted.get())
+				.unwrap_or(false);
+
 			if !ref_exists {
 				info!(
 					"Repair block ref: marking block_ref as deleted: {:?}",
 					block_ref
 				);
-				self.garage
-					.block_ref_table
-					.insert(&BlockRef {
-						block: block_ref.block,
-						version: block_ref.version,
-						deleted: true.into(),
-					})
-					.await?;
+				block_ref.deleted.set();
+				garage.block_ref_table.insert(&block_ref).await?;
+				return Ok(true);
 			}
 		}
 
-		self.counter += 1;
-		self.pos = next_pos;
+		Ok(false)
+	}
+}
 
-		Ok(WorkerState::Busy)
+// ----
+
+struct RepairMpu;
+
+#[async_trait]
+impl TableRepair for RepairMpu {
+	type T = MultipartUploadTable;
+
+	fn table(garage: &Garage) -> &Table<Self::T, TableShardedReplication> {
+		&garage.mpu_table
 	}
 
-	async fn wait_for_work(&mut self) -> WorkerState {
-		unreachable!()
+	async fn process(&mut self, garage: &Garage, mut mpu: MultipartUpload) -> Result<bool, Error> {
+		if !mpu.deleted.get() {
+			let ref_exists = garage
+				.object_table
+				.get(&mpu.bucket_id, &mpu.key)
+				.await?
+				.map(|o| {
+					o.versions()
+						.iter()
+						.any(|x| x.uuid == mpu.upload_id && x.is_uploading(Some(true)))
+				})
+				.unwrap_or(false);
+
+			if !ref_exists {
+				info!(
+					"Repair multipart uploads: marking mpu as deleted: {:?}",
+					mpu
+				);
+				mpu.parts.clear();
+				mpu.deleted.set();
+				garage.mpu_table.insert(&mpu).await?;
+				return Ok(true);
+			}
+		}
+
+		Ok(false)
 	}
 }