Implement DeleteObjects

7 files changed, 162 insertions, 25 deletions

diff --git a/Cargo.lock b/Cargo.lock
index f7c86808..97e05222 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -375,6 +375,7 @@ dependencies = [
  "hyper 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -462,6 +463,7 @@ dependencies = [
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1015,6 +1017,14 @@ dependencies = [
 ]
 
 [[package]]
+name = "roxmltree"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "xmlparser 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
 name = "rustls"
 version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1489,6 +1499,11 @@ dependencies = [
  "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "xmlparser"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [metadata]
 "checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada"
 "checksum arc-swap 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62"
@@ -1594,6 +1609,7 @@ dependencies = [
 "checksum ring 0.16.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1ba5a8ec64ee89a76c98c549af81ff14813df09c3e6dc4766c3856da48597a0c"
 "checksum rmp 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)" = "0f10b46df14cf1ee1ac7baa4d2fbc2c52c0622a4b82fa8740e37bc452ac0184f"
 "checksum rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4c1ee98f14fe8b8e9c5ea13d25da7b2a1796169202c57a09d7288de90d56222b"
+"checksum roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d5001f134077069d87f77c8b9452b690df2445f7a43f1c7ca4a1af8dd505789d"
 "checksum rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1"
 "checksum rustversion 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6"
 "checksum ryu 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76"
@@ -1651,3 +1667,4 @@ dependencies = [
 "checksum winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
+"checksum xmlparser 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ccb4240203dadf40be2de9369e5c6dec1bf427528115b030baca3334c18362d7"
diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml
index 11a42be9..b8674fba 100644
--- a/src/api/Cargo.toml
+++ b/src/api/Cargo.toml
@@ -31,4 +31,4 @@ hyper = "0.13"
 url = "2.1"
 httpdate = "0.3"
 percent-encoding = "2.1.0"
-
+roxmltree = "0.11"
diff --git a/src/api/api_server.rs b/src/api/api_server.rs
index 642697da..913ff0af 100644
--- a/src/api/api_server.rs
+++ b/src/api/api_server.rs
@@ -74,8 +74,7 @@ async fn handler_inner(
 ) -> Result<Response<BodyType>, Error> {
 	let path = req.uri().path().to_string();
 	let path = percent_encoding::percent_decode_str(&path)
-		.decode_utf8()
-		.map_err(|e| Error::BadRequest(format!("Invalid utf8 key ({})", e)))?;
+		.decode_utf8()?;
 
 	let (bucket, key) = parse_bucket_key(&path)?;
 	if bucket.len() == 0 {
@@ -125,10 +124,7 @@ async fn handler_inner(
 					// CopyObject query
 					let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
 					let copy_source = percent_encoding::percent_decode_str(&copy_source)
-						.decode_utf8()
-						.map_err(|e| {
-							Error::BadRequest(format!("Invalid utf8 copy_source ({})", e))
-						})?;
+						.decode_utf8()?;
 					let (source_bucket, source_key) = parse_bucket_key(&copy_source)?;
 					if !api_key.allow_read(&source_bucket) {
 						return Err(Error::Forbidden(format!(
@@ -153,8 +149,7 @@ async fn handler_inner(
 					Ok(handle_abort_multipart_upload(garage, &bucket, &key, upload_id).await?)
 				} else {
 					// DeleteObject query
-					let version_uuid = handle_delete(garage, &bucket, &key).await?;
-					Ok(put_response(version_uuid))
+					Ok(handle_delete(garage, &bucket, &key).await?)
 				}
 			}
 			&Method::POST => {
@@ -233,7 +228,22 @@ async fn handler_inner(
 				)
 				.await?)
 			}
-			_ => Err(Error::BadRequest(format!("Invalid method"))),
+            &Method::POST => {
+                if params.contains_key(&"delete".to_string()) {
+                    // DeleteObjects
+                    Ok(handle_delete_objects(garage, bucket, req).await?)
+                } else {
+                    println!(
+                        "Body: {}",
+                        std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
+                            .unwrap_or("<invalid utf8>")
+                    );
+                    Err(Error::BadRequest(format!("Unsupported call")))
+                }
+            }
+			_ => {
+                Err(Error::BadRequest(format!("Invalid method")))
+            }
 		}
 	}
 }
diff --git a/src/api/s3_delete.rs b/src/api/s3_delete.rs
index 4d6805fb..001eb162 100644
--- a/src/api/s3_delete.rs
+++ b/src/api/s3_delete.rs
@@ -1,4 +1,7 @@
 use std::sync::Arc;
+use std::fmt::Write;
+
+use hyper::{Body, Request, Response};
 
 use garage_util::data::*;
 use garage_util::error::Error;
@@ -6,7 +9,10 @@ use garage_util::error::Error;
 use garage_core::garage::Garage;
 use garage_core::object_table::*;
 
-pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Result<UUID, Error> {
+use crate::http_util::*;
+use crate::encoding::*;
+
+async fn handle_delete_internal(garage: &Garage, bucket: &str, key: &str) -> Result<(UUID, UUID), Error> {
 	let object = match garage
 		.object_table
 		.get(&bucket.to_string(), &key.to_string())
@@ -14,7 +20,7 @@ pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Resu
 	{
 		None => {
 			// No need to delete
-			return Ok([0u8; 32].into());
+			return Err(Error::NotFound);
 		}
 		Some(o) => o,
 	};
@@ -23,16 +29,19 @@ pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Resu
 		v.data != ObjectVersionData::DeleteMarker && v.state != ObjectVersionState::Aborted
 	});
 
-	let mut must_delete = false;
+	let mut must_delete = None;
 	let mut timestamp = now_msec();
 	for v in interesting_versions {
-		must_delete = true;
+        if v.timestamp + 1 > timestamp || must_delete.is_none() {
+		    must_delete = Some(v.uuid);
+        }
 		timestamp = std::cmp::max(timestamp, v.timestamp + 1);
 	}
 
-	if !must_delete {
-		return Ok([0u8; 32].into());
-	}
+    let deleted_version = match must_delete {
+        None => return Err(Error::NotFound),
+        Some(v) => v,
+    };
 
 	let version_uuid = gen_uuid();
 
@@ -50,5 +59,88 @@ pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Resu
 	);
 
 	garage.object_table.insert(&object).await?;
-	return Ok(version_uuid);
+	return Ok((deleted_version, version_uuid));
+}
+
+pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Result<Response<BodyType>, Error> {
+    let (_deleted_version, delete_marker_version) = handle_delete_internal(&garage, bucket, key).await?;
+
+	Ok(Response::builder()
+		.header("x-amz-version-id", hex::encode(delete_marker_version))
+		.body(empty_body())
+		.unwrap())
+}
+
+pub async fn handle_delete_objects(garage: Arc<Garage>, bucket: &str, req: Request<Body>) -> Result<Response<BodyType>, Error> {
+    let body = hyper::body::to_bytes(req.into_body()).await?;
+    let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;
+    let cmd = parse_delete_objects_xml(&cmd_xml)
+        .map_err(|e| Error::BadRequest(format!("Invald delete XML query: {}", e)))?;
+
+    let mut retxml = String::new();
+	writeln!(&mut retxml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
+	writeln!(&mut retxml, "<DeleteObjectsOutput>").unwrap();
+
+    for obj in cmd.objects.iter() {
+        match handle_delete_internal(&garage, bucket, &obj.key).await {
+            Ok((deleted_version, delete_marker_version)) => {
+                writeln!(&mut retxml, "\t<Deleted>").unwrap();
+                writeln!(&mut retxml, "\t\t<Key>{}</Key>", obj.key).unwrap();
+                writeln!(&mut retxml, "\t\t<VersionId>{}</VersionId>", hex::encode(deleted_version)).unwrap();
+                writeln!(&mut retxml, "\t\t<DeleteMarkerVersionId>{}</DeleteMarkerVersionId>", hex::encode(delete_marker_version)).unwrap();
+                writeln!(&mut retxml, "\t</Deleted>").unwrap();
+            }
+            Err(e) => {
+                writeln!(&mut retxml, "\t<Error>").unwrap();
+                writeln!(&mut retxml, "\t\t<Code>{}</Code>", e.http_status_code()).unwrap();
+                writeln!(&mut retxml, "\t\t<Key>{}</Key>", obj.key).unwrap();
+                writeln!(&mut retxml, "\t\t<Message>{}</Message>", xml_escape(&format!("{}", e))).unwrap();
+                writeln!(&mut retxml, "\t</Error>").unwrap();
+            }
+        }
+    }
+
+	writeln!(&mut retxml, "</DeleteObjectsOutput>").unwrap();
+
+	Ok(Response::new(Box::new(BytesBody::from(retxml.into_bytes()))))
+}
+
+struct DeleteRequest {
+    objects: Vec<DeleteObject>,
 }
+
+struct DeleteObject {
+    key: String,
+}
+
+fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Result<DeleteRequest, String> {
+    let mut ret = DeleteRequest{objects: vec![]};
+
+    let root = xml.root();
+    let delete = match root.first_child() {
+        Some(del) => del,
+        None => return Err(format!("Delete tag not found")),
+    };
+    if !delete.has_tag_name("Delete") {
+        return Err(format!("Invalid root tag: {:?}", root));
+    }
+
+    for item in delete.children() {
+        if item.has_tag_name("Object") {
+            if let Some(key) = item.children().find(|e| e.has_tag_name("Key")) {
+                if let Some(key_str) = key.text() {
+                    ret.objects.push(DeleteObject{key: key_str.to_string()});
+                } else {
+                    return Err(format!("No text for key: {:?}", key));
+                }
+            } else {
+                return Err(format!("No delete key for item: {:?}", item));
+            }
+        } else {
+            return Err(format!("Invalid delete item: {:?}", item));
+        }
+    }
+
+    Ok(ret)
+}
+
diff --git a/src/api/s3_list.rs b/src/api/s3_list.rs
index df792da2..d4d8161e 100644
--- a/src/api/s3_list.rs
+++ b/src/api/s3_list.rs
@@ -30,34 +30,38 @@ pub async fn handle_list(
 	let mut result_keys = BTreeMap::<String, ListResultInfo>::new();
 	let mut result_common_prefixes = BTreeSet::<String>::new();
 
-	let mut truncated = true;
 	let mut next_chunk_start = marker.unwrap_or(prefix).to_string();
 
 	debug!("List request: `{}` {} `{}`", delimiter, max_keys, prefix);
 
-	while result_keys.len() + result_common_prefixes.len() < max_keys && truncated {
+	let truncated;
+	'query_loop: loop {
 		let objects = garage
 			.object_table
 			.get_range(
 				&bucket.to_string(),
 				Some(next_chunk_start.clone()),
 				Some(()),
-				max_keys,
+				max_keys + 1,
 			)
 			.await?;
 		debug!(
 			"List: get range {} (max {}), results: {}",
 			next_chunk_start,
-			max_keys,
+			max_keys + 1,
 			objects.len()
 		);
 
 		for object in objects.iter() {
 			if !object.key.starts_with(prefix) {
 				truncated = false;
-				break;
+				break 'query_loop;
 			}
 			if let Some(version) = object.versions().iter().find(|x| x.is_data()) {
+                if result_keys.len() + result_common_prefixes.len() >= max_keys {
+                    truncated = true;
+                    break 'query_loop;
+                }
 				let common_prefix = if delimiter.len() > 0 {
 					let relative_key = &object.key[prefix.len()..];
 					match relative_key.find(delimiter) {
@@ -83,8 +87,9 @@ pub async fn handle_list(
 				};
 			}
 		}
-		if objects.len() < max_keys {
-			truncated = false;
+		if objects.len() < max_keys + 1 {
+            truncated = false;
+            break 'query_loop;
 		}
 		if objects.len() > 0 {
 			next_chunk_start = objects[objects.len() - 1].key.clone();
diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml
index 6f61a586..d8c39c62 100644
--- a/src/util/Cargo.toml
+++ b/src/util/Cargo.toml
@@ -32,4 +32,5 @@ hyper = "0.13"
 rustls = "0.17"
 webpki = "0.21"
 
+roxmltree = "0.11"
 
diff --git a/src/util/error.rs b/src/util/error.rs
index cb99cbbc..d2ed1ccf 100644
--- a/src/util/error.rs
+++ b/src/util/error.rs
@@ -114,3 +114,15 @@ impl<T> From<tokio::sync::mpsc::error::SendError<T>> for Error {
 		Error::Message(format!("MPSC send error"))
 	}
 }
+
+impl From<std::str::Utf8Error> for Error {
+    fn from(e: std::str::Utf8Error) -> Error {
+        Error::BadRequest(format!("Invalid UTF-8: {}", e))
+    }
+}
+
+impl From<roxmltree::Error> for Error {
+    fn from(e: roxmltree::Error) -> Error {
+        Error::BadRequest(format!("Invalid XML: {}", e))
+    }
+}