aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Cargo.lock17
-rw-r--r--src/api/Cargo.toml2
-rw-r--r--src/api/api_server.rs28
-rw-r--r--src/api/s3_delete.rs108
-rw-r--r--src/api/s3_list.rs19
-rw-r--r--src/util/Cargo.toml1
-rw-r--r--src/util/error.rs12
7 files changed, 162 insertions, 25 deletions
diff --git a/Cargo.lock b/Cargo.lock
index f7c86808..97e05222 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -375,6 +375,7 @@ dependencies = [
"hyper 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -462,6 +463,7 @@ dependencies = [
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1015,6 +1017,14 @@ dependencies = [
]
[[package]]
+name = "roxmltree"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "xmlparser 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
name = "rustls"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1489,6 +1499,11 @@ dependencies = [
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
+[[package]]
+name = "xmlparser"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
[metadata]
"checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada"
"checksum arc-swap 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62"
@@ -1594,6 +1609,7 @@ dependencies = [
"checksum ring 0.16.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1ba5a8ec64ee89a76c98c549af81ff14813df09c3e6dc4766c3856da48597a0c"
"checksum rmp 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)" = "0f10b46df14cf1ee1ac7baa4d2fbc2c52c0622a4b82fa8740e37bc452ac0184f"
"checksum rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4c1ee98f14fe8b8e9c5ea13d25da7b2a1796169202c57a09d7288de90d56222b"
+"checksum roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d5001f134077069d87f77c8b9452b690df2445f7a43f1c7ca4a1af8dd505789d"
"checksum rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1"
"checksum rustversion 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6"
"checksum ryu 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76"
@@ -1651,3 +1667,4 @@ dependencies = [
"checksum winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
+"checksum xmlparser 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ccb4240203dadf40be2de9369e5c6dec1bf427528115b030baca3334c18362d7"
diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml
index 11a42be9..b8674fba 100644
--- a/src/api/Cargo.toml
+++ b/src/api/Cargo.toml
@@ -31,4 +31,4 @@ hyper = "0.13"
url = "2.1"
httpdate = "0.3"
percent-encoding = "2.1.0"
-
+roxmltree = "0.11"
diff --git a/src/api/api_server.rs b/src/api/api_server.rs
index 642697da..913ff0af 100644
--- a/src/api/api_server.rs
+++ b/src/api/api_server.rs
@@ -74,8 +74,7 @@ async fn handler_inner(
) -> Result<Response<BodyType>, Error> {
let path = req.uri().path().to_string();
let path = percent_encoding::percent_decode_str(&path)
- .decode_utf8()
- .map_err(|e| Error::BadRequest(format!("Invalid utf8 key ({})", e)))?;
+ .decode_utf8()?;
let (bucket, key) = parse_bucket_key(&path)?;
if bucket.len() == 0 {
@@ -125,10 +124,7 @@ async fn handler_inner(
// CopyObject query
let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
let copy_source = percent_encoding::percent_decode_str(&copy_source)
- .decode_utf8()
- .map_err(|e| {
- Error::BadRequest(format!("Invalid utf8 copy_source ({})", e))
- })?;
+ .decode_utf8()?;
let (source_bucket, source_key) = parse_bucket_key(&copy_source)?;
if !api_key.allow_read(&source_bucket) {
return Err(Error::Forbidden(format!(
@@ -153,8 +149,7 @@ async fn handler_inner(
Ok(handle_abort_multipart_upload(garage, &bucket, &key, upload_id).await?)
} else {
// DeleteObject query
- let version_uuid = handle_delete(garage, &bucket, &key).await?;
- Ok(put_response(version_uuid))
+ Ok(handle_delete(garage, &bucket, &key).await?)
}
}
&Method::POST => {
@@ -233,7 +228,22 @@ async fn handler_inner(
)
.await?)
}
- _ => Err(Error::BadRequest(format!("Invalid method"))),
+ &Method::POST => {
+ if params.contains_key(&"delete".to_string()) {
+ // DeleteObjects
+ Ok(handle_delete_objects(garage, bucket, req).await?)
+ } else {
+ println!(
+ "Body: {}",
+ std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
+ .unwrap_or("<invalid utf8>")
+ );
+ Err(Error::BadRequest(format!("Unsupported call")))
+ }
+ }
+ _ => {
+ Err(Error::BadRequest(format!("Invalid method")))
+ }
}
}
}
diff --git a/src/api/s3_delete.rs b/src/api/s3_delete.rs
index 4d6805fb..001eb162 100644
--- a/src/api/s3_delete.rs
+++ b/src/api/s3_delete.rs
@@ -1,4 +1,7 @@
use std::sync::Arc;
+use std::fmt::Write;
+
+use hyper::{Body, Request, Response};
use garage_util::data::*;
use garage_util::error::Error;
@@ -6,7 +9,10 @@ use garage_util::error::Error;
use garage_core::garage::Garage;
use garage_core::object_table::*;
-pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Result<UUID, Error> {
+use crate::http_util::*;
+use crate::encoding::*;
+
+async fn handle_delete_internal(garage: &Garage, bucket: &str, key: &str) -> Result<(UUID, UUID), Error> {
let object = match garage
.object_table
.get(&bucket.to_string(), &key.to_string())
@@ -14,7 +20,7 @@ pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Resu
{
None => {
// No need to delete
- return Ok([0u8; 32].into());
+ return Err(Error::NotFound);
}
Some(o) => o,
};
@@ -23,16 +29,19 @@ pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Resu
v.data != ObjectVersionData::DeleteMarker && v.state != ObjectVersionState::Aborted
});
- let mut must_delete = false;
+ let mut must_delete = None;
let mut timestamp = now_msec();
for v in interesting_versions {
- must_delete = true;
+ if v.timestamp + 1 > timestamp || must_delete.is_none() {
+ must_delete = Some(v.uuid);
+ }
timestamp = std::cmp::max(timestamp, v.timestamp + 1);
}
- if !must_delete {
- return Ok([0u8; 32].into());
- }
+ let deleted_version = match must_delete {
+ None => return Err(Error::NotFound),
+ Some(v) => v,
+ };
let version_uuid = gen_uuid();
@@ -50,5 +59,88 @@ pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Resu
);
garage.object_table.insert(&object).await?;
- return Ok(version_uuid);
+ return Ok((deleted_version, version_uuid));
+}
+
+pub async fn handle_delete(garage: Arc<Garage>, bucket: &str, key: &str) -> Result<Response<BodyType>, Error> {
+ let (_deleted_version, delete_marker_version) = handle_delete_internal(&garage, bucket, key).await?;
+
+ Ok(Response::builder()
+ .header("x-amz-version-id", hex::encode(delete_marker_version))
+ .body(empty_body())
+ .unwrap())
+}
+
+pub async fn handle_delete_objects(garage: Arc<Garage>, bucket: &str, req: Request<Body>) -> Result<Response<BodyType>, Error> {
+ let body = hyper::body::to_bytes(req.into_body()).await?;
+ let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;
+ let cmd = parse_delete_objects_xml(&cmd_xml)
+ .map_err(|e| Error::BadRequest(format!("Invald delete XML query: {}", e)))?;
+
+ let mut retxml = String::new();
+ writeln!(&mut retxml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
+ writeln!(&mut retxml, "<DeleteObjectsOutput>").unwrap();
+
+ for obj in cmd.objects.iter() {
+ match handle_delete_internal(&garage, bucket, &obj.key).await {
+ Ok((deleted_version, delete_marker_version)) => {
+ writeln!(&mut retxml, "\t<Deleted>").unwrap();
+ writeln!(&mut retxml, "\t\t<Key>{}</Key>", obj.key).unwrap();
+ writeln!(&mut retxml, "\t\t<VersionId>{}</VersionId>", hex::encode(deleted_version)).unwrap();
+ writeln!(&mut retxml, "\t\t<DeleteMarkerVersionId>{}</DeleteMarkerVersionId>", hex::encode(delete_marker_version)).unwrap();
+ writeln!(&mut retxml, "\t</Deleted>").unwrap();
+ }
+ Err(e) => {
+ writeln!(&mut retxml, "\t<Error>").unwrap();
+ writeln!(&mut retxml, "\t\t<Code>{}</Code>", e.http_status_code()).unwrap();
+ writeln!(&mut retxml, "\t\t<Key>{}</Key>", obj.key).unwrap();
+ writeln!(&mut retxml, "\t\t<Message>{}</Message>", xml_escape(&format!("{}", e))).unwrap();
+ writeln!(&mut retxml, "\t</Error>").unwrap();
+ }
+ }
+ }
+
+ writeln!(&mut retxml, "</DeleteObjectsOutput>").unwrap();
+
+ Ok(Response::new(Box::new(BytesBody::from(retxml.into_bytes()))))
+}
+
+struct DeleteRequest {
+ objects: Vec<DeleteObject>,
}
+
+struct DeleteObject {
+ key: String,
+}
+
+fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Result<DeleteRequest, String> {
+ let mut ret = DeleteRequest{objects: vec![]};
+
+ let root = xml.root();
+ let delete = match root.first_child() {
+ Some(del) => del,
+ None => return Err(format!("Delete tag not found")),
+ };
+ if !delete.has_tag_name("Delete") {
+ return Err(format!("Invalid root tag: {:?}", root));
+ }
+
+ for item in delete.children() {
+ if item.has_tag_name("Object") {
+ if let Some(key) = item.children().find(|e| e.has_tag_name("Key")) {
+ if let Some(key_str) = key.text() {
+ ret.objects.push(DeleteObject{key: key_str.to_string()});
+ } else {
+ return Err(format!("No text for key: {:?}", key));
+ }
+ } else {
+ return Err(format!("No delete key for item: {:?}", item));
+ }
+ } else {
+ return Err(format!("Invalid delete item: {:?}", item));
+ }
+ }
+
+ Ok(ret)
+}
+
diff --git a/src/api/s3_list.rs b/src/api/s3_list.rs
index df792da2..d4d8161e 100644
--- a/src/api/s3_list.rs
+++ b/src/api/s3_list.rs
@@ -30,34 +30,38 @@ pub async fn handle_list(
let mut result_keys = BTreeMap::<String, ListResultInfo>::new();
let mut result_common_prefixes = BTreeSet::<String>::new();
- let mut truncated = true;
let mut next_chunk_start = marker.unwrap_or(prefix).to_string();
debug!("List request: `{}` {} `{}`", delimiter, max_keys, prefix);
- while result_keys.len() + result_common_prefixes.len() < max_keys && truncated {
+ let truncated;
+ 'query_loop: loop {
let objects = garage
.object_table
.get_range(
&bucket.to_string(),
Some(next_chunk_start.clone()),
Some(()),
- max_keys,
+ max_keys + 1,
)
.await?;
debug!(
"List: get range {} (max {}), results: {}",
next_chunk_start,
- max_keys,
+ max_keys + 1,
objects.len()
);
for object in objects.iter() {
if !object.key.starts_with(prefix) {
truncated = false;
- break;
+ break 'query_loop;
}
if let Some(version) = object.versions().iter().find(|x| x.is_data()) {
+ if result_keys.len() + result_common_prefixes.len() >= max_keys {
+ truncated = true;
+ break 'query_loop;
+ }
let common_prefix = if delimiter.len() > 0 {
let relative_key = &object.key[prefix.len()..];
match relative_key.find(delimiter) {
@@ -83,8 +87,9 @@ pub async fn handle_list(
};
}
}
- if objects.len() < max_keys {
- truncated = false;
+ if objects.len() < max_keys + 1 {
+ truncated = false;
+ break 'query_loop;
}
if objects.len() > 0 {
next_chunk_start = objects[objects.len() - 1].key.clone();
diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml
index 6f61a586..d8c39c62 100644
--- a/src/util/Cargo.toml
+++ b/src/util/Cargo.toml
@@ -32,4 +32,5 @@ hyper = "0.13"
rustls = "0.17"
webpki = "0.21"
+roxmltree = "0.11"
diff --git a/src/util/error.rs b/src/util/error.rs
index cb99cbbc..d2ed1ccf 100644
--- a/src/util/error.rs
+++ b/src/util/error.rs
@@ -114,3 +114,15 @@ impl<T> From<tokio::sync::mpsc::error::SendError<T>> for Error {
Error::Message(format!("MPSC send error"))
}
}
+
+impl From<std::str::Utf8Error> for Error {
+ fn from(e: std::str::Utf8Error) -> Error {
+ Error::BadRequest(format!("Invalid UTF-8: {}", e))
+ }
+}
+
+impl From<roxmltree::Error> for Error {
+ fn from(e: roxmltree::Error) -> Error {
+ Error::BadRequest(format!("Invalid XML: {}", e))
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-07 18:51:41 +0000