S3 API: support ListBucketsfeature/s3/list-buckets

author: Quentin Dufour <quentin@deuxfleurs.fr> 2021-05-02 22:30:56 +0200
committer: Quentin Dufour <quentin@deuxfleurs.fr> 2021-05-02 22:30:56 +0200
commit: a8766626403c310d80139ab0660221f5d5aa4f22 (patch)
tree: d77d49f3c94ededa95200c1855ab422a19699fb9
parent: ef4d6e782a67bd422489d643e8807eaf7fb853c4 (diff)
download: garage-feature/s3/list-buckets.tar.gz
garage-feature/s3/list-buckets.zip
6 files changed, 115 insertions, 2 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 5dc83dfa..1ea13711 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -397,7 +397,9 @@ dependencies = [
  "log",
  "md-5",
  "percent-encoding",
+ "quick-xml",
  "roxmltree",
+ "serde",
  "sha2",
  "tokio",
  "url",
@@ -1035,6 +1037,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
 [[package]]
+name = "quick-xml"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0452695941410a58c8ce4391707ba9bad26a247173bd9886a05a5e8a8babec75"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
+[[package]]
 name = "quote"
 version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/script/test-smoke.sh b/script/test-smoke.sh
index e8d41ed3..9fe06de9 100755
--- a/script/test-smoke.sh
+++ b/script/test-smoke.sh
@@ -32,6 +32,7 @@ echo "🧪 S3 API testing..."
 if [ -z "$SKIP_AWS" ]; then
   echo "🛠️ Testing with awscli"
   source ${SCRIPT_FOLDER}/dev-env-aws.sh
+  aws s3 ls
   for idx in $(seq 1 3); do
     aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
     aws s3 ls s3://eprouvette
@@ -46,6 +47,7 @@ fi
 if [ -z "$SKIP_S3CMD" ]; then
   echo "🛠️ Testing with s3cmd"
   source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
+  s3cmd ls
   for idx in $(seq 1 3); do
     s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
     s3cmd ls s3://eprouvette
@@ -60,6 +62,7 @@ fi
 if [ -z "$SKIP_MC" ]; then
   echo "🛠️ Testing with mc (minio client)"
   source ${SCRIPT_FOLDER}/dev-env-mc.sh
+  mc ls garage/
   for idx in $(seq 1 3); do
     mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc"
     mc ls garage/eprouvette
@@ -74,6 +77,7 @@ fi
 if [ -z "$SKIP_RCLONE" ]; then
   echo "🛠️ Testing with rclone"
   source ${SCRIPT_FOLDER}/dev-env-rclone.sh
+  rclone lsd garage:
   for idx in $(seq 1 3); do
     cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
     rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/"
diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml
index 0b824ca3..b9fc4bfc 100644
--- a/src/api/Cargo.toml
+++ b/src/api/Cargo.toml
@@ -38,4 +38,6 @@ http-range = "0.1"
 hyper = "0.14"
 percent-encoding = "2.1.0"
 roxmltree = "0.14"
+serde = { version = "1.0", features = ["derive"] }
+quick-xml = { version = "0.21", features = [ "serialize" ] }
 url = "2.1"
diff --git a/src/api/api_server.rs b/src/api/api_server.rs
index ab8bd736..8a51b851 100644
--- a/src/api/api_server.rs
+++ b/src/api/api_server.rs
@@ -81,10 +81,12 @@ async fn handler(
 async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
 	let path = req.uri().path().to_string();
 	let path = percent_encoding::percent_decode_str(&path).decode_utf8()?;
+	let (api_key, content_sha256) = check_signature(&garage, &req).await?;
+	if path == "/" {
+		return handle_list_buckets(&api_key);
+	}
 
 	let (bucket, key) = parse_bucket_key(&path)?;
-
-	let (api_key, content_sha256) = check_signature(&garage, &req).await?;
 	let allowed = match req.method() {
 		&Method::HEAD | &Method::GET => api_key.allow_read(&bucket),
 		_ => api_key.allow_write(&bucket),
diff --git a/src/api/error.rs b/src/api/error.rs
index a3cdfdbd..bd340fa6 100644
--- a/src/api/error.rs
+++ b/src/api/error.rs
@@ -72,6 +72,12 @@ impl From<roxmltree::Error> for Error {
 	}
 }
 
+impl From<quick_xml::de::DeError> for Error {
+	fn from(err: quick_xml::de::DeError) -> Self {
+		Self::InvalidXML(format!("{}", err))
+	}
+}
+
 impl Error {
 	/// Get the HTTP status code that best represents the meaning of the error for the client
 	pub fn http_status_code(&self) -> StatusCode {
diff --git a/src/api/s3_bucket.rs b/src/api/s3_bucket.rs
index cbefd005..d1a4425a 100644
--- a/src/api/s3_bucket.rs
+++ b/src/api/s3_bucket.rs
@@ -2,11 +2,62 @@ use std::fmt::Write;
 use std::sync::Arc;
 
 use hyper::{Body, Response};
+use quick_xml::se::to_string;
+use serde::Serialize;
 
 use garage_model::garage::Garage;
+use garage_model::key_table::Key;
+use garage_util::time::*;
 
 use crate::error::*;
 
+#[derive(Debug, Serialize, PartialEq)]
+struct CreationDate {
+	#[serde(rename = "$value")]
+	pub body: String,
+}
+#[derive(Debug, Serialize, PartialEq)]
+struct Name {
+	#[serde(rename = "$value")]
+	pub body: String,
+}
+#[derive(Debug, Serialize, PartialEq)]
+struct Bucket {
+	#[serde(rename = "CreationDate")]
+	pub creation_date: CreationDate,
+	#[serde(rename = "Name")]
+	pub name: Name,
+}
+#[derive(Debug, Serialize, PartialEq)]
+struct DisplayName {
+	#[serde(rename = "$value")]
+	pub body: String,
+}
+#[derive(Debug, Serialize, PartialEq)]
+struct ID {
+	#[serde(rename = "$value")]
+	pub body: String,
+}
+#[derive(Debug, Serialize, PartialEq)]
+struct Owner {
+	#[serde(rename = "DisplayName")]
+	display_name: DisplayName,
+	#[serde(rename = "ID")]
+	id: ID,
+}
+#[derive(Debug, Serialize, PartialEq)]
+struct BucketList {
+	#[serde(rename = "Bucket")]
+	pub entries: Vec<Bucket>,
+}
+#[derive(Debug, Serialize, PartialEq)]
+struct ListAllMyBucketsResult {
+	#[serde(rename = "Buckets")]
+	buckets: BucketList,
+	#[serde(rename = "Owner")]
+	owner: Owner,
+}
+
 pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>, Error> {
 	let mut xml = String::new();
 
@@ -22,3 +73,39 @@ pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>,
 		.header("Content-Type", "application/xml")
 		.body(Body::from(xml.into_bytes()))?)
 }
+
+pub fn handle_list_buckets(api_key: &Key) -> Result<Response<Body>, Error> {
+	let list_buckets = ListAllMyBucketsResult {
+		owner: Owner {
+			display_name: DisplayName {
+				body: api_key.name.get().to_string(),
+			},
+			id: ID {
+				body: api_key.key_id.to_string(),
+			},
+		},
+		buckets: BucketList {
+			entries: api_key
+				.authorized_buckets
+				.items()
+				.iter()
+				.map(|(name, ts, _)| Bucket {
+					creation_date: CreationDate {
+						body: msec_to_rfc3339(*ts),
+					},
+					name: Name {
+						body: name.to_string(),
+					},
+				})
+				.collect(),
+		},
+	};
+
+	let mut xml = r#"<?xml version="1.0" encoding="UTF-8"?>"#.to_string();
+	xml.push_str(&to_string(&list_buckets)?);
+	trace!("xml: {}", xml);
+
+	Ok(Response::builder()
+		.header("Content-Type", "application/xml")
+		.body(Body::from(xml))?)
+}
author	Quentin Dufour <quentin@deuxfleurs.fr>	2021-05-02 22:30:56 +0200
committer	Quentin Dufour <quentin@deuxfleurs.fr>	2021-05-02 22:30:56 +0200
commit	a8766626403c310d80139ab0660221f5d5aa4f22 (patch)
tree	d77d49f3c94ededa95200c1855ab422a19699fb9
parent	ef4d6e782a67bd422489d643e8807eaf7fb853c4 (diff)
download	garage-feature/s3/list-buckets.tar.gz garage-feature/s3/list-buckets.zip